# 24584 == 3073 * 8
primitive type CIFAR10Record 24584 end
# for v0.5.x
# bitstype 24584 CIFAR10Record
isbits(CIFAR10Record)
true
sizeof(CIFAR10Record)
3073
function Base.read(stream::IO, ::Type{CIFAR10Record})
bytes = read(stream, UInt8, 3073)
reinterpret(CIFAR10Record, bytes)[1]
end
※Assumed that https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz
has been downloaded and expanded.
record0 = open("cifar-10-batches-bin/test_batch.bin", "r") do f
return read(f, CIFAR10Record)
end;
record0
CIFAR10Record(0x6e817c91afb6a7917b8283919889766459626c686c656972737b8486868495a06b7e82a4bec5aa907f7f86989c90878078736c6a6d666873707d868b8f8494a87f6c92a8b5b39a857d8388898a8b898380898780756969728571848b928994b188678ca4b2978a87888a8b88878278767883878b82807977d39f7d8a8d8f93a18b8eb0b699858b93948b8a8c84827e7a77747b857c817e8ceae98a7e858a95a2a9b6b29e8a8c8d969a968d837f7b777777777781837b7299f7eaa47e7a8293a5b6a7989a959398968a86837b7571707173797f7f7d8076cdf9d18fa7887c8ca2958c90928e8e9186827d7d787173757574787977767faff5f2b589bfb086828d858b868789898b84807e7d7d74717873656b69696f99e4f1eda4aec0bcbca8878489848e8d8782837d736e6f65687794989ca2959de8fcf7d17196b6b1c0d58573707078777a6f6e6657443a3e6d8fb5c5d0c0a49390e1fa9f5b93b3a8b0c87c3f453d4c524b5345293336393f426594b1ad745649576fb9926fa0b4acafbb922e323440443c5d34322c4e7669674976a87b566c337e8782846398b8b9b5bbb42e2d363e3b283c27343a565c5d785d5d8564606c37506f8d8f6085bcc1bdc0cd364044404e4135333d73692040665f58624959595f693d6f916669bac2b8b4d72e372d37525e4d4861767e6d683a575c5a58758d9b852552717e5aacbcb2a0cc3030365d6363574450658a92764b5f65708289e6d3745d344d584d8eb5ae88c52f35506f685a5b53697a7faa95465f7d868c9fa9a9675822373e436dbcac70c4224551635e48464c506888ae93446d6b759dc89b7f677827323a4262c0a367c1184049595155504f536f8cb28c2f6d696281b7945e7e982c2f354485b99069c21a30374e515e584b52828ec071224f595f6a9f9785a4bc502c373b88b29974bd2e1f213b494f604d5c8a899a4d284d456868849b83bce2b32f33327aa89e7aaf392c0f2b474e4e4a59727a703a35474f4f6b7d9177a6c6923c2f337095698fa32d3b231b35434c45465d7b703d49524d4a6580794e3b5f4232292b52713b699027333220313a4a545851708c6b4552584f5d6b4d3133333127242a3829265c5a27303a2427454d675e5a6c84794a404839464b3b3427252a2b282b301c115039282c3a31252f566160747a786e4c2d2b1f2b3a4e552a27252929272c3539403624242e302d2f46435d69766f62381f32473c386ba43b2a292c2f282b2f3130292427272b2e2d27343242544c412f433a362f294b5f392d2c30302e2b2c1f20282122222327333730322f323431362f272325263339302b2b2d2d2c2a3024212f22201f22262d2e32303437353229211e212829343629262e2d2f2b31382d283321242426292b272c2b2d2d2c2b2726282c2d29313429292c2d2f292e35332f3143544d638792836d575f606f776857463a3f49454942454f51575c595659696b40525273919c826958585f71776d655f565049474a43485551585c5a5b556674554061748384705b53585e5f6163635e5c66645d52474d596b4f59575b58647c5d3c5a6e7d665d5a5c5d5f5c5c58504f536064675f5e6164c5855458585c616c60627e8165555d65665d5c615959575452505761575e687ee4db6a535659626e7d8a826c5b5f61696e6a615a585653545352525c5e575c8ff5e5915f53566372897b696c6c6d6e6c5f5c59565350515252535a5a585b5dc6fdd588946a555e6f675e63666567675c595556534f52545654535554545e9ff5fabd88b39966575b5a5f5e60605e5d5957595a565251514f484b4c505888e7fbf5a6aab6aea889595f61616c68605b5e5d55545452575d7c85858d848fe0fdfcd37092b1acb7c5695a5c606865635455514636313e6d85acc0c4b69c8d8cdef7995790b3a9b2c87869716d7b7d644739212e343d4d4e6494b6ad765a4d5a69af87679db5adb2c19c6b6f7179765b5a31333156817c774b76b08661763b86887c795a95b6b3b0bdc06f73787769453e293b44636d758b605c8f756f79405a80948b5883b8b5afb8d36e74705e5d463838457e78386080675b6e5b696669745d89956066b6b8a7a6d7686c58535b5d4f4c69828d8a8f5a65656a6a8498a38f50797e7d56aab6a492cb6860566e6d685848556c939f8658687079828aecdd83785e7d785389b2a781c55f5560746e605d566d8085af984764888f8ca5b4b97b6a4a7a77596dbaa76bc44959586564504950556e90b59847727583acd8ae957d8448747c6a6ebc9a5fc0374a4957575f5453587695bb933272727195caa672909a446d7b7b9cb48660c23837354b57685d50598b98c9792654626b77aba18eabb65f5d7b78a7af906cbf512e243b4f586653649393a5552c504c6f6c899f87bfe0c55976709ca99774b7674b1e314c555551627c857d44394a54556e839983b4d6b4767474959a648bb0676a3b233949524d4f67877d474e5551506f8f8d695987797a75717c793a69a16167532f353c515d625d7c99764d585f576a836e5e6d71777472756f402b60735d63623d2f47536f6663768e82544b5343576c626b787475767376754a255f645f6269543a3a5b6460747a797153383a324b667a8a76746f737471737064686d5d5e656053494e485e66726e674535506c6f729cd6807471747871727273726b5e63656662553e463f4b5a5a57506f7072736f82927b75737778756e706d6e6b595f626363676056554d4e5a606f72747172736f6d7273737575736a6f6f6d6e585b5f62616666696466686e73706f73737577716e6f6e747275717074726e70555b5f616162656b6a6b6b6d706f71757775756f6f74716f71736d7076746f7015221830596959432d3435424a3d2f2012131d191d161a24262828282b2d3836141d1a3c5d6d553c2b2b31454b423e382f241d1b1e171b2a282c2a272b23313d280d2638494f3e2a21272d3235393c38353a3831261b1f31472a2b25261f2a44330f1f2f3f2c282527282a2e312e292a2d363a3d35343640aa682e2921202332332e4343281c272f302727332e2f302f2d28303a30364161d3ca4e2c231d23324c554933242b2e363a362d2c2c2b2b2e2f2c2c3638313a7cf0e08042291f2334534632363b403e3b2e2b28282724292b2d2e3434323642bbffd7808049221e30332e32383b3e382d2a26272322272a2f32313332323d8df1fec284a6833b1d1d2d333537322e2e2a2b2e2e262a2e2a2d303131353c6ee3fdf6a5a5b2a89060292a2b303b332b3338383332343c4444687a747c727cd0fcfdd36d8cb1b0aeaf4f373b454e48453a3d3a3122233d6e7da8c2beb0958685dbf596538ab5b2b1c375787f8090906c3c2e18262e3c57596496bfb27a5d505d67ab836398b5b1b3c3a5959ea4aba3795a2f3433598689834974b8916c814690897a775690b1aeabbbc7979aa09b8556412c404b6b7584965c5593817c864d6887998d567cb0ada7b2d48c968f7466473a3c4b858044769065567366747276816a949c605eb0b7a4a3d690977d6b655e4f4e6d87949cab70696571718ca2ad9a608a8a7e4ea8bfa892cb8879677870695747577098a8905f6c79838791f2e68f82739b8c4e82b8ac84c5756164736d5f5b556e8289b3984767929a91acbec68a7364a79c6369baa86cc25e615a6463504850577295ba9a47757e8db4e2bda7919367a7ad8877b9985dbd494f4954565f54545b7b9bc2963273787ba0d8ba89a8a8619fb2a6aeaf7f5abd4c3e374955685e535d92a0d27d2654667586bdb9aac7c27586ada6bbaa8764bc683b293b4e5867576b9b9daf5b2d504e787d9cb7a4dcedd57a9f9bb1a58e6cb7845f28324b5457566a858f894c3b4a565d7d94ae9ccfedcf999c9eae9a5e86b489854a28384756535771928a5150545256799c9d7d71aea5a6a3a1997c3667aa888a6b39313655646a6688a37c4d555b58749082788ca1aba8a5a78f462a6383868d844f2e3f55736b697c928453484e44647f8193a4a6a9aaa8aa99582f6d7f868a8c6c433a5b635e72777a765942454261809db4a3a5a3a7a7a5a19382859481858a806955544a5d616a7071554e6f8e9297c3f6ada4a4a6a9a3a9aaa19c9b7f888c8c847153564a505c626768959ea4a29db1bfa7a5a5a9a9a7a7aea09b9b78828b8e8c8f86787265626e79919faaa9a7a6a19d9ea2a3a5a5a3a0a79e9797777d888f8e9193958d8c8b91979aa0a9ababaaa39f9b9ca3a2a4a0a2a69f9798747e898f8d8c8f959496959ca0a0a2a7aaa9a6a1a0a19f9e9fa29ca0a6a59f9e03)
CIFAR10Record
as Simple String representation¶function Base.show(io::IO, record::CIFAR10Record)
bytes = reinterpret(UInt8, [record])
## print(io, "CIFAR10Record($(repr(bytes[1])), $(repr(hash(bytes[2:end]))))")
print(io, "CIFAR10Record(")
# show 1st byte(=label)
show(io, bytes[1])
print(io, ", ")
# show hashcode of the rest of bytes(=image)
show(io, hash(bytes[2:end]))
print(io, ')')
end
record0
CIFAR10Record(0x03, 0xd0b45b812aae12b1)
CIFAR10Record
as Image¶const CRC32_TABLE = let poly::UInt32=0xedb88320
tab = zeros(UInt32, 256)
for i in 0:255
crc = UInt32(i)
for _ in 1:8
if (crc & 1) == 1
crc = (crc >> 1) ⊻ poly
else
crc >>= 1
end
end
tab[i+1] = crc
end
tab
end;
function crc32(data::Vector{UInt8}, crc::UInt32=zero(UInt32))
crc = ~crc
for b in data
crc = CRC32_TABLE[(UInt8(crc & 0xff) ⊻ b) + 1] ⊻ (crc >> 8)
end
~crc
end
crc32 (generic function with 2 methods)
const MOD_ADLER = UInt32(65521)
function adler32(data::Vector{UInt8})
a = one(UInt32)
b = zero(UInt32)
l = length(data)
for i in 1:5550:l
e = min(i + 5549, l)
for v in data[i:e]
a += v
b += a
end
a %= MOD_ADLER
b %= MOD_ADLER
end
(b << 16) | a
end
adler32 (generic function with 1 method)
const PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n";
write_png_signature(io::IO) = write(io, PNG_SIGNATURE)
write_png_signature (generic function with 1 method)
const IHDR_00 = b"\0\0\0\rIHDR";
# True color (24bit-depth) RGB
const IHDR_10 = b"\b\x02\0\0\0";
function write_png_ihdr(io::IO, width::Int, height::Int)
# write IHDR_00, width, height, IHDR_10, crc to IO
c = write(io, IHDR_00)
crc = crc32(IHDR_00[5:end])
ihdrw = reinterpret(UInt8, [hton(width % UInt32)])
ihdrh = reinterpret(UInt8, [hton(height % UInt32)])
c += write(io, ihdrw)
crc = crc32(ihdrw, crc)
c += write(io, ihdrh)
crc = crc32(ihdrh, crc)
c += write(io, IHDR_10)
crc = crc32(IHDR_10, crc)
c += write(io, hton(crc))
c
end
write_png_ihdr (generic function with 1 method)
const IDAT_04 = b"IDAT";
# 圧縮方式+フラグ(Deflate, 圧縮レベル0)
const CMF_FLG = b"\b\x1d";
# Deflate ブロックヘッダ(最終ブロック、無圧縮)
const BH = b"\x01";
function write_png_idat(io::IO, img_src::AbstractArray{UInt8,3})
depth, width, height = size(img_src)
# @assert depth == 3
# write length, IDAT_04, CM_FLG, BH, LEN, NLEN, DAT, ADL, crc to IO
l = height * (1 + width * depth)
c = write(io, hton((l + 11) % UInt32))
c += write(io, IDAT_04)
crc = crc32(IDAT_04)
c += write(io, CMF_FLG)
crc = crc32(CMF_FLG, crc)
c += write(io, BH)
crc = crc32(BH, crc)
LEN = htol(l % UInt16)
c += write(io, LEN)
crc = crc32(reinterpret(UInt8, [LEN]), crc)
NLEN = ~LEN
c += write(io, NLEN)
crc = crc32(reinterpret(UInt8, [NLEN]), crc)
IDAT_DAT = vec([zeros(UInt8, 1, height);reshape(img_src, :, height)])
c += write(io, IDAT_DAT)
crc = crc32(IDAT_DAT, crc)
ADL = hton(adler32(IDAT_DAT))
c += write(io, ADL)
crc = crc32(reinterpret(UInt8, [ADL]), crc)
c += write(io, hton(crc))
c
end
write_png_idat (generic function with 1 method)
※↑LEN
の制限から、あまり大きな画像はこの方法では処理できない(だいたい 147x147 くらいまで)
(CIFAR10の画像は 32x32 なので大丈夫)
const IEND = b"\0\0\0\0IEND\xaeB`\x82";
write_png_iend(io::IO) = write(io, IEND)
write_png_iend (generic function with 1 method)
function write_png(io::IO, record::CIFAR10Record)
img_src = permutedims(reshape(reinterpret(UInt8, [record])[2:end], (32, 32, 3)), (3, 1, 2))
c = write_png_signature(io)
c += write_png_ihdr(io, 32, 32)
c += write_png_idat(io, img_src)
c += write_png_iend(io)
c
end
write_png (generic function with 1 method)
open("sample1.png", "w") do f
write_png(f, record0)
end
3172
![sample1.png](sample1.png)
↓
Base.mimewritable(::MIME"image/png", ::CIFAR10Record) = true
function Base.show(io::IO, ::MIME"image/png", record::CIFAR10Record)
write_png(io, record)
end
record0
records5 = open("cifar-10-batches-bin/test_batch.bin", "r") do f
_randidxs = shuffle(0:9999)
_read(f::IO, idx::Int) = (seek(f, idx * 3073); read(f, CIFAR10Record))
return [_read(f, _randidxs[i]) for i=1:5]
end;
records5
5-element Array{CIFAR10Record,1}: CIFAR10Record(0x08, 0xba65649d8b8a8b2b) CIFAR10Record(0x00, 0xf0209500d9acf5a2) CIFAR10Record(0x06, 0xdf512a631374e8de) CIFAR10Record(0x07, 0x4adfc7cb47e7111e) CIFAR10Record(0x00, 0x310f064e20cace12)
Base.mimewritable(::MIME"text/html", ::CIFAR10Record) = true
function Base.show(io::IO, ::MIME"text/html", record::CIFAR10Record)
print(io, "<img src=\"data:image/png;base64,")
iobuf = IOBuffer()
b64pipe = Base64EncodePipe(iobuf)
write_png(b64pipe, record)
write(io, read(seekstart(iobuf)))
print(io, "\">")
end
Base.mimewritable(::MIME"text/html", ::AbstractArray{CIFAR10Record}) = true
function Base.show(io::IO, mime::MIME"text/html", records::AbstractArray{CIFAR10Record})
print(io, "<table>")
for record in records
print(io, "<tr><td>")
show(io, mime, record)
print(io, "</td></tr>")
end
print(io, "</table>")
end
records5