VERSION
v"0.5.2"
# 24584 == 3073 * 8
bitstype 24584 CIFAR10Record
isbits(CIFAR10Record)
true
sizeof(CIFAR10Record)
3073
function Base.read(stream::IO, ::Type{CIFAR10Record})
bytes = read(stream, UInt8, 3073)
reinterpret(CIFAR10Record, bytes)[1]
end
※Assumed that https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz
has been downloaded and expanded.
record0 = open("cifar-10-batches-bin/test_batch.bin", "r") do f
return read(f, CIFAR10Record)
end;
record0
# => CIFAR10Record(0x6e817c91afb6a7917b82839198897664…a9a6a1a0a19f9e9fa29ca0a6a59f9e03)
CIFAR10Record
as Simple String representation¶function Base.show(io::IO, record::CIFAR10Record)
bytes = reinterpret(UInt8, [record])
## print(io, "CIFAR10Record($(repr(bytes[1])), $(repr(hash(bytes[2:end]))))")
print(io, "CIFAR10Record(")
# show 1st byte(=label)
show(io, bytes[1])
print(io, ", ")
# show hashcode of the rest of bytes(=image)
show(io, hash(bytes[2:end]))
print(io, ')')
end
record0
# => CIFAR10Record(0x03, 0xd0b45b812aae12b1)
CIFAR10Record
as Image¶const CRC32_TABLE = let poly::UInt32=0xedb88320
tab = zeros(UInt32, 256)
for i in 0:255
crc = UInt32(i)
for _ in 1:8
if (crc & 1) == 1
crc = (crc >> 1) $ poly
else
crc >>= 1
end
end
tab[i+1] = crc
end
tab
end;
function crc32(data::Vector{UInt8}, crc::UInt32=zero(UInt32))
crc = ~crc
for b in data
crc = CRC32_TABLE[(UInt8(crc & 0xff) $ b) + 1] $ (crc >> 8)
end
~crc
end
crc32 (generic function with 2 methods)
const MOD_ADLER = UInt32(65521)
function adler32(data::Vector{UInt8})
a = one(UInt32)
b = zero(UInt32)
l = length(data)
for i in 1:5550:l
e = min(i + 5549, l)
for v in data[i:e]
a += v
b += a
end
a %= MOD_ADLER
b %= MOD_ADLER
end
(b << 16) | a
end
adler32 (generic function with 1 method)
const PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n";
write_png_signature(io::IO) = write(io, PNG_SIGNATURE)
write_png_signature (generic function with 1 method)
const IHDR_00 = b"\0\0\0\rIHDR";
# True color (24bit-depth) RGB
const IHDR_10 = b"\b\x02\0\0\0";
function write_png_ihdr(io::IO, width::Int, height::Int)
# write IHDR_00, width, height, IHDR_10, crc to IO
c = write(io, IHDR_00)
crc = crc32(IHDR_00[5:end])
ihdrw = reinterpret(UInt8, [hton(width % UInt32)])
ihdrh = reinterpret(UInt8, [hton(height % UInt32)])
c += write(io, ihdrw)
crc = crc32(ihdrw, crc)
c += write(io, ihdrh)
crc = crc32(ihdrh, crc)
c += write(io, IHDR_10)
crc = crc32(IHDR_10, crc)
c += write(io, hton(crc))
c
end
write_png_ihdr (generic function with 1 method)
const IDAT_04 = b"IDAT";
# 圧縮方式+フラグ(Deflate, 圧縮レベル0)
const CMF_FLG = b"\b\x1d";
# Deflate ブロックヘッダ(最終ブロック、無圧縮)
const BH = b"\x01";
function write_png_idat(io::IO, img_src::AbstractArray{UInt8,3})
depth, width, height = size(img_src)
# @assert depth == 3
# write length, IDAT_04, CM_FLG, BH, LEN, NLEN, DAT, ADL, crc to IO
l = height * (1 + width * depth)
c = write(io, hton((l + 11) % UInt32))
c += write(io, IDAT_04)
crc = crc32(IDAT_04)
c += write(io, CMF_FLG)
crc = crc32(CMF_FLG, crc)
c += write(io, BH)
crc = crc32(BH, crc)
LEN = htol(l % UInt16)
c += write(io, LEN)
crc = crc32(reinterpret(UInt8, [LEN]), crc)
NLEN = ~LEN
c += write(io, NLEN)
crc = crc32(reinterpret(UInt8, [NLEN]), crc)
IDAT_DAT = vec([zeros(UInt8, 1, height);reshape(img_src, depth*width, height)])
c += write(io, IDAT_DAT)
crc = crc32(IDAT_DAT, crc)
ADL = hton(adler32(IDAT_DAT))
c += write(io, ADL)
crc = crc32(reinterpret(UInt8, [ADL]), crc)
c += write(io, hton(crc))
c
end
write_png_idat (generic function with 1 method)
※↑LEN
の制限から、あまり大きな画像はこの方法では処理できない(だいたい 147x147 くらいまで)
(CIFAR10の画像は 32x32 なので大丈夫)
const IEND = b"\0\0\0\0IEND\xaeB`\x82";
write_png_iend(io::IO) = write(io, IEND)
write_png_iend (generic function with 1 method)
function write_png(io::IO, record::CIFAR10Record)
img_src = permutedims(reshape(reinterpret(UInt8, [record])[2:end], (32, 32, 3)), (3, 1, 2))
c = write_png_signature(io)
c += write_png_ihdr(io, 32, 32)
c += write_png_idat(io, img_src)
c += write_png_iend(io)
c
end
write_png (generic function with 1 method)
open("sample1.png", "w") do f
write_png(f, record0)
end
3172
![sample1.png](sample1.png)
↓
Base.mimewritable(::MIME"image/png", ::CIFAR10Record) = true
function Base.show(io::IO, ::MIME"image/png", record::CIFAR10Record)
write_png(io, record)
end
record0
records5 = open("cifar-10-batches-bin/test_batch.bin", "r") do f
_randidxs = shuffle(0:9999)
_read(f::IO, idx::Int) = (seek(f, idx * 3073); read(f, CIFAR10Record))
return [_read(f, _randidxs[i]) for i=1:5]
end;
records5
5-element Array{CIFAR10Record,1}: CIFAR10Record(0x04, 0xd82b840aa56a565b) CIFAR10Record(0x08, 0xad0caad260edc6e5) CIFAR10Record(0x04, 0x4db2d981c6476102) CIFAR10Record(0x05, 0x882277a28e9cfeb2) CIFAR10Record(0x08, 0xb4a97d79abfeb709)
Base.mimewritable(::MIME"text/html", ::CIFAR10Record) = true
function Base.show(io::IO, ::MIME"text/html", record::CIFAR10Record)
print(io, "<img src=\"data:image/png;base64,")
iobuf = IOBuffer()
b64pipe = Base64EncodePipe(iobuf)
write_png(b64pipe, record)
write(io, read(seekstart(iobuf)))
print(io, "\">")
end
Base.mimewritable(::MIME"text/html", ::AbstractArray{CIFAR10Record}) = true
function Base.show(io::IO, mime::MIME"text/html", records::AbstractArray{CIFAR10Record})
print(io, "<table>")
for record in records
print(io, "<tr><td>")
show(io, mime, record)
print(io, "</td></tr>")
end
print(io, "</table>")
end
records5