Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.
%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -p torch
Sebastian Raschka CPython 3.7.1 IPython 7.2.0 torch 1.0.0
A simple variational autoencoder that compresses 768-pixel MNIST images down to a 15-pixel latent vector representation.
import time
import numpy as np
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision import transforms
if torch.cuda.is_available():
torch.backends.cudnn.deterministic = True
##########################
### SETTINGS
##########################
# Device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('Device:', device)
# Hyperparameters
random_seed = 0
learning_rate = 0.001
num_epochs = 50
batch_size = 128
# Architecture
num_features = 784
num_hidden_1 = 500
num_latent = 15
##########################
### MNIST DATASET
##########################
# Note transforms.ToTensor() scales input images
# to 0-1 range
train_dataset = datasets.MNIST(root='data',
train=True,
transform=transforms.ToTensor(),
download=True)
test_dataset = datasets.MNIST(root='data',
train=False,
transform=transforms.ToTensor())
train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
test_loader = DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)
# Checking the dataset
for images, labels in train_loader:
print('Image batch dimensions:', images.shape)
print('Image label dimensions:', labels.shape)
break
Device: cuda:0 Image batch dimensions: torch.Size([128, 1, 28, 28]) Image label dimensions: torch.Size([128])
##########################
### MODEL
##########################
class VariationalAutoencoder(torch.nn.Module):
def __init__(self, num_features, num_hidden_1, num_latent):
super(VariationalAutoencoder, self).__init__()
### ENCODER
self.hidden_1 = torch.nn.Linear(num_features, num_hidden_1)
self.z_mean = torch.nn.Linear(num_hidden_1, num_latent)
# in the original paper (Kingma & Welling 2015, we use
# have a z_mean and z_var, but the problem is that
# the z_var can be negative, which would cause issues
# in the log later. Hence we assume that latent vector
# has a z_mean and z_log_var component, and when we need
# the regular variance or std_dev, we simply use
# an exponential function
self.z_log_var = torch.nn.Linear(num_hidden_1, num_latent)
### DECODER
self.linear_3 = torch.nn.Linear(num_latent, num_hidden_1)
self.linear_4 = torch.nn.Linear(num_hidden_1, num_features)
def reparameterize(self, z_mu, z_log_var):
# Sample epsilon from standard normal distribution
eps = torch.randn(z_mu.size(0), z_mu.size(1)).to(device)
# note that log(x^2) = 2*log(x); hence divide by 2 to get std_dev
# i.e., std_dev = exp(log(std_dev^2)/2) = exp(log(var)/2)
z = z_mu + eps * torch.exp(z_log_var/2.)
return z
def encoder(self, features):
x = self.hidden_1(features)
x = F.leaky_relu(x, negative_slope=0.0001)
z_mean = self.z_mean(x)
z_log_var = self.z_log_var(x)
encoded = self.reparameterize(z_mean, z_log_var)
return z_mean, z_log_var, encoded
def decoder(self, encoded):
x = self.linear_3(encoded)
x = F.leaky_relu(x, negative_slope=0.0001)
x = self.linear_4(x)
decoded = torch.sigmoid(x)
return decoded
def forward(self, features):
z_mean, z_log_var, encoded = self.encoder(features)
decoded = self.decoder(encoded)
return z_mean, z_log_var, encoded, decoded
torch.manual_seed(random_seed)
model = VariationalAutoencoder(num_features,
num_hidden_1,
num_latent)
model = model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
start_time = time.time()
for epoch in range(num_epochs):
for batch_idx, (features, targets) in enumerate(train_loader):
# don't need labels, only the images (features)
features = features.view(-1, 28*28).to(device)
### FORWARD AND BACK PROP
z_mean, z_log_var, encoded, decoded = model(features)
# cost = reconstruction loss + Kullback-Leibler divergence
kl_divergence = (0.5 * (z_mean**2 +
torch.exp(z_log_var) - z_log_var - 1)).sum()
pixelwise_bce = F.binary_cross_entropy(decoded, features, reduction='sum')
cost = kl_divergence + pixelwise_bce
optimizer.zero_grad()
cost.backward()
### UPDATE MODEL PARAMETERS
optimizer.step()
### LOGGING
if not batch_idx % 50:
print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f'
%(epoch+1, num_epochs, batch_idx,
len(train_loader), cost))
print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))
print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))
Epoch: 001/050 | Batch 000/469 | Cost: 70481.2422 Epoch: 001/050 | Batch 050/469 | Cost: 27139.5547 Epoch: 001/050 | Batch 100/469 | Cost: 22833.3730 Epoch: 001/050 | Batch 150/469 | Cost: 19493.1523 Epoch: 001/050 | Batch 200/469 | Cost: 18727.4688 Epoch: 001/050 | Batch 250/469 | Cost: 18074.2676 Epoch: 001/050 | Batch 300/469 | Cost: 16633.2852 Epoch: 001/050 | Batch 350/469 | Cost: 17136.7852 Epoch: 001/050 | Batch 400/469 | Cost: 16402.5293 Epoch: 001/050 | Batch 450/469 | Cost: 16062.9814 Epoch: 002/050 | Batch 000/469 | Cost: 16577.0840 Epoch: 002/050 | Batch 050/469 | Cost: 15451.8242 Epoch: 002/050 | Batch 100/469 | Cost: 15667.8535 Epoch: 002/050 | Batch 150/469 | Cost: 15734.0801 Epoch: 002/050 | Batch 200/469 | Cost: 15145.4365 Epoch: 002/050 | Batch 250/469 | Cost: 15326.6953 Epoch: 002/050 | Batch 300/469 | Cost: 15408.5801 Epoch: 002/050 | Batch 350/469 | Cost: 15637.5430 Epoch: 002/050 | Batch 400/469 | Cost: 14793.0332 Epoch: 002/050 | Batch 450/469 | Cost: 15046.4414 Epoch: 003/050 | Batch 000/469 | Cost: 14457.0537 Epoch: 003/050 | Batch 050/469 | Cost: 14483.2910 Epoch: 003/050 | Batch 100/469 | Cost: 14374.9258 Epoch: 003/050 | Batch 150/469 | Cost: 13934.8672 Epoch: 003/050 | Batch 200/469 | Cost: 15053.9336 Epoch: 003/050 | Batch 250/469 | Cost: 14673.1025 Epoch: 003/050 | Batch 300/469 | Cost: 14324.3916 Epoch: 003/050 | Batch 350/469 | Cost: 14318.4229 Epoch: 003/050 | Batch 400/469 | Cost: 14501.4912 Epoch: 003/050 | Batch 450/469 | Cost: 13753.9082 Epoch: 004/050 | Batch 000/469 | Cost: 15024.3789 Epoch: 004/050 | Batch 050/469 | Cost: 14310.9219 Epoch: 004/050 | Batch 100/469 | Cost: 14723.5176 Epoch: 004/050 | Batch 150/469 | Cost: 15469.9473 Epoch: 004/050 | Batch 200/469 | Cost: 14126.0586 Epoch: 004/050 | Batch 250/469 | Cost: 14321.4062 Epoch: 004/050 | Batch 300/469 | Cost: 13834.0576 Epoch: 004/050 | Batch 350/469 | Cost: 14363.6494 Epoch: 004/050 | Batch 400/469 | Cost: 14136.7422 Epoch: 004/050 | Batch 450/469 | Cost: 13603.2012 Epoch: 005/050 | Batch 000/469 | Cost: 14002.0479 Epoch: 005/050 | Batch 050/469 | Cost: 14221.5488 Epoch: 005/050 | Batch 100/469 | Cost: 13972.6787 Epoch: 005/050 | Batch 150/469 | Cost: 13918.2402 Epoch: 005/050 | Batch 200/469 | Cost: 13839.3809 Epoch: 005/050 | Batch 250/469 | Cost: 14421.0020 Epoch: 005/050 | Batch 300/469 | Cost: 14611.1816 Epoch: 005/050 | Batch 350/469 | Cost: 13653.8027 Epoch: 005/050 | Batch 400/469 | Cost: 13632.8047 Epoch: 005/050 | Batch 450/469 | Cost: 13612.9375 Epoch: 006/050 | Batch 000/469 | Cost: 13993.7344 Epoch: 006/050 | Batch 050/469 | Cost: 13976.1006 Epoch: 006/050 | Batch 100/469 | Cost: 14309.5527 Epoch: 006/050 | Batch 150/469 | Cost: 13427.8916 Epoch: 006/050 | Batch 200/469 | Cost: 13811.6260 Epoch: 006/050 | Batch 250/469 | Cost: 14130.3496 Epoch: 006/050 | Batch 300/469 | Cost: 12895.7324 Epoch: 006/050 | Batch 350/469 | Cost: 13445.3213 Epoch: 006/050 | Batch 400/469 | Cost: 13374.8242 Epoch: 006/050 | Batch 450/469 | Cost: 13549.5098 Epoch: 007/050 | Batch 000/469 | Cost: 13913.4043 Epoch: 007/050 | Batch 050/469 | Cost: 13703.5654 Epoch: 007/050 | Batch 100/469 | Cost: 14132.1758 Epoch: 007/050 | Batch 150/469 | Cost: 14052.9814 Epoch: 007/050 | Batch 200/469 | Cost: 13750.3535 Epoch: 007/050 | Batch 250/469 | Cost: 14316.6953 Epoch: 007/050 | Batch 300/469 | Cost: 13224.3281 Epoch: 007/050 | Batch 350/469 | Cost: 14139.7979 Epoch: 007/050 | Batch 400/469 | Cost: 13795.6016 Epoch: 007/050 | Batch 450/469 | Cost: 13915.5020 Epoch: 008/050 | Batch 000/469 | Cost: 13548.9512 Epoch: 008/050 | Batch 050/469 | Cost: 13558.6338 Epoch: 008/050 | Batch 100/469 | Cost: 13883.1074 Epoch: 008/050 | Batch 150/469 | Cost: 13128.7617 Epoch: 008/050 | Batch 200/469 | Cost: 13133.5879 Epoch: 008/050 | Batch 250/469 | Cost: 13518.8672 Epoch: 008/050 | Batch 300/469 | Cost: 13679.2324 Epoch: 008/050 | Batch 350/469 | Cost: 13928.9824 Epoch: 008/050 | Batch 400/469 | Cost: 14079.2256 Epoch: 008/050 | Batch 450/469 | Cost: 13294.2021 Epoch: 009/050 | Batch 000/469 | Cost: 13619.6504 Epoch: 009/050 | Batch 050/469 | Cost: 13831.6201 Epoch: 009/050 | Batch 100/469 | Cost: 13848.1406 Epoch: 009/050 | Batch 150/469 | Cost: 14622.0889 Epoch: 009/050 | Batch 200/469 | Cost: 13843.3887 Epoch: 009/050 | Batch 250/469 | Cost: 13673.2441 Epoch: 009/050 | Batch 300/469 | Cost: 13646.6543 Epoch: 009/050 | Batch 350/469 | Cost: 13411.1816 Epoch: 009/050 | Batch 400/469 | Cost: 14463.7988 Epoch: 009/050 | Batch 450/469 | Cost: 13585.7891 Epoch: 010/050 | Batch 000/469 | Cost: 13929.6816 Epoch: 010/050 | Batch 050/469 | Cost: 13659.5176 Epoch: 010/050 | Batch 100/469 | Cost: 13504.2568 Epoch: 010/050 | Batch 150/469 | Cost: 13717.9434 Epoch: 010/050 | Batch 200/469 | Cost: 13711.8818 Epoch: 010/050 | Batch 250/469 | Cost: 13554.4062 Epoch: 010/050 | Batch 300/469 | Cost: 13317.5156 Epoch: 010/050 | Batch 350/469 | Cost: 13279.9912 Epoch: 010/050 | Batch 400/469 | Cost: 13069.9648 Epoch: 010/050 | Batch 450/469 | Cost: 13087.7695 Epoch: 011/050 | Batch 000/469 | Cost: 13800.6113 Epoch: 011/050 | Batch 050/469 | Cost: 13924.1973 Epoch: 011/050 | Batch 100/469 | Cost: 13173.4414 Epoch: 011/050 | Batch 150/469 | Cost: 13963.7402 Epoch: 011/050 | Batch 200/469 | Cost: 13682.3281 Epoch: 011/050 | Batch 250/469 | Cost: 13664.8027 Epoch: 011/050 | Batch 300/469 | Cost: 14188.4707 Epoch: 011/050 | Batch 350/469 | Cost: 13625.5840 Epoch: 011/050 | Batch 400/469 | Cost: 13482.8643 Epoch: 011/050 | Batch 450/469 | Cost: 13912.9238 Epoch: 012/050 | Batch 000/469 | Cost: 13048.4648 Epoch: 012/050 | Batch 050/469 | Cost: 13041.4395 Epoch: 012/050 | Batch 100/469 | Cost: 13212.3662 Epoch: 012/050 | Batch 150/469 | Cost: 13304.4463 Epoch: 012/050 | Batch 200/469 | Cost: 13445.9287 Epoch: 012/050 | Batch 250/469 | Cost: 13693.2676 Epoch: 012/050 | Batch 300/469 | Cost: 13072.9004 Epoch: 012/050 | Batch 350/469 | Cost: 13414.5361 Epoch: 012/050 | Batch 400/469 | Cost: 13669.4121 Epoch: 012/050 | Batch 450/469 | Cost: 13366.8633 Epoch: 013/050 | Batch 000/469 | Cost: 13785.0518 Epoch: 013/050 | Batch 050/469 | Cost: 13788.7734 Epoch: 013/050 | Batch 100/469 | Cost: 13442.9023 Epoch: 013/050 | Batch 150/469 | Cost: 13771.4902 Epoch: 013/050 | Batch 200/469 | Cost: 13357.2217 Epoch: 013/050 | Batch 250/469 | Cost: 13402.6758 Epoch: 013/050 | Batch 300/469 | Cost: 13852.4033 Epoch: 013/050 | Batch 350/469 | Cost: 13301.3457 Epoch: 013/050 | Batch 400/469 | Cost: 13379.6172 Epoch: 013/050 | Batch 450/469 | Cost: 14275.3047 Epoch: 014/050 | Batch 000/469 | Cost: 13433.3906 Epoch: 014/050 | Batch 050/469 | Cost: 13059.2354 Epoch: 014/050 | Batch 100/469 | Cost: 14031.3721 Epoch: 014/050 | Batch 150/469 | Cost: 13950.9883 Epoch: 014/050 | Batch 200/469 | Cost: 13684.6611 Epoch: 014/050 | Batch 250/469 | Cost: 13630.9336 Epoch: 014/050 | Batch 300/469 | Cost: 13527.6230 Epoch: 014/050 | Batch 350/469 | Cost: 13746.0527 Epoch: 014/050 | Batch 400/469 | Cost: 13490.6982 Epoch: 014/050 | Batch 450/469 | Cost: 13669.7402 Epoch: 015/050 | Batch 000/469 | Cost: 13422.4238 Epoch: 015/050 | Batch 050/469 | Cost: 13303.6855 Epoch: 015/050 | Batch 100/469 | Cost: 13421.2900 Epoch: 015/050 | Batch 150/469 | Cost: 13129.2764 Epoch: 015/050 | Batch 200/469 | Cost: 13276.9336 Epoch: 015/050 | Batch 250/469 | Cost: 13776.0889 Epoch: 015/050 | Batch 300/469 | Cost: 13634.2188 Epoch: 015/050 | Batch 350/469 | Cost: 13438.3828 Epoch: 015/050 | Batch 400/469 | Cost: 13401.1045 Epoch: 015/050 | Batch 450/469 | Cost: 13567.6631 Epoch: 016/050 | Batch 000/469 | Cost: 13150.5625 Epoch: 016/050 | Batch 050/469 | Cost: 13414.6553 Epoch: 016/050 | Batch 100/469 | Cost: 12979.0029 Epoch: 016/050 | Batch 150/469 | Cost: 13146.0801 Epoch: 016/050 | Batch 200/469 | Cost: 13257.3301 Epoch: 016/050 | Batch 250/469 | Cost: 14350.7471 Epoch: 016/050 | Batch 300/469 | Cost: 13836.4316 Epoch: 016/050 | Batch 350/469 | Cost: 13865.9902 Epoch: 016/050 | Batch 400/469 | Cost: 13237.3877 Epoch: 016/050 | Batch 450/469 | Cost: 13339.0303 Epoch: 017/050 | Batch 000/469 | Cost: 13266.2793 Epoch: 017/050 | Batch 050/469 | Cost: 13568.0957 Epoch: 017/050 | Batch 100/469 | Cost: 12923.4482 Epoch: 017/050 | Batch 150/469 | Cost: 14093.2070 Epoch: 017/050 | Batch 200/469 | Cost: 13326.7510 Epoch: 017/050 | Batch 250/469 | Cost: 13965.0625 Epoch: 017/050 | Batch 300/469 | Cost: 13380.1445 Epoch: 017/050 | Batch 350/469 | Cost: 13277.1875 Epoch: 017/050 | Batch 400/469 | Cost: 13872.7607 Epoch: 017/050 | Batch 450/469 | Cost: 13272.6797 Epoch: 018/050 | Batch 000/469 | Cost: 13048.2461 Epoch: 018/050 | Batch 050/469 | Cost: 13508.2314 Epoch: 018/050 | Batch 100/469 | Cost: 12814.9834 Epoch: 018/050 | Batch 150/469 | Cost: 13623.1924 Epoch: 018/050 | Batch 200/469 | Cost: 13246.6113 Epoch: 018/050 | Batch 250/469 | Cost: 13471.1328 Epoch: 018/050 | Batch 300/469 | Cost: 13271.7930 Epoch: 018/050 | Batch 350/469 | Cost: 13494.4883 Epoch: 018/050 | Batch 400/469 | Cost: 13280.4316 Epoch: 018/050 | Batch 450/469 | Cost: 13408.9775 Epoch: 019/050 | Batch 000/469 | Cost: 13347.4941 Epoch: 019/050 | Batch 050/469 | Cost: 13403.6006 Epoch: 019/050 | Batch 100/469 | Cost: 12944.8574 Epoch: 019/050 | Batch 150/469 | Cost: 13410.6201 Epoch: 019/050 | Batch 200/469 | Cost: 13398.5342 Epoch: 019/050 | Batch 250/469 | Cost: 13786.6992 Epoch: 019/050 | Batch 300/469 | Cost: 12185.1465 Epoch: 019/050 | Batch 350/469 | Cost: 13143.7744 Epoch: 019/050 | Batch 400/469 | Cost: 13101.7451 Epoch: 019/050 | Batch 450/469 | Cost: 13410.3252 Epoch: 020/050 | Batch 000/469 | Cost: 13459.7676 Epoch: 020/050 | Batch 050/469 | Cost: 13551.5127 Epoch: 020/050 | Batch 100/469 | Cost: 13246.3486 Epoch: 020/050 | Batch 150/469 | Cost: 13524.1133 Epoch: 020/050 | Batch 200/469 | Cost: 13695.5605 Epoch: 020/050 | Batch 250/469 | Cost: 13447.3887 Epoch: 020/050 | Batch 300/469 | Cost: 13389.4941 Epoch: 020/050 | Batch 350/469 | Cost: 13180.2422 Epoch: 020/050 | Batch 400/469 | Cost: 13606.3457 Epoch: 020/050 | Batch 450/469 | Cost: 13646.9355 Epoch: 021/050 | Batch 000/469 | Cost: 13557.3623 Epoch: 021/050 | Batch 050/469 | Cost: 13147.0098 Epoch: 021/050 | Batch 100/469 | Cost: 13287.7227 Epoch: 021/050 | Batch 150/469 | Cost: 12849.9639 Epoch: 021/050 | Batch 200/469 | Cost: 13058.6406 Epoch: 021/050 | Batch 250/469 | Cost: 13192.6367 Epoch: 021/050 | Batch 300/469 | Cost: 13393.4082 Epoch: 021/050 | Batch 350/469 | Cost: 13834.2705 Epoch: 021/050 | Batch 400/469 | Cost: 13503.1680 Epoch: 021/050 | Batch 450/469 | Cost: 13592.5518 Epoch: 022/050 | Batch 000/469 | Cost: 13658.5986 Epoch: 022/050 | Batch 050/469 | Cost: 13389.6855 Epoch: 022/050 | Batch 100/469 | Cost: 13313.9707 Epoch: 022/050 | Batch 150/469 | Cost: 13508.8438 Epoch: 022/050 | Batch 200/469 | Cost: 12984.4082 Epoch: 022/050 | Batch 250/469 | Cost: 13159.5137 Epoch: 022/050 | Batch 300/469 | Cost: 13195.3516 Epoch: 022/050 | Batch 350/469 | Cost: 13606.1777 Epoch: 022/050 | Batch 400/469 | Cost: 12865.0508 Epoch: 022/050 | Batch 450/469 | Cost: 13227.6514 Epoch: 023/050 | Batch 000/469 | Cost: 13067.6016 Epoch: 023/050 | Batch 050/469 | Cost: 13425.5498 Epoch: 023/050 | Batch 100/469 | Cost: 13016.2773 Epoch: 023/050 | Batch 150/469 | Cost: 13322.1260 Epoch: 023/050 | Batch 200/469 | Cost: 12861.3926 Epoch: 023/050 | Batch 250/469 | Cost: 13000.5967 Epoch: 023/050 | Batch 300/469 | Cost: 13761.4629 Epoch: 023/050 | Batch 350/469 | Cost: 13482.9814 Epoch: 023/050 | Batch 400/469 | Cost: 12838.6201 Epoch: 023/050 | Batch 450/469 | Cost: 13252.9746 Epoch: 024/050 | Batch 000/469 | Cost: 13445.4590 Epoch: 024/050 | Batch 050/469 | Cost: 13583.6416 Epoch: 024/050 | Batch 100/469 | Cost: 13507.9443 Epoch: 024/050 | Batch 150/469 | Cost: 13385.5938 Epoch: 024/050 | Batch 200/469 | Cost: 13271.1357 Epoch: 024/050 | Batch 250/469 | Cost: 13643.7109 Epoch: 024/050 | Batch 300/469 | Cost: 13713.0889 Epoch: 024/050 | Batch 350/469 | Cost: 12844.5703 Epoch: 024/050 | Batch 400/469 | Cost: 12984.4746 Epoch: 024/050 | Batch 450/469 | Cost: 13015.4365 Epoch: 025/050 | Batch 000/469 | Cost: 13575.6875 Epoch: 025/050 | Batch 050/469 | Cost: 13195.2832 Epoch: 025/050 | Batch 100/469 | Cost: 13478.4746 Epoch: 025/050 | Batch 150/469 | Cost: 13194.2852 Epoch: 025/050 | Batch 200/469 | Cost: 12877.8242 Epoch: 025/050 | Batch 250/469 | Cost: 13061.2148 Epoch: 025/050 | Batch 300/469 | Cost: 13397.2266 Epoch: 025/050 | Batch 350/469 | Cost: 12763.3711 Epoch: 025/050 | Batch 400/469 | Cost: 13262.5332 Epoch: 025/050 | Batch 450/469 | Cost: 13390.2393 Epoch: 026/050 | Batch 000/469 | Cost: 13211.5508 Epoch: 026/050 | Batch 050/469 | Cost: 13458.3652 Epoch: 026/050 | Batch 100/469 | Cost: 12846.0557 Epoch: 026/050 | Batch 150/469 | Cost: 12842.9570 Epoch: 026/050 | Batch 200/469 | Cost: 13594.9395 Epoch: 026/050 | Batch 250/469 | Cost: 13021.0605 Epoch: 026/050 | Batch 300/469 | Cost: 13126.7686 Epoch: 026/050 | Batch 350/469 | Cost: 12951.5898 Epoch: 026/050 | Batch 400/469 | Cost: 13600.2119 Epoch: 026/050 | Batch 450/469 | Cost: 13313.3535 Epoch: 027/050 | Batch 000/469 | Cost: 12835.4717 Epoch: 027/050 | Batch 050/469 | Cost: 12731.1875 Epoch: 027/050 | Batch 100/469 | Cost: 13234.9297 Epoch: 027/050 | Batch 150/469 | Cost: 13105.2148 Epoch: 027/050 | Batch 200/469 | Cost: 13234.0684 Epoch: 027/050 | Batch 250/469 | Cost: 13147.0801 Epoch: 027/050 | Batch 300/469 | Cost: 13271.8262 Epoch: 027/050 | Batch 350/469 | Cost: 12936.5947 Epoch: 027/050 | Batch 400/469 | Cost: 13336.0293 Epoch: 027/050 | Batch 450/469 | Cost: 13387.3662 Epoch: 028/050 | Batch 000/469 | Cost: 13452.3438 Epoch: 028/050 | Batch 050/469 | Cost: 13245.0342 Epoch: 028/050 | Batch 100/469 | Cost: 13007.5234 Epoch: 028/050 | Batch 150/469 | Cost: 13068.0166 Epoch: 028/050 | Batch 200/469 | Cost: 12575.0166 Epoch: 028/050 | Batch 250/469 | Cost: 13051.9434 Epoch: 028/050 | Batch 300/469 | Cost: 13185.3330 Epoch: 028/050 | Batch 350/469 | Cost: 13587.7715 Epoch: 028/050 | Batch 400/469 | Cost: 12877.1436 Epoch: 028/050 | Batch 450/469 | Cost: 13305.9297 Epoch: 029/050 | Batch 000/469 | Cost: 13244.1865 Epoch: 029/050 | Batch 050/469 | Cost: 13002.6309 Epoch: 029/050 | Batch 100/469 | Cost: 13432.6504 Epoch: 029/050 | Batch 150/469 | Cost: 13128.8027 Epoch: 029/050 | Batch 200/469 | Cost: 12879.6543 Epoch: 029/050 | Batch 250/469 | Cost: 13248.0068 Epoch: 029/050 | Batch 300/469 | Cost: 13176.9912 Epoch: 029/050 | Batch 350/469 | Cost: 13055.7490 Epoch: 029/050 | Batch 400/469 | Cost: 13092.9580 Epoch: 029/050 | Batch 450/469 | Cost: 13179.1875 Epoch: 030/050 | Batch 000/469 | Cost: 13205.4668 Epoch: 030/050 | Batch 050/469 | Cost: 13425.4883 Epoch: 030/050 | Batch 100/469 | Cost: 12924.2070 Epoch: 030/050 | Batch 150/469 | Cost: 13293.8105 Epoch: 030/050 | Batch 200/469 | Cost: 12805.0674 Epoch: 030/050 | Batch 250/469 | Cost: 12823.4629 Epoch: 030/050 | Batch 300/469 | Cost: 12680.0322 Epoch: 030/050 | Batch 350/469 | Cost: 13412.4023 Epoch: 030/050 | Batch 400/469 | Cost: 13796.5479 Epoch: 030/050 | Batch 450/469 | Cost: 13084.7051 Epoch: 031/050 | Batch 000/469 | Cost: 13054.2988 Epoch: 031/050 | Batch 050/469 | Cost: 13315.4570 Epoch: 031/050 | Batch 100/469 | Cost: 13284.9463 Epoch: 031/050 | Batch 150/469 | Cost: 13184.4668 Epoch: 031/050 | Batch 200/469 | Cost: 13099.4189 Epoch: 031/050 | Batch 250/469 | Cost: 13391.0918 Epoch: 031/050 | Batch 300/469 | Cost: 13057.3223 Epoch: 031/050 | Batch 350/469 | Cost: 13442.3750 Epoch: 031/050 | Batch 400/469 | Cost: 13491.5635 Epoch: 031/050 | Batch 450/469 | Cost: 13054.0693 Epoch: 032/050 | Batch 000/469 | Cost: 13219.3789 Epoch: 032/050 | Batch 050/469 | Cost: 12822.7051 Epoch: 032/050 | Batch 100/469 | Cost: 13439.6436 Epoch: 032/050 | Batch 150/469 | Cost: 12843.7061 Epoch: 032/050 | Batch 200/469 | Cost: 13097.7012 Epoch: 032/050 | Batch 250/469 | Cost: 12950.4707 Epoch: 032/050 | Batch 300/469 | Cost: 13238.1094 Epoch: 032/050 | Batch 350/469 | Cost: 13027.9121 Epoch: 032/050 | Batch 400/469 | Cost: 13150.9277 Epoch: 032/050 | Batch 450/469 | Cost: 13239.6348 Epoch: 033/050 | Batch 000/469 | Cost: 12967.9863 Epoch: 033/050 | Batch 050/469 | Cost: 13261.3467 Epoch: 033/050 | Batch 100/469 | Cost: 13218.9023 Epoch: 033/050 | Batch 150/469 | Cost: 13092.8994 Epoch: 033/050 | Batch 200/469 | Cost: 12983.0459 Epoch: 033/050 | Batch 250/469 | Cost: 13031.2188 Epoch: 033/050 | Batch 300/469 | Cost: 12894.7129 Epoch: 033/050 | Batch 350/469 | Cost: 13563.2578 Epoch: 033/050 | Batch 400/469 | Cost: 13094.8340 Epoch: 033/050 | Batch 450/469 | Cost: 13279.9639 Epoch: 034/050 | Batch 000/469 | Cost: 12986.0615 Epoch: 034/050 | Batch 050/469 | Cost: 12981.4004 Epoch: 034/050 | Batch 100/469 | Cost: 13308.1504 Epoch: 034/050 | Batch 150/469 | Cost: 13338.7227 Epoch: 034/050 | Batch 200/469 | Cost: 13310.7227 Epoch: 034/050 | Batch 250/469 | Cost: 13158.7334 Epoch: 034/050 | Batch 300/469 | Cost: 13248.9336 Epoch: 034/050 | Batch 350/469 | Cost: 13256.2227 Epoch: 034/050 | Batch 400/469 | Cost: 12818.7148 Epoch: 034/050 | Batch 450/469 | Cost: 12835.6738 Epoch: 035/050 | Batch 000/469 | Cost: 12766.6123 Epoch: 035/050 | Batch 050/469 | Cost: 12521.5166 Epoch: 035/050 | Batch 100/469 | Cost: 12340.0430 Epoch: 035/050 | Batch 150/469 | Cost: 12873.6191 Epoch: 035/050 | Batch 200/469 | Cost: 13027.2266 Epoch: 035/050 | Batch 250/469 | Cost: 13575.8379 Epoch: 035/050 | Batch 300/469 | Cost: 13458.8867 Epoch: 035/050 | Batch 350/469 | Cost: 12816.4980 Epoch: 035/050 | Batch 400/469 | Cost: 12663.2207 Epoch: 035/050 | Batch 450/469 | Cost: 12733.6777 Epoch: 036/050 | Batch 000/469 | Cost: 13078.8682 Epoch: 036/050 | Batch 050/469 | Cost: 13072.0742 Epoch: 036/050 | Batch 100/469 | Cost: 12666.5215 Epoch: 036/050 | Batch 150/469 | Cost: 13091.2852 Epoch: 036/050 | Batch 200/469 | Cost: 13462.2529 Epoch: 036/050 | Batch 250/469 | Cost: 12630.4287 Epoch: 036/050 | Batch 300/469 | Cost: 13213.3223 Epoch: 036/050 | Batch 350/469 | Cost: 13298.2490 Epoch: 036/050 | Batch 400/469 | Cost: 12989.6328 Epoch: 036/050 | Batch 450/469 | Cost: 12918.6348 Epoch: 037/050 | Batch 000/469 | Cost: 12605.0732 Epoch: 037/050 | Batch 050/469 | Cost: 13055.5742 Epoch: 037/050 | Batch 100/469 | Cost: 12719.5420 Epoch: 037/050 | Batch 150/469 | Cost: 12599.2461 Epoch: 037/050 | Batch 200/469 | Cost: 12545.8223 Epoch: 037/050 | Batch 250/469 | Cost: 12449.0918 Epoch: 037/050 | Batch 300/469 | Cost: 13342.7930 Epoch: 037/050 | Batch 350/469 | Cost: 13066.0029 Epoch: 037/050 | Batch 400/469 | Cost: 13258.0957 Epoch: 037/050 | Batch 450/469 | Cost: 13180.6914 Epoch: 038/050 | Batch 000/469 | Cost: 12527.9854 Epoch: 038/050 | Batch 050/469 | Cost: 13618.6875 Epoch: 038/050 | Batch 100/469 | Cost: 13039.2627 Epoch: 038/050 | Batch 150/469 | Cost: 13062.9453 Epoch: 038/050 | Batch 200/469 | Cost: 13139.8945 Epoch: 038/050 | Batch 250/469 | Cost: 13168.6621 Epoch: 038/050 | Batch 300/469 | Cost: 12623.4629 Epoch: 038/050 | Batch 350/469 | Cost: 12757.8447 Epoch: 038/050 | Batch 400/469 | Cost: 12830.0762 Epoch: 038/050 | Batch 450/469 | Cost: 12733.7969 Epoch: 039/050 | Batch 000/469 | Cost: 12927.6729 Epoch: 039/050 | Batch 050/469 | Cost: 13016.1133 Epoch: 039/050 | Batch 100/469 | Cost: 12955.1621 Epoch: 039/050 | Batch 150/469 | Cost: 12945.2852 Epoch: 039/050 | Batch 200/469 | Cost: 12680.2188 Epoch: 039/050 | Batch 250/469 | Cost: 12958.4688 Epoch: 039/050 | Batch 300/469 | Cost: 13075.4912 Epoch: 039/050 | Batch 350/469 | Cost: 12962.3750 Epoch: 039/050 | Batch 400/469 | Cost: 12863.8867 Epoch: 039/050 | Batch 450/469 | Cost: 13399.3818 Epoch: 040/050 | Batch 000/469 | Cost: 12694.0283 Epoch: 040/050 | Batch 050/469 | Cost: 13524.2754 Epoch: 040/050 | Batch 100/469 | Cost: 12840.9316 Epoch: 040/050 | Batch 150/469 | Cost: 12661.5918 Epoch: 040/050 | Batch 200/469 | Cost: 13256.4902 Epoch: 040/050 | Batch 250/469 | Cost: 13027.6816 Epoch: 040/050 | Batch 300/469 | Cost: 12941.4727 Epoch: 040/050 | Batch 350/469 | Cost: 12656.1348 Epoch: 040/050 | Batch 400/469 | Cost: 12979.4785 Epoch: 040/050 | Batch 450/469 | Cost: 12705.2158 Epoch: 041/050 | Batch 000/469 | Cost: 12759.9707 Epoch: 041/050 | Batch 050/469 | Cost: 12406.5781 Epoch: 041/050 | Batch 100/469 | Cost: 12696.9307 Epoch: 041/050 | Batch 150/469 | Cost: 13398.3613 Epoch: 041/050 | Batch 200/469 | Cost: 12777.3418 Epoch: 041/050 | Batch 250/469 | Cost: 12854.2783 Epoch: 041/050 | Batch 300/469 | Cost: 13037.7236 Epoch: 041/050 | Batch 350/469 | Cost: 13410.0801 Epoch: 041/050 | Batch 400/469 | Cost: 13350.9121 Epoch: 041/050 | Batch 450/469 | Cost: 12898.7432 Epoch: 042/050 | Batch 000/469 | Cost: 12766.4316 Epoch: 042/050 | Batch 050/469 | Cost: 13303.4766 Epoch: 042/050 | Batch 100/469 | Cost: 13112.1465 Epoch: 042/050 | Batch 150/469 | Cost: 12951.8428 Epoch: 042/050 | Batch 200/469 | Cost: 13367.4814 Epoch: 042/050 | Batch 250/469 | Cost: 13274.8955 Epoch: 042/050 | Batch 300/469 | Cost: 12908.4805 Epoch: 042/050 | Batch 350/469 | Cost: 12858.0723 Epoch: 042/050 | Batch 400/469 | Cost: 13279.5410 Epoch: 042/050 | Batch 450/469 | Cost: 12669.7422 Epoch: 043/050 | Batch 000/469 | Cost: 13124.0225 Epoch: 043/050 | Batch 050/469 | Cost: 12976.3857 Epoch: 043/050 | Batch 100/469 | Cost: 12655.5703 Epoch: 043/050 | Batch 150/469 | Cost: 12876.7061 Epoch: 043/050 | Batch 200/469 | Cost: 13277.1592 Epoch: 043/050 | Batch 250/469 | Cost: 12657.5117 Epoch: 043/050 | Batch 300/469 | Cost: 12915.8867 Epoch: 043/050 | Batch 350/469 | Cost: 13254.9941 Epoch: 043/050 | Batch 400/469 | Cost: 12649.9102 Epoch: 043/050 | Batch 450/469 | Cost: 13198.5771 Epoch: 044/050 | Batch 000/469 | Cost: 13573.9121 Epoch: 044/050 | Batch 050/469 | Cost: 12972.9453 Epoch: 044/050 | Batch 100/469 | Cost: 12764.2188 Epoch: 044/050 | Batch 150/469 | Cost: 13482.2910 Epoch: 044/050 | Batch 200/469 | Cost: 13304.8975 Epoch: 044/050 | Batch 250/469 | Cost: 13446.4141 Epoch: 044/050 | Batch 300/469 | Cost: 13096.8887 Epoch: 044/050 | Batch 350/469 | Cost: 13551.5537 Epoch: 044/050 | Batch 400/469 | Cost: 12693.8301 Epoch: 044/050 | Batch 450/469 | Cost: 12812.8682 Epoch: 045/050 | Batch 000/469 | Cost: 12698.2207 Epoch: 045/050 | Batch 050/469 | Cost: 12705.6787 Epoch: 045/050 | Batch 100/469 | Cost: 13046.6162 Epoch: 045/050 | Batch 150/469 | Cost: 13085.3457 Epoch: 045/050 | Batch 200/469 | Cost: 12922.5312 Epoch: 045/050 | Batch 250/469 | Cost: 13367.9189 Epoch: 045/050 | Batch 300/469 | Cost: 12917.3457 Epoch: 045/050 | Batch 350/469 | Cost: 12896.9463 Epoch: 045/050 | Batch 400/469 | Cost: 13378.9902 Epoch: 045/050 | Batch 450/469 | Cost: 12873.3105 Epoch: 046/050 | Batch 000/469 | Cost: 12739.6260 Epoch: 046/050 | Batch 050/469 | Cost: 13021.6465 Epoch: 046/050 | Batch 100/469 | Cost: 13027.7256 Epoch: 046/050 | Batch 150/469 | Cost: 12995.7490 Epoch: 046/050 | Batch 200/469 | Cost: 12588.5645 Epoch: 046/050 | Batch 250/469 | Cost: 13288.1494 Epoch: 046/050 | Batch 300/469 | Cost: 12766.4707 Epoch: 046/050 | Batch 350/469 | Cost: 12326.2334 Epoch: 046/050 | Batch 400/469 | Cost: 13174.7734 Epoch: 046/050 | Batch 450/469 | Cost: 12531.1074 Epoch: 047/050 | Batch 000/469 | Cost: 13050.0781 Epoch: 047/050 | Batch 050/469 | Cost: 12920.9609 Epoch: 047/050 | Batch 100/469 | Cost: 12954.7383 Epoch: 047/050 | Batch 150/469 | Cost: 12598.8203 Epoch: 047/050 | Batch 200/469 | Cost: 12969.3066 Epoch: 047/050 | Batch 250/469 | Cost: 12893.0693 Epoch: 047/050 | Batch 300/469 | Cost: 12975.1309 Epoch: 047/050 | Batch 350/469 | Cost: 13452.9775 Epoch: 047/050 | Batch 400/469 | Cost: 13320.0781 Epoch: 047/050 | Batch 450/469 | Cost: 13015.5547 Epoch: 048/050 | Batch 000/469 | Cost: 12687.9102 Epoch: 048/050 | Batch 050/469 | Cost: 12957.4678 Epoch: 048/050 | Batch 100/469 | Cost: 13027.3281 Epoch: 048/050 | Batch 150/469 | Cost: 13472.4619 Epoch: 048/050 | Batch 200/469 | Cost: 12705.8525 Epoch: 048/050 | Batch 250/469 | Cost: 13201.4590 Epoch: 048/050 | Batch 300/469 | Cost: 13181.9707 Epoch: 048/050 | Batch 350/469 | Cost: 13372.9746 Epoch: 048/050 | Batch 400/469 | Cost: 12831.2305 Epoch: 048/050 | Batch 450/469 | Cost: 12963.0156 Epoch: 049/050 | Batch 000/469 | Cost: 12832.8057 Epoch: 049/050 | Batch 050/469 | Cost: 12771.7109 Epoch: 049/050 | Batch 100/469 | Cost: 13143.8457 Epoch: 049/050 | Batch 150/469 | Cost: 12863.8740 Epoch: 049/050 | Batch 200/469 | Cost: 12841.9248 Epoch: 049/050 | Batch 250/469 | Cost: 13240.2529 Epoch: 049/050 | Batch 300/469 | Cost: 12889.4521 Epoch: 049/050 | Batch 350/469 | Cost: 13022.1709 Epoch: 049/050 | Batch 400/469 | Cost: 12963.3125 Epoch: 049/050 | Batch 450/469 | Cost: 12778.5381 Epoch: 050/050 | Batch 000/469 | Cost: 12820.4805 Epoch: 050/050 | Batch 050/469 | Cost: 12769.4893 Epoch: 050/050 | Batch 100/469 | Cost: 12682.0566 Epoch: 050/050 | Batch 150/469 | Cost: 13312.6172 Epoch: 050/050 | Batch 200/469 | Cost: 13716.5430 Epoch: 050/050 | Batch 250/469 | Cost: 12201.1973 Epoch: 050/050 | Batch 300/469 | Cost: 13228.4199 Epoch: 050/050 | Batch 350/469 | Cost: 12986.6201 Epoch: 050/050 | Batch 400/469 | Cost: 13097.1562 Epoch: 050/050 | Batch 450/469 | Cost: 13272.6777
%matplotlib inline
import matplotlib.pyplot as plt
##########################
### VISUALIZATION
##########################
n_images = 15
image_width = 28
fig, axes = plt.subplots(nrows=2, ncols=n_images,
sharex=True, sharey=True, figsize=(20, 2.5))
orig_images = features[:n_images]
decoded_images = decoded[:n_images]
for i in range(n_images):
for ax, img in zip(axes, [orig_images, decoded_images]):
curr_img = img[i].detach().to(torch.device('cpu'))
ax[i].imshow(curr_img.view((image_width, image_width)), cmap='binary')
for i in range(10):
##########################
### RANDOM SAMPLE
##########################
n_images = 10
rand_features = torch.randn(n_images, num_latent).to(device)
new_images = model.decoder(rand_features)
##########################
### VISUALIZATION
##########################
image_width = 28
fig, axes = plt.subplots(nrows=1, ncols=n_images, figsize=(10, 2.5), sharey=True)
decoded_images = new_images[:n_images]
for ax, img in zip(axes, decoded_images):
curr_img = img.detach().to(torch.device('cpu'))
ax.imshow(curr_img.view((image_width, image_width)), cmap='binary')
plt.show()
%watermark -iv
numpy 1.15.4 torch 1.0.0