Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.

In [1]:
%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -p torch
Sebastian Raschka 

CPython 3.7.1
IPython 7.2.0

torch 1.0.0
  • Runs on CPU or GPU (if available)

Model Zoo -- Variational Autoencoder

A simple variational autoencoder that compresses 768-pixel MNIST images down to a 15-pixel latent vector representation.

Imports

In [2]:
import time
import numpy as np
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision import transforms


if torch.cuda.is_available():
    torch.backends.cudnn.deterministic = True
In [3]:
##########################
### SETTINGS
##########################

# Device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('Device:', device)

# Hyperparameters
random_seed = 0
learning_rate = 0.001
num_epochs = 50
batch_size = 128

# Architecture
num_features = 784
num_hidden_1 = 500
num_latent = 15


##########################
### MNIST DATASET
##########################

# Note transforms.ToTensor() scales input images
# to 0-1 range
train_dataset = datasets.MNIST(root='data', 
                               train=True, 
                               transform=transforms.ToTensor(),
                               download=True)

test_dataset = datasets.MNIST(root='data', 
                              train=False, 
                              transform=transforms.ToTensor())


train_loader = DataLoader(dataset=train_dataset, 
                          batch_size=batch_size, 
                          shuffle=True)

test_loader = DataLoader(dataset=test_dataset, 
                         batch_size=batch_size, 
                         shuffle=False)

# Checking the dataset
for images, labels in train_loader:  
    print('Image batch dimensions:', images.shape)
    print('Image label dimensions:', labels.shape)
    break
Device: cuda:0
Image batch dimensions: torch.Size([128, 1, 28, 28])
Image label dimensions: torch.Size([128])

Model

In [4]:
##########################
### MODEL
##########################

class VariationalAutoencoder(torch.nn.Module):

    def __init__(self, num_features, num_hidden_1, num_latent):
        super(VariationalAutoencoder, self).__init__()
        
        ### ENCODER
        self.hidden_1 = torch.nn.Linear(num_features, num_hidden_1)
        self.z_mean = torch.nn.Linear(num_hidden_1, num_latent)
        # in the original paper (Kingma & Welling 2015, we use
        # have a z_mean and z_var, but the problem is that
        # the z_var can be negative, which would cause issues
        # in the log later. Hence we assume that latent vector
        # has a z_mean and z_log_var component, and when we need
        # the regular variance or std_dev, we simply use 
        # an exponential function
        self.z_log_var = torch.nn.Linear(num_hidden_1, num_latent)
        
        
        ### DECODER
        self.linear_3 = torch.nn.Linear(num_latent, num_hidden_1)
        self.linear_4 = torch.nn.Linear(num_hidden_1, num_features)

    def reparameterize(self, z_mu, z_log_var):
        # Sample epsilon from standard normal distribution
        eps = torch.randn(z_mu.size(0), z_mu.size(1)).to(device)
        # note that log(x^2) = 2*log(x); hence divide by 2 to get std_dev
        # i.e., std_dev = exp(log(std_dev^2)/2) = exp(log(var)/2)
        z = z_mu + eps * torch.exp(z_log_var/2.) 
        return z
        
    def encoder(self, features):
        x = self.hidden_1(features)
        x = F.leaky_relu(x, negative_slope=0.0001)
        z_mean = self.z_mean(x)
        z_log_var = self.z_log_var(x)
        encoded = self.reparameterize(z_mean, z_log_var)
        return z_mean, z_log_var, encoded
    
    def decoder(self, encoded):
        x = self.linear_3(encoded)
        x = F.leaky_relu(x, negative_slope=0.0001)
        x = self.linear_4(x)
        decoded = torch.sigmoid(x)
        return decoded

    def forward(self, features):
        
        z_mean, z_log_var, encoded = self.encoder(features)
        decoded = self.decoder(encoded)
        
        return z_mean, z_log_var, encoded, decoded

    
torch.manual_seed(random_seed)
model = VariationalAutoencoder(num_features,
                               num_hidden_1,
                               num_latent)
model = model.to(device)
    
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)  

Training

In [5]:
start_time = time.time()
for epoch in range(num_epochs):
    for batch_idx, (features, targets) in enumerate(train_loader):
        
        # don't need labels, only the images (features)
        features = features.view(-1, 28*28).to(device)

        ### FORWARD AND BACK PROP
        z_mean, z_log_var, encoded, decoded = model(features)

        # cost = reconstruction loss + Kullback-Leibler divergence
        kl_divergence = (0.5 * (z_mean**2 + 
                                torch.exp(z_log_var) - z_log_var - 1)).sum()
        pixelwise_bce = F.binary_cross_entropy(decoded, features, reduction='sum')
        cost = kl_divergence + pixelwise_bce
        
        optimizer.zero_grad()
        cost.backward()
        
        ### UPDATE MODEL PARAMETERS
        optimizer.step()
        
        ### LOGGING
        if not batch_idx % 50:
            print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' 
                   %(epoch+1, num_epochs, batch_idx, 
                     len(train_loader), cost))
            
    print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))
    
print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))
Epoch: 001/050 | Batch 000/469 | Cost: 70481.2422
Epoch: 001/050 | Batch 050/469 | Cost: 27139.5547
Epoch: 001/050 | Batch 100/469 | Cost: 22833.3730
Epoch: 001/050 | Batch 150/469 | Cost: 19493.1523
Epoch: 001/050 | Batch 200/469 | Cost: 18727.4688
Epoch: 001/050 | Batch 250/469 | Cost: 18074.2676
Epoch: 001/050 | Batch 300/469 | Cost: 16633.2852
Epoch: 001/050 | Batch 350/469 | Cost: 17136.7852
Epoch: 001/050 | Batch 400/469 | Cost: 16402.5293
Epoch: 001/050 | Batch 450/469 | Cost: 16062.9814
Epoch: 002/050 | Batch 000/469 | Cost: 16577.0840
Epoch: 002/050 | Batch 050/469 | Cost: 15451.8242
Epoch: 002/050 | Batch 100/469 | Cost: 15667.8535
Epoch: 002/050 | Batch 150/469 | Cost: 15734.0801
Epoch: 002/050 | Batch 200/469 | Cost: 15145.4365
Epoch: 002/050 | Batch 250/469 | Cost: 15326.6953
Epoch: 002/050 | Batch 300/469 | Cost: 15408.5801
Epoch: 002/050 | Batch 350/469 | Cost: 15637.5430
Epoch: 002/050 | Batch 400/469 | Cost: 14793.0332
Epoch: 002/050 | Batch 450/469 | Cost: 15046.4414
Epoch: 003/050 | Batch 000/469 | Cost: 14457.0537
Epoch: 003/050 | Batch 050/469 | Cost: 14483.2910
Epoch: 003/050 | Batch 100/469 | Cost: 14374.9258
Epoch: 003/050 | Batch 150/469 | Cost: 13934.8672
Epoch: 003/050 | Batch 200/469 | Cost: 15053.9336
Epoch: 003/050 | Batch 250/469 | Cost: 14673.1025
Epoch: 003/050 | Batch 300/469 | Cost: 14324.3916
Epoch: 003/050 | Batch 350/469 | Cost: 14318.4229
Epoch: 003/050 | Batch 400/469 | Cost: 14501.4912
Epoch: 003/050 | Batch 450/469 | Cost: 13753.9082
Epoch: 004/050 | Batch 000/469 | Cost: 15024.3789
Epoch: 004/050 | Batch 050/469 | Cost: 14310.9219
Epoch: 004/050 | Batch 100/469 | Cost: 14723.5176
Epoch: 004/050 | Batch 150/469 | Cost: 15469.9473
Epoch: 004/050 | Batch 200/469 | Cost: 14126.0586
Epoch: 004/050 | Batch 250/469 | Cost: 14321.4062
Epoch: 004/050 | Batch 300/469 | Cost: 13834.0576
Epoch: 004/050 | Batch 350/469 | Cost: 14363.6494
Epoch: 004/050 | Batch 400/469 | Cost: 14136.7422
Epoch: 004/050 | Batch 450/469 | Cost: 13603.2012
Epoch: 005/050 | Batch 000/469 | Cost: 14002.0479
Epoch: 005/050 | Batch 050/469 | Cost: 14221.5488
Epoch: 005/050 | Batch 100/469 | Cost: 13972.6787
Epoch: 005/050 | Batch 150/469 | Cost: 13918.2402
Epoch: 005/050 | Batch 200/469 | Cost: 13839.3809
Epoch: 005/050 | Batch 250/469 | Cost: 14421.0020
Epoch: 005/050 | Batch 300/469 | Cost: 14611.1816
Epoch: 005/050 | Batch 350/469 | Cost: 13653.8027
Epoch: 005/050 | Batch 400/469 | Cost: 13632.8047
Epoch: 005/050 | Batch 450/469 | Cost: 13612.9375
Epoch: 006/050 | Batch 000/469 | Cost: 13993.7344
Epoch: 006/050 | Batch 050/469 | Cost: 13976.1006
Epoch: 006/050 | Batch 100/469 | Cost: 14309.5527
Epoch: 006/050 | Batch 150/469 | Cost: 13427.8916
Epoch: 006/050 | Batch 200/469 | Cost: 13811.6260
Epoch: 006/050 | Batch 250/469 | Cost: 14130.3496
Epoch: 006/050 | Batch 300/469 | Cost: 12895.7324
Epoch: 006/050 | Batch 350/469 | Cost: 13445.3213
Epoch: 006/050 | Batch 400/469 | Cost: 13374.8242
Epoch: 006/050 | Batch 450/469 | Cost: 13549.5098
Epoch: 007/050 | Batch 000/469 | Cost: 13913.4043
Epoch: 007/050 | Batch 050/469 | Cost: 13703.5654
Epoch: 007/050 | Batch 100/469 | Cost: 14132.1758
Epoch: 007/050 | Batch 150/469 | Cost: 14052.9814
Epoch: 007/050 | Batch 200/469 | Cost: 13750.3535
Epoch: 007/050 | Batch 250/469 | Cost: 14316.6953
Epoch: 007/050 | Batch 300/469 | Cost: 13224.3281
Epoch: 007/050 | Batch 350/469 | Cost: 14139.7979
Epoch: 007/050 | Batch 400/469 | Cost: 13795.6016
Epoch: 007/050 | Batch 450/469 | Cost: 13915.5020
Epoch: 008/050 | Batch 000/469 | Cost: 13548.9512
Epoch: 008/050 | Batch 050/469 | Cost: 13558.6338
Epoch: 008/050 | Batch 100/469 | Cost: 13883.1074
Epoch: 008/050 | Batch 150/469 | Cost: 13128.7617
Epoch: 008/050 | Batch 200/469 | Cost: 13133.5879
Epoch: 008/050 | Batch 250/469 | Cost: 13518.8672
Epoch: 008/050 | Batch 300/469 | Cost: 13679.2324
Epoch: 008/050 | Batch 350/469 | Cost: 13928.9824
Epoch: 008/050 | Batch 400/469 | Cost: 14079.2256
Epoch: 008/050 | Batch 450/469 | Cost: 13294.2021
Epoch: 009/050 | Batch 000/469 | Cost: 13619.6504
Epoch: 009/050 | Batch 050/469 | Cost: 13831.6201
Epoch: 009/050 | Batch 100/469 | Cost: 13848.1406
Epoch: 009/050 | Batch 150/469 | Cost: 14622.0889
Epoch: 009/050 | Batch 200/469 | Cost: 13843.3887
Epoch: 009/050 | Batch 250/469 | Cost: 13673.2441
Epoch: 009/050 | Batch 300/469 | Cost: 13646.6543
Epoch: 009/050 | Batch 350/469 | Cost: 13411.1816
Epoch: 009/050 | Batch 400/469 | Cost: 14463.7988
Epoch: 009/050 | Batch 450/469 | Cost: 13585.7891
Epoch: 010/050 | Batch 000/469 | Cost: 13929.6816
Epoch: 010/050 | Batch 050/469 | Cost: 13659.5176
Epoch: 010/050 | Batch 100/469 | Cost: 13504.2568
Epoch: 010/050 | Batch 150/469 | Cost: 13717.9434
Epoch: 010/050 | Batch 200/469 | Cost: 13711.8818
Epoch: 010/050 | Batch 250/469 | Cost: 13554.4062
Epoch: 010/050 | Batch 300/469 | Cost: 13317.5156
Epoch: 010/050 | Batch 350/469 | Cost: 13279.9912
Epoch: 010/050 | Batch 400/469 | Cost: 13069.9648
Epoch: 010/050 | Batch 450/469 | Cost: 13087.7695
Epoch: 011/050 | Batch 000/469 | Cost: 13800.6113
Epoch: 011/050 | Batch 050/469 | Cost: 13924.1973
Epoch: 011/050 | Batch 100/469 | Cost: 13173.4414
Epoch: 011/050 | Batch 150/469 | Cost: 13963.7402
Epoch: 011/050 | Batch 200/469 | Cost: 13682.3281
Epoch: 011/050 | Batch 250/469 | Cost: 13664.8027
Epoch: 011/050 | Batch 300/469 | Cost: 14188.4707
Epoch: 011/050 | Batch 350/469 | Cost: 13625.5840
Epoch: 011/050 | Batch 400/469 | Cost: 13482.8643
Epoch: 011/050 | Batch 450/469 | Cost: 13912.9238
Epoch: 012/050 | Batch 000/469 | Cost: 13048.4648
Epoch: 012/050 | Batch 050/469 | Cost: 13041.4395
Epoch: 012/050 | Batch 100/469 | Cost: 13212.3662
Epoch: 012/050 | Batch 150/469 | Cost: 13304.4463
Epoch: 012/050 | Batch 200/469 | Cost: 13445.9287
Epoch: 012/050 | Batch 250/469 | Cost: 13693.2676
Epoch: 012/050 | Batch 300/469 | Cost: 13072.9004
Epoch: 012/050 | Batch 350/469 | Cost: 13414.5361
Epoch: 012/050 | Batch 400/469 | Cost: 13669.4121
Epoch: 012/050 | Batch 450/469 | Cost: 13366.8633
Epoch: 013/050 | Batch 000/469 | Cost: 13785.0518
Epoch: 013/050 | Batch 050/469 | Cost: 13788.7734
Epoch: 013/050 | Batch 100/469 | Cost: 13442.9023
Epoch: 013/050 | Batch 150/469 | Cost: 13771.4902
Epoch: 013/050 | Batch 200/469 | Cost: 13357.2217
Epoch: 013/050 | Batch 250/469 | Cost: 13402.6758
Epoch: 013/050 | Batch 300/469 | Cost: 13852.4033
Epoch: 013/050 | Batch 350/469 | Cost: 13301.3457
Epoch: 013/050 | Batch 400/469 | Cost: 13379.6172
Epoch: 013/050 | Batch 450/469 | Cost: 14275.3047
Epoch: 014/050 | Batch 000/469 | Cost: 13433.3906
Epoch: 014/050 | Batch 050/469 | Cost: 13059.2354
Epoch: 014/050 | Batch 100/469 | Cost: 14031.3721
Epoch: 014/050 | Batch 150/469 | Cost: 13950.9883
Epoch: 014/050 | Batch 200/469 | Cost: 13684.6611
Epoch: 014/050 | Batch 250/469 | Cost: 13630.9336
Epoch: 014/050 | Batch 300/469 | Cost: 13527.6230
Epoch: 014/050 | Batch 350/469 | Cost: 13746.0527
Epoch: 014/050 | Batch 400/469 | Cost: 13490.6982
Epoch: 014/050 | Batch 450/469 | Cost: 13669.7402
Epoch: 015/050 | Batch 000/469 | Cost: 13422.4238
Epoch: 015/050 | Batch 050/469 | Cost: 13303.6855
Epoch: 015/050 | Batch 100/469 | Cost: 13421.2900
Epoch: 015/050 | Batch 150/469 | Cost: 13129.2764
Epoch: 015/050 | Batch 200/469 | Cost: 13276.9336
Epoch: 015/050 | Batch 250/469 | Cost: 13776.0889
Epoch: 015/050 | Batch 300/469 | Cost: 13634.2188
Epoch: 015/050 | Batch 350/469 | Cost: 13438.3828
Epoch: 015/050 | Batch 400/469 | Cost: 13401.1045
Epoch: 015/050 | Batch 450/469 | Cost: 13567.6631
Epoch: 016/050 | Batch 000/469 | Cost: 13150.5625
Epoch: 016/050 | Batch 050/469 | Cost: 13414.6553
Epoch: 016/050 | Batch 100/469 | Cost: 12979.0029
Epoch: 016/050 | Batch 150/469 | Cost: 13146.0801
Epoch: 016/050 | Batch 200/469 | Cost: 13257.3301
Epoch: 016/050 | Batch 250/469 | Cost: 14350.7471
Epoch: 016/050 | Batch 300/469 | Cost: 13836.4316
Epoch: 016/050 | Batch 350/469 | Cost: 13865.9902
Epoch: 016/050 | Batch 400/469 | Cost: 13237.3877
Epoch: 016/050 | Batch 450/469 | Cost: 13339.0303
Epoch: 017/050 | Batch 000/469 | Cost: 13266.2793
Epoch: 017/050 | Batch 050/469 | Cost: 13568.0957
Epoch: 017/050 | Batch 100/469 | Cost: 12923.4482
Epoch: 017/050 | Batch 150/469 | Cost: 14093.2070
Epoch: 017/050 | Batch 200/469 | Cost: 13326.7510
Epoch: 017/050 | Batch 250/469 | Cost: 13965.0625
Epoch: 017/050 | Batch 300/469 | Cost: 13380.1445
Epoch: 017/050 | Batch 350/469 | Cost: 13277.1875
Epoch: 017/050 | Batch 400/469 | Cost: 13872.7607
Epoch: 017/050 | Batch 450/469 | Cost: 13272.6797
Epoch: 018/050 | Batch 000/469 | Cost: 13048.2461
Epoch: 018/050 | Batch 050/469 | Cost: 13508.2314
Epoch: 018/050 | Batch 100/469 | Cost: 12814.9834
Epoch: 018/050 | Batch 150/469 | Cost: 13623.1924
Epoch: 018/050 | Batch 200/469 | Cost: 13246.6113
Epoch: 018/050 | Batch 250/469 | Cost: 13471.1328
Epoch: 018/050 | Batch 300/469 | Cost: 13271.7930
Epoch: 018/050 | Batch 350/469 | Cost: 13494.4883
Epoch: 018/050 | Batch 400/469 | Cost: 13280.4316
Epoch: 018/050 | Batch 450/469 | Cost: 13408.9775
Epoch: 019/050 | Batch 000/469 | Cost: 13347.4941
Epoch: 019/050 | Batch 050/469 | Cost: 13403.6006
Epoch: 019/050 | Batch 100/469 | Cost: 12944.8574
Epoch: 019/050 | Batch 150/469 | Cost: 13410.6201
Epoch: 019/050 | Batch 200/469 | Cost: 13398.5342
Epoch: 019/050 | Batch 250/469 | Cost: 13786.6992
Epoch: 019/050 | Batch 300/469 | Cost: 12185.1465
Epoch: 019/050 | Batch 350/469 | Cost: 13143.7744
Epoch: 019/050 | Batch 400/469 | Cost: 13101.7451
Epoch: 019/050 | Batch 450/469 | Cost: 13410.3252
Epoch: 020/050 | Batch 000/469 | Cost: 13459.7676
Epoch: 020/050 | Batch 050/469 | Cost: 13551.5127
Epoch: 020/050 | Batch 100/469 | Cost: 13246.3486
Epoch: 020/050 | Batch 150/469 | Cost: 13524.1133
Epoch: 020/050 | Batch 200/469 | Cost: 13695.5605
Epoch: 020/050 | Batch 250/469 | Cost: 13447.3887
Epoch: 020/050 | Batch 300/469 | Cost: 13389.4941
Epoch: 020/050 | Batch 350/469 | Cost: 13180.2422
Epoch: 020/050 | Batch 400/469 | Cost: 13606.3457
Epoch: 020/050 | Batch 450/469 | Cost: 13646.9355
Epoch: 021/050 | Batch 000/469 | Cost: 13557.3623
Epoch: 021/050 | Batch 050/469 | Cost: 13147.0098
Epoch: 021/050 | Batch 100/469 | Cost: 13287.7227
Epoch: 021/050 | Batch 150/469 | Cost: 12849.9639
Epoch: 021/050 | Batch 200/469 | Cost: 13058.6406
Epoch: 021/050 | Batch 250/469 | Cost: 13192.6367
Epoch: 021/050 | Batch 300/469 | Cost: 13393.4082
Epoch: 021/050 | Batch 350/469 | Cost: 13834.2705
Epoch: 021/050 | Batch 400/469 | Cost: 13503.1680
Epoch: 021/050 | Batch 450/469 | Cost: 13592.5518
Epoch: 022/050 | Batch 000/469 | Cost: 13658.5986
Epoch: 022/050 | Batch 050/469 | Cost: 13389.6855
Epoch: 022/050 | Batch 100/469 | Cost: 13313.9707
Epoch: 022/050 | Batch 150/469 | Cost: 13508.8438
Epoch: 022/050 | Batch 200/469 | Cost: 12984.4082
Epoch: 022/050 | Batch 250/469 | Cost: 13159.5137
Epoch: 022/050 | Batch 300/469 | Cost: 13195.3516
Epoch: 022/050 | Batch 350/469 | Cost: 13606.1777
Epoch: 022/050 | Batch 400/469 | Cost: 12865.0508
Epoch: 022/050 | Batch 450/469 | Cost: 13227.6514
Epoch: 023/050 | Batch 000/469 | Cost: 13067.6016
Epoch: 023/050 | Batch 050/469 | Cost: 13425.5498
Epoch: 023/050 | Batch 100/469 | Cost: 13016.2773
Epoch: 023/050 | Batch 150/469 | Cost: 13322.1260
Epoch: 023/050 | Batch 200/469 | Cost: 12861.3926
Epoch: 023/050 | Batch 250/469 | Cost: 13000.5967
Epoch: 023/050 | Batch 300/469 | Cost: 13761.4629
Epoch: 023/050 | Batch 350/469 | Cost: 13482.9814
Epoch: 023/050 | Batch 400/469 | Cost: 12838.6201
Epoch: 023/050 | Batch 450/469 | Cost: 13252.9746
Epoch: 024/050 | Batch 000/469 | Cost: 13445.4590
Epoch: 024/050 | Batch 050/469 | Cost: 13583.6416
Epoch: 024/050 | Batch 100/469 | Cost: 13507.9443
Epoch: 024/050 | Batch 150/469 | Cost: 13385.5938
Epoch: 024/050 | Batch 200/469 | Cost: 13271.1357
Epoch: 024/050 | Batch 250/469 | Cost: 13643.7109
Epoch: 024/050 | Batch 300/469 | Cost: 13713.0889
Epoch: 024/050 | Batch 350/469 | Cost: 12844.5703
Epoch: 024/050 | Batch 400/469 | Cost: 12984.4746
Epoch: 024/050 | Batch 450/469 | Cost: 13015.4365
Epoch: 025/050 | Batch 000/469 | Cost: 13575.6875
Epoch: 025/050 | Batch 050/469 | Cost: 13195.2832
Epoch: 025/050 | Batch 100/469 | Cost: 13478.4746
Epoch: 025/050 | Batch 150/469 | Cost: 13194.2852
Epoch: 025/050 | Batch 200/469 | Cost: 12877.8242
Epoch: 025/050 | Batch 250/469 | Cost: 13061.2148
Epoch: 025/050 | Batch 300/469 | Cost: 13397.2266
Epoch: 025/050 | Batch 350/469 | Cost: 12763.3711
Epoch: 025/050 | Batch 400/469 | Cost: 13262.5332
Epoch: 025/050 | Batch 450/469 | Cost: 13390.2393
Epoch: 026/050 | Batch 000/469 | Cost: 13211.5508
Epoch: 026/050 | Batch 050/469 | Cost: 13458.3652
Epoch: 026/050 | Batch 100/469 | Cost: 12846.0557
Epoch: 026/050 | Batch 150/469 | Cost: 12842.9570
Epoch: 026/050 | Batch 200/469 | Cost: 13594.9395
Epoch: 026/050 | Batch 250/469 | Cost: 13021.0605
Epoch: 026/050 | Batch 300/469 | Cost: 13126.7686
Epoch: 026/050 | Batch 350/469 | Cost: 12951.5898
Epoch: 026/050 | Batch 400/469 | Cost: 13600.2119
Epoch: 026/050 | Batch 450/469 | Cost: 13313.3535
Epoch: 027/050 | Batch 000/469 | Cost: 12835.4717
Epoch: 027/050 | Batch 050/469 | Cost: 12731.1875
Epoch: 027/050 | Batch 100/469 | Cost: 13234.9297
Epoch: 027/050 | Batch 150/469 | Cost: 13105.2148
Epoch: 027/050 | Batch 200/469 | Cost: 13234.0684
Epoch: 027/050 | Batch 250/469 | Cost: 13147.0801
Epoch: 027/050 | Batch 300/469 | Cost: 13271.8262
Epoch: 027/050 | Batch 350/469 | Cost: 12936.5947
Epoch: 027/050 | Batch 400/469 | Cost: 13336.0293
Epoch: 027/050 | Batch 450/469 | Cost: 13387.3662
Epoch: 028/050 | Batch 000/469 | Cost: 13452.3438
Epoch: 028/050 | Batch 050/469 | Cost: 13245.0342
Epoch: 028/050 | Batch 100/469 | Cost: 13007.5234
Epoch: 028/050 | Batch 150/469 | Cost: 13068.0166
Epoch: 028/050 | Batch 200/469 | Cost: 12575.0166
Epoch: 028/050 | Batch 250/469 | Cost: 13051.9434
Epoch: 028/050 | Batch 300/469 | Cost: 13185.3330
Epoch: 028/050 | Batch 350/469 | Cost: 13587.7715
Epoch: 028/050 | Batch 400/469 | Cost: 12877.1436
Epoch: 028/050 | Batch 450/469 | Cost: 13305.9297
Epoch: 029/050 | Batch 000/469 | Cost: 13244.1865
Epoch: 029/050 | Batch 050/469 | Cost: 13002.6309
Epoch: 029/050 | Batch 100/469 | Cost: 13432.6504
Epoch: 029/050 | Batch 150/469 | Cost: 13128.8027
Epoch: 029/050 | Batch 200/469 | Cost: 12879.6543
Epoch: 029/050 | Batch 250/469 | Cost: 13248.0068
Epoch: 029/050 | Batch 300/469 | Cost: 13176.9912
Epoch: 029/050 | Batch 350/469 | Cost: 13055.7490
Epoch: 029/050 | Batch 400/469 | Cost: 13092.9580
Epoch: 029/050 | Batch 450/469 | Cost: 13179.1875
Epoch: 030/050 | Batch 000/469 | Cost: 13205.4668
Epoch: 030/050 | Batch 050/469 | Cost: 13425.4883
Epoch: 030/050 | Batch 100/469 | Cost: 12924.2070
Epoch: 030/050 | Batch 150/469 | Cost: 13293.8105
Epoch: 030/050 | Batch 200/469 | Cost: 12805.0674
Epoch: 030/050 | Batch 250/469 | Cost: 12823.4629
Epoch: 030/050 | Batch 300/469 | Cost: 12680.0322
Epoch: 030/050 | Batch 350/469 | Cost: 13412.4023
Epoch: 030/050 | Batch 400/469 | Cost: 13796.5479
Epoch: 030/050 | Batch 450/469 | Cost: 13084.7051
Epoch: 031/050 | Batch 000/469 | Cost: 13054.2988
Epoch: 031/050 | Batch 050/469 | Cost: 13315.4570
Epoch: 031/050 | Batch 100/469 | Cost: 13284.9463
Epoch: 031/050 | Batch 150/469 | Cost: 13184.4668
Epoch: 031/050 | Batch 200/469 | Cost: 13099.4189
Epoch: 031/050 | Batch 250/469 | Cost: 13391.0918
Epoch: 031/050 | Batch 300/469 | Cost: 13057.3223
Epoch: 031/050 | Batch 350/469 | Cost: 13442.3750
Epoch: 031/050 | Batch 400/469 | Cost: 13491.5635
Epoch: 031/050 | Batch 450/469 | Cost: 13054.0693
Epoch: 032/050 | Batch 000/469 | Cost: 13219.3789
Epoch: 032/050 | Batch 050/469 | Cost: 12822.7051
Epoch: 032/050 | Batch 100/469 | Cost: 13439.6436
Epoch: 032/050 | Batch 150/469 | Cost: 12843.7061
Epoch: 032/050 | Batch 200/469 | Cost: 13097.7012
Epoch: 032/050 | Batch 250/469 | Cost: 12950.4707
Epoch: 032/050 | Batch 300/469 | Cost: 13238.1094
Epoch: 032/050 | Batch 350/469 | Cost: 13027.9121
Epoch: 032/050 | Batch 400/469 | Cost: 13150.9277
Epoch: 032/050 | Batch 450/469 | Cost: 13239.6348
Epoch: 033/050 | Batch 000/469 | Cost: 12967.9863
Epoch: 033/050 | Batch 050/469 | Cost: 13261.3467
Epoch: 033/050 | Batch 100/469 | Cost: 13218.9023
Epoch: 033/050 | Batch 150/469 | Cost: 13092.8994
Epoch: 033/050 | Batch 200/469 | Cost: 12983.0459
Epoch: 033/050 | Batch 250/469 | Cost: 13031.2188
Epoch: 033/050 | Batch 300/469 | Cost: 12894.7129
Epoch: 033/050 | Batch 350/469 | Cost: 13563.2578
Epoch: 033/050 | Batch 400/469 | Cost: 13094.8340
Epoch: 033/050 | Batch 450/469 | Cost: 13279.9639
Epoch: 034/050 | Batch 000/469 | Cost: 12986.0615
Epoch: 034/050 | Batch 050/469 | Cost: 12981.4004
Epoch: 034/050 | Batch 100/469 | Cost: 13308.1504
Epoch: 034/050 | Batch 150/469 | Cost: 13338.7227
Epoch: 034/050 | Batch 200/469 | Cost: 13310.7227
Epoch: 034/050 | Batch 250/469 | Cost: 13158.7334
Epoch: 034/050 | Batch 300/469 | Cost: 13248.9336
Epoch: 034/050 | Batch 350/469 | Cost: 13256.2227
Epoch: 034/050 | Batch 400/469 | Cost: 12818.7148
Epoch: 034/050 | Batch 450/469 | Cost: 12835.6738
Epoch: 035/050 | Batch 000/469 | Cost: 12766.6123
Epoch: 035/050 | Batch 050/469 | Cost: 12521.5166
Epoch: 035/050 | Batch 100/469 | Cost: 12340.0430
Epoch: 035/050 | Batch 150/469 | Cost: 12873.6191
Epoch: 035/050 | Batch 200/469 | Cost: 13027.2266
Epoch: 035/050 | Batch 250/469 | Cost: 13575.8379
Epoch: 035/050 | Batch 300/469 | Cost: 13458.8867
Epoch: 035/050 | Batch 350/469 | Cost: 12816.4980
Epoch: 035/050 | Batch 400/469 | Cost: 12663.2207
Epoch: 035/050 | Batch 450/469 | Cost: 12733.6777
Epoch: 036/050 | Batch 000/469 | Cost: 13078.8682
Epoch: 036/050 | Batch 050/469 | Cost: 13072.0742
Epoch: 036/050 | Batch 100/469 | Cost: 12666.5215
Epoch: 036/050 | Batch 150/469 | Cost: 13091.2852
Epoch: 036/050 | Batch 200/469 | Cost: 13462.2529
Epoch: 036/050 | Batch 250/469 | Cost: 12630.4287
Epoch: 036/050 | Batch 300/469 | Cost: 13213.3223
Epoch: 036/050 | Batch 350/469 | Cost: 13298.2490
Epoch: 036/050 | Batch 400/469 | Cost: 12989.6328
Epoch: 036/050 | Batch 450/469 | Cost: 12918.6348
Epoch: 037/050 | Batch 000/469 | Cost: 12605.0732
Epoch: 037/050 | Batch 050/469 | Cost: 13055.5742
Epoch: 037/050 | Batch 100/469 | Cost: 12719.5420
Epoch: 037/050 | Batch 150/469 | Cost: 12599.2461
Epoch: 037/050 | Batch 200/469 | Cost: 12545.8223
Epoch: 037/050 | Batch 250/469 | Cost: 12449.0918
Epoch: 037/050 | Batch 300/469 | Cost: 13342.7930
Epoch: 037/050 | Batch 350/469 | Cost: 13066.0029
Epoch: 037/050 | Batch 400/469 | Cost: 13258.0957
Epoch: 037/050 | Batch 450/469 | Cost: 13180.6914
Epoch: 038/050 | Batch 000/469 | Cost: 12527.9854
Epoch: 038/050 | Batch 050/469 | Cost: 13618.6875
Epoch: 038/050 | Batch 100/469 | Cost: 13039.2627
Epoch: 038/050 | Batch 150/469 | Cost: 13062.9453
Epoch: 038/050 | Batch 200/469 | Cost: 13139.8945
Epoch: 038/050 | Batch 250/469 | Cost: 13168.6621
Epoch: 038/050 | Batch 300/469 | Cost: 12623.4629
Epoch: 038/050 | Batch 350/469 | Cost: 12757.8447
Epoch: 038/050 | Batch 400/469 | Cost: 12830.0762
Epoch: 038/050 | Batch 450/469 | Cost: 12733.7969
Epoch: 039/050 | Batch 000/469 | Cost: 12927.6729
Epoch: 039/050 | Batch 050/469 | Cost: 13016.1133
Epoch: 039/050 | Batch 100/469 | Cost: 12955.1621
Epoch: 039/050 | Batch 150/469 | Cost: 12945.2852
Epoch: 039/050 | Batch 200/469 | Cost: 12680.2188
Epoch: 039/050 | Batch 250/469 | Cost: 12958.4688
Epoch: 039/050 | Batch 300/469 | Cost: 13075.4912
Epoch: 039/050 | Batch 350/469 | Cost: 12962.3750
Epoch: 039/050 | Batch 400/469 | Cost: 12863.8867
Epoch: 039/050 | Batch 450/469 | Cost: 13399.3818
Epoch: 040/050 | Batch 000/469 | Cost: 12694.0283
Epoch: 040/050 | Batch 050/469 | Cost: 13524.2754
Epoch: 040/050 | Batch 100/469 | Cost: 12840.9316
Epoch: 040/050 | Batch 150/469 | Cost: 12661.5918
Epoch: 040/050 | Batch 200/469 | Cost: 13256.4902
Epoch: 040/050 | Batch 250/469 | Cost: 13027.6816
Epoch: 040/050 | Batch 300/469 | Cost: 12941.4727
Epoch: 040/050 | Batch 350/469 | Cost: 12656.1348
Epoch: 040/050 | Batch 400/469 | Cost: 12979.4785
Epoch: 040/050 | Batch 450/469 | Cost: 12705.2158
Epoch: 041/050 | Batch 000/469 | Cost: 12759.9707
Epoch: 041/050 | Batch 050/469 | Cost: 12406.5781
Epoch: 041/050 | Batch 100/469 | Cost: 12696.9307
Epoch: 041/050 | Batch 150/469 | Cost: 13398.3613
Epoch: 041/050 | Batch 200/469 | Cost: 12777.3418
Epoch: 041/050 | Batch 250/469 | Cost: 12854.2783
Epoch: 041/050 | Batch 300/469 | Cost: 13037.7236
Epoch: 041/050 | Batch 350/469 | Cost: 13410.0801
Epoch: 041/050 | Batch 400/469 | Cost: 13350.9121
Epoch: 041/050 | Batch 450/469 | Cost: 12898.7432
Epoch: 042/050 | Batch 000/469 | Cost: 12766.4316
Epoch: 042/050 | Batch 050/469 | Cost: 13303.4766
Epoch: 042/050 | Batch 100/469 | Cost: 13112.1465
Epoch: 042/050 | Batch 150/469 | Cost: 12951.8428
Epoch: 042/050 | Batch 200/469 | Cost: 13367.4814
Epoch: 042/050 | Batch 250/469 | Cost: 13274.8955
Epoch: 042/050 | Batch 300/469 | Cost: 12908.4805
Epoch: 042/050 | Batch 350/469 | Cost: 12858.0723
Epoch: 042/050 | Batch 400/469 | Cost: 13279.5410
Epoch: 042/050 | Batch 450/469 | Cost: 12669.7422
Epoch: 043/050 | Batch 000/469 | Cost: 13124.0225
Epoch: 043/050 | Batch 050/469 | Cost: 12976.3857
Epoch: 043/050 | Batch 100/469 | Cost: 12655.5703
Epoch: 043/050 | Batch 150/469 | Cost: 12876.7061
Epoch: 043/050 | Batch 200/469 | Cost: 13277.1592
Epoch: 043/050 | Batch 250/469 | Cost: 12657.5117
Epoch: 043/050 | Batch 300/469 | Cost: 12915.8867
Epoch: 043/050 | Batch 350/469 | Cost: 13254.9941
Epoch: 043/050 | Batch 400/469 | Cost: 12649.9102
Epoch: 043/050 | Batch 450/469 | Cost: 13198.5771
Epoch: 044/050 | Batch 000/469 | Cost: 13573.9121
Epoch: 044/050 | Batch 050/469 | Cost: 12972.9453
Epoch: 044/050 | Batch 100/469 | Cost: 12764.2188
Epoch: 044/050 | Batch 150/469 | Cost: 13482.2910
Epoch: 044/050 | Batch 200/469 | Cost: 13304.8975
Epoch: 044/050 | Batch 250/469 | Cost: 13446.4141
Epoch: 044/050 | Batch 300/469 | Cost: 13096.8887
Epoch: 044/050 | Batch 350/469 | Cost: 13551.5537
Epoch: 044/050 | Batch 400/469 | Cost: 12693.8301
Epoch: 044/050 | Batch 450/469 | Cost: 12812.8682
Epoch: 045/050 | Batch 000/469 | Cost: 12698.2207
Epoch: 045/050 | Batch 050/469 | Cost: 12705.6787
Epoch: 045/050 | Batch 100/469 | Cost: 13046.6162
Epoch: 045/050 | Batch 150/469 | Cost: 13085.3457
Epoch: 045/050 | Batch 200/469 | Cost: 12922.5312
Epoch: 045/050 | Batch 250/469 | Cost: 13367.9189
Epoch: 045/050 | Batch 300/469 | Cost: 12917.3457
Epoch: 045/050 | Batch 350/469 | Cost: 12896.9463
Epoch: 045/050 | Batch 400/469 | Cost: 13378.9902
Epoch: 045/050 | Batch 450/469 | Cost: 12873.3105
Epoch: 046/050 | Batch 000/469 | Cost: 12739.6260
Epoch: 046/050 | Batch 050/469 | Cost: 13021.6465
Epoch: 046/050 | Batch 100/469 | Cost: 13027.7256
Epoch: 046/050 | Batch 150/469 | Cost: 12995.7490
Epoch: 046/050 | Batch 200/469 | Cost: 12588.5645
Epoch: 046/050 | Batch 250/469 | Cost: 13288.1494
Epoch: 046/050 | Batch 300/469 | Cost: 12766.4707
Epoch: 046/050 | Batch 350/469 | Cost: 12326.2334
Epoch: 046/050 | Batch 400/469 | Cost: 13174.7734
Epoch: 046/050 | Batch 450/469 | Cost: 12531.1074
Epoch: 047/050 | Batch 000/469 | Cost: 13050.0781
Epoch: 047/050 | Batch 050/469 | Cost: 12920.9609
Epoch: 047/050 | Batch 100/469 | Cost: 12954.7383
Epoch: 047/050 | Batch 150/469 | Cost: 12598.8203
Epoch: 047/050 | Batch 200/469 | Cost: 12969.3066
Epoch: 047/050 | Batch 250/469 | Cost: 12893.0693
Epoch: 047/050 | Batch 300/469 | Cost: 12975.1309
Epoch: 047/050 | Batch 350/469 | Cost: 13452.9775
Epoch: 047/050 | Batch 400/469 | Cost: 13320.0781
Epoch: 047/050 | Batch 450/469 | Cost: 13015.5547
Epoch: 048/050 | Batch 000/469 | Cost: 12687.9102
Epoch: 048/050 | Batch 050/469 | Cost: 12957.4678
Epoch: 048/050 | Batch 100/469 | Cost: 13027.3281
Epoch: 048/050 | Batch 150/469 | Cost: 13472.4619
Epoch: 048/050 | Batch 200/469 | Cost: 12705.8525
Epoch: 048/050 | Batch 250/469 | Cost: 13201.4590
Epoch: 048/050 | Batch 300/469 | Cost: 13181.9707
Epoch: 048/050 | Batch 350/469 | Cost: 13372.9746
Epoch: 048/050 | Batch 400/469 | Cost: 12831.2305
Epoch: 048/050 | Batch 450/469 | Cost: 12963.0156
Epoch: 049/050 | Batch 000/469 | Cost: 12832.8057
Epoch: 049/050 | Batch 050/469 | Cost: 12771.7109
Epoch: 049/050 | Batch 100/469 | Cost: 13143.8457
Epoch: 049/050 | Batch 150/469 | Cost: 12863.8740
Epoch: 049/050 | Batch 200/469 | Cost: 12841.9248
Epoch: 049/050 | Batch 250/469 | Cost: 13240.2529
Epoch: 049/050 | Batch 300/469 | Cost: 12889.4521
Epoch: 049/050 | Batch 350/469 | Cost: 13022.1709
Epoch: 049/050 | Batch 400/469 | Cost: 12963.3125
Epoch: 049/050 | Batch 450/469 | Cost: 12778.5381
Epoch: 050/050 | Batch 000/469 | Cost: 12820.4805
Epoch: 050/050 | Batch 050/469 | Cost: 12769.4893
Epoch: 050/050 | Batch 100/469 | Cost: 12682.0566
Epoch: 050/050 | Batch 150/469 | Cost: 13312.6172
Epoch: 050/050 | Batch 200/469 | Cost: 13716.5430
Epoch: 050/050 | Batch 250/469 | Cost: 12201.1973
Epoch: 050/050 | Batch 300/469 | Cost: 13228.4199
Epoch: 050/050 | Batch 350/469 | Cost: 12986.6201
Epoch: 050/050 | Batch 400/469 | Cost: 13097.1562
Epoch: 050/050 | Batch 450/469 | Cost: 13272.6777

Evaluation

Reconstruction

In [6]:
%matplotlib inline
import matplotlib.pyplot as plt

##########################
### VISUALIZATION
##########################

n_images = 15
image_width = 28

fig, axes = plt.subplots(nrows=2, ncols=n_images, 
                         sharex=True, sharey=True, figsize=(20, 2.5))
orig_images = features[:n_images]
decoded_images = decoded[:n_images]

for i in range(n_images):
    for ax, img in zip(axes, [orig_images, decoded_images]):
        curr_img = img[i].detach().to(torch.device('cpu'))
        ax[i].imshow(curr_img.view((image_width, image_width)), cmap='binary')

Generate new images

In [7]:
for i in range(10):

    ##########################
    ### RANDOM SAMPLE
    ##########################    
    
    n_images = 10
    rand_features = torch.randn(n_images, num_latent).to(device)
    new_images = model.decoder(rand_features)

    ##########################
    ### VISUALIZATION
    ##########################

    image_width = 28

    fig, axes = plt.subplots(nrows=1, ncols=n_images, figsize=(10, 2.5), sharey=True)
    decoded_images = new_images[:n_images]

    for ax, img in zip(axes, decoded_images):
        curr_img = img.detach().to(torch.device('cpu'))
        ax.imshow(curr_img.view((image_width, image_width)), cmap='binary')
        
    plt.show()
In [8]:
%watermark -iv
numpy       1.15.4
torch       1.0.0