Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.
%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -p torch
Sebastian Raschka CPython 3.7.3 IPython 7.9.0 torch 1.4.0
Implementation of a deep convolutional GAN (DCGAN) to generate cats and dogs images based on seeing examples of Kaggle's Dogs and Cats dataset (https://www.kaggle.com/c/dogs-vs-cats/data).
This DCGAN architecture is based on Radford et al.'s Unsupervised representation learning with deep convolutional generative adversarial networks [1], where the generator consists of
and the discriminator consists of
import time
import os
import numpy as np
import torch
import random
import torch.nn.functional as F
import torch.nn as nn
import torchvision.utils as vutils
from PIL import Image
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
from torchvision import transforms
if torch.cuda.is_available():
torch.backends.cudnn.deterministic = True
import matplotlib.pyplot as plt
%matplotlib inline
##########################
### SETTINGS
##########################
# Device
CUDA = 'cuda:2'
DEVICE = torch.device(CUDA if torch.cuda.is_available() else "cpu")
# Hyperparameters
RANDOM_SEED = 42
GENERATOR_LEARNING_RATE = 0.0002
DISCRIMINATOR_LEARNING_RATE = 0.0002
NUM_EPOCHS = 100
BATCH_SIZE = 128
NUM_WORKERS = 4 # workers for data loader
IMAGE_SIZE = (64, 64, 3)
# Size of the latent vector
LATENT_DIM = 100
# Number of feature maps in generator and discriminator
NUM_MAPS_GEN = 64
NUM_MAPS_DIS = 64
# Set random seeds for reproducibility
random.seed(RANDOM_SEED)
np.random.seed(RANDOM_SEED)
torch.manual_seed(RANDOM_SEED);
Download the Kaggle Cats and Dogs Dataset from https://www.kaggle.com/c/dogs-vs-cats/data by clicking on the "Download All" link:
Then, unzip the dataset.
The dataset folder consists of two subfolders, train
, and test1
, which contain the training and test images in jpg format, respectively. Note that the test set examples are unlabeled.
import os
num_train_cats = len([i for i in os.listdir(os.path.join('dogs-vs-cats', 'train'))
if i.endswith('.jpg') and i.startswith('cat')])
num_train_dogs = len([i for i in os.listdir(os.path.join('dogs-vs-cats', 'train'))
if i.endswith('.jpg') and i.startswith('dog')])
print(f'Training set cats: {num_train_cats}')
print(f'Training set dogs: {num_train_dogs}')
The naming scheme within each of these subfolders is <class>.<imagenumber>.jpg
.
img = Image.open(os.path.join('dogs-vs-cats','train', 'cat.59.jpg'))
print(np.asarray(img, dtype=np.uint8).shape)
plt.imshow(img);
(331, 464, 3)
if not os.path.exists(os.path.join('dogs-vs-cats', 'test')):
os.mkdir(os.path.join('dogs-vs-cats', 'test'))
if not os.path.exists(os.path.join('dogs-vs-cats', 'valid')):
os.mkdir(os.path.join('dogs-vs-cats', 'valid'))
for fname in os.listdir(os.path.join('dogs-vs-cats', 'train')):
if not fname.endswith('.jpg'):
continue
_, img_num, _ = fname.split('.')
filepath = os.path.join('dogs-vs-cats', 'train', fname)
img_num = int(img_num)
if img_num > 11249:
os.rename(filepath, filepath.replace('train', 'test'))
elif img_num > 9999:
os.rename(filepath, filepath.replace('train', 'valid'))
class CatsDogsDataset(Dataset):
"""Custom Dataset for loading CelebA face images"""
def __init__(self, img_dir, transform=None):
self.img_dir = img_dir
self.img_names = [i for i in
os.listdir(img_dir)
if i.endswith('.jpg')]
self.y = []
for i in self.img_names:
if i.split('.')[0] == 'cat':
self.y.append(0)
else:
self.y.append(1)
self.transform = transform
def __getitem__(self, index):
img = Image.open(os.path.join(self.img_dir,
self.img_names[index]))
if self.transform is not None:
img = self.transform(img)
label = self.y[index]
return img, label
def __len__(self):
return len(self.y)
data_transforms = {
'train': transforms.Compose([
#transforms.RandomRotation(5),
#transforms.RandomHorizontalFlip(),
transforms.RandomResizedCrop(IMAGE_SIZE[0], scale=(0.96, 1.0), ratio=(0.95, 1.05)),
transforms.ToTensor(),
# normalize images to [-1, 1] range
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
]),
'valid': transforms.Compose([
transforms.Resize([IMAGE_SIZE[0], IMAGE_SIZE[1]]),
transforms.ToTensor(),
# normalize images to [-1, 1] range
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
]),
}
train_dataset = CatsDogsDataset(img_dir=os.path.join('dogs-vs-cats', 'train'),
transform=data_transforms['train'])
train_loader = DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
drop_last=True,
num_workers=NUM_WORKERS,
shuffle=True)
# We don't need validation and test sets for GANs, which are unsupervised models
"""
valid_dataset = CatsDogsDataset(img_dir=os.path.join('dogs-vs-cats', 'valid'),
transform=data_transforms['valid'])
valid_loader = DataLoader(dataset=valid_dataset,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
shuffle=False)
test_dataset = CatsDogsDataset(img_dir=os.path.join('dogs-vs-cats', 'test'),
transform=data_transforms['valid'])
test_loader = DataLoader(dataset=test_dataset,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
shuffle=False)
"""
"\nvalid_dataset = CatsDogsDataset(img_dir=os.path.join('dogs-vs-cats', 'valid'), \n transform=data_transforms['valid'])\n\nvalid_loader = DataLoader(dataset=valid_dataset, \n batch_size=BATCH_SIZE, \n num_workers=NUM_WORKERS,\n shuffle=False)\n\ntest_dataset = CatsDogsDataset(img_dir=os.path.join('dogs-vs-cats', 'test'), \n transform=data_transforms['valid'])\n\ntest_loader = DataLoader(dataset=test_dataset, \n batch_size=BATCH_SIZE, \n num_workers=NUM_WORKERS,\n shuffle=False)\n"
real_batch = next(iter(train_loader))
plt.figure(figsize=(8, 8))
plt.axis("off")
plt.title("Training Images")
plt.imshow(np.transpose(vutils.make_grid(real_batch[0][:64],
padding=2, normalize=True),
(1, 2, 0)))
<matplotlib.image.AxesImage at 0x7f89e80dd4a8>
# Some model code is loosely inspired by
# https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html
def weights_init(module):
"""
Function that initializes weights according to
Radford et al.'s DCGAN paper
"""
classname = module.__class__.__name__
if classname.find('Conv') != -1:
nn.init.normal_(module.weight.data, 0.0, 0.02)
elif classname.find('BatchNorm') != -1:
nn.init.normal_(module.weight.data, 1.0, 0.02)
nn.init.constant_(module.bias.data, 0)
##########################
### MODEL
##########################
class DCGAN(torch.nn.Module):
def __init__(self):
super(DCGAN, self).__init__()
self.generator = nn.Sequential(
#
# input size: vector z of size LATENT_DIM
#
nn.ConvTranspose2d(LATENT_DIM, NUM_MAPS_GEN*8,
kernel_size=4, stride=1, padding=0,
bias=False), # bias is redundant when using BatchNorm
nn.BatchNorm2d(NUM_MAPS_GEN*8),
nn.ReLU(True),
#
# size: NUM_MAPS_GEN*8 x 4 x 4
#
nn.ConvTranspose2d(NUM_MAPS_GEN*8, NUM_MAPS_GEN*4,
kernel_size=4, stride=2, padding=1,
bias=False),
nn.BatchNorm2d(NUM_MAPS_GEN*4),
nn.ReLU(True),
#
# size: NUM_MAPS_GEN*4 x 8 x 8
#
nn.ConvTranspose2d(NUM_MAPS_GEN*4, NUM_MAPS_GEN*2,
kernel_size=4, stride=2, padding=1,
bias=False),
nn.BatchNorm2d(NUM_MAPS_GEN*2),
nn.ReLU(True),
#
# size: NUM_MAPS_GEN*2 x 16 x 16
#
nn.ConvTranspose2d(NUM_MAPS_GEN*2, NUM_MAPS_GEN,
kernel_size=4, stride=2, padding=1,
bias=False),
nn.BatchNorm2d(NUM_MAPS_GEN),
nn.ReLU(True),
#
# size: NUM_MAPS_GEN x 32 x 32
#
nn.ConvTranspose2d(NUM_MAPS_GEN, IMAGE_SIZE[2],
kernel_size=4, stride=2, padding=1,
bias=False),
#
# size: IMAGE_SIZE[2] x 64 x 64
#
nn.Tanh()
)
self.discriminator = nn.Sequential(
#
# input size IMAGE_SIZE[2] x IMAGE_SIZE[0] x IMAGE_SIZE[1]
#
nn.Conv2d(IMAGE_SIZE[2], NUM_MAPS_DIS,
kernel_size=4, stride=2, padding=1),
nn.LeakyReLU(0.2, inplace=True),
#
# size: NUM_MAPS_DIS x 32 x 32
#
nn.Conv2d(NUM_MAPS_DIS, NUM_MAPS_DIS*2,
kernel_size=4, stride=2, padding=1,
bias=False),
nn.BatchNorm2d(NUM_MAPS_DIS*2),
nn.LeakyReLU(0.2, inplace=True),
#
# size: NUM_MAPS_DIS*2 x 16 x 16
#
nn.Conv2d(NUM_MAPS_DIS*2, NUM_MAPS_DIS*4,
kernel_size=4, stride=2, padding=1,
bias=False),
nn.BatchNorm2d(NUM_MAPS_DIS*4),
nn.LeakyReLU(0.2, inplace=True),
#
# size: NUM_MAPS_DIS*4 x 8 x 8
#
nn.Conv2d(NUM_MAPS_DIS*4, NUM_MAPS_DIS*8,
kernel_size=4, stride=2, padding=1,
bias=False),
nn.BatchNorm2d(NUM_MAPS_DIS*8),
nn.LeakyReLU(0.2, inplace=True),
#
# size: NUM_MAPS_DIS*8 x 4 x 4
#
nn.Conv2d(NUM_MAPS_DIS*8, 1,
kernel_size=4, stride=1, padding=0),
nn.Sigmoid()
)
def generator_forward(self, z):
img = self.generator(z)
return img
def discriminator_forward(self, img):
pred = model.discriminator(img)
return pred
torch.manual_seed(RANDOM_SEED)
loss_function = nn.BCELoss()
real_label = 1
fake_label = 0
# Batch of latent (noise) vectors for
# evaluating / visualizing the training progress
# of the generator
fixed_noise = torch.randn(64, LATENT_DIM, 1, 1, device=DEVICE)
model = DCGAN()
model = model.to(DEVICE)
model.apply(weights_init)
print(model)
DCGAN( (generator): Sequential( (0): ConvTranspose2d(100, 512, kernel_size=(4, 4), stride=(1, 1), bias=False) (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (2): ReLU(inplace=True) (3): ConvTranspose2d(512, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (5): ReLU(inplace=True) (6): ConvTranspose2d(256, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (8): ReLU(inplace=True) (9): ConvTranspose2d(128, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) (10): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (11): ReLU(inplace=True) (12): ConvTranspose2d(64, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) (13): Tanh() ) (discriminator): Sequential( (0): Conv2d(3, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) (1): LeakyReLU(negative_slope=0.2, inplace=True) (2): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (4): LeakyReLU(negative_slope=0.2, inplace=True) (5): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) (6): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (7): LeakyReLU(negative_slope=0.2, inplace=True) (8): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) (9): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (10): LeakyReLU(negative_slope=0.2, inplace=True) (11): Conv2d(512, 1, kernel_size=(4, 4), stride=(1, 1)) (12): Sigmoid() ) )
from torchsummary import summary
# torchsummary can only use default cuda device, which
# causes issues if e.g., cuda:1 is used
with torch.cuda.device(int(CUDA.split(':')[-1])):
summary(model.generator, input_size=(100, 1, 1), device='cuda')
summary(model.discriminator, input_size=((IMAGE_SIZE[2], IMAGE_SIZE[0], IMAGE_SIZE[1])), device='cuda')
---------------------------------------------------------------- Layer (type) Output Shape Param # ================================================================ ConvTranspose2d-1 [-1, 512, 4, 4] 819,200 BatchNorm2d-2 [-1, 512, 4, 4] 1,024 ReLU-3 [-1, 512, 4, 4] 0 ConvTranspose2d-4 [-1, 256, 8, 8] 2,097,152 BatchNorm2d-5 [-1, 256, 8, 8] 512 ReLU-6 [-1, 256, 8, 8] 0 ConvTranspose2d-7 [-1, 128, 16, 16] 524,288 BatchNorm2d-8 [-1, 128, 16, 16] 256 ReLU-9 [-1, 128, 16, 16] 0 ConvTranspose2d-10 [-1, 64, 32, 32] 131,072 BatchNorm2d-11 [-1, 64, 32, 32] 128 ReLU-12 [-1, 64, 32, 32] 0 ConvTranspose2d-13 [-1, 3, 64, 64] 3,072 Tanh-14 [-1, 3, 64, 64] 0 ================================================================ Total params: 3,576,704 Trainable params: 3,576,704 Non-trainable params: 0 ---------------------------------------------------------------- Input size (MB): 0.00 Forward/backward pass size (MB): 3.00 Params size (MB): 13.64 Estimated Total Size (MB): 16.64 ---------------------------------------------------------------- ---------------------------------------------------------------- Layer (type) Output Shape Param # ================================================================ Conv2d-1 [-1, 64, 32, 32] 3,136 LeakyReLU-2 [-1, 64, 32, 32] 0 Conv2d-3 [-1, 128, 16, 16] 131,072 BatchNorm2d-4 [-1, 128, 16, 16] 256 LeakyReLU-5 [-1, 128, 16, 16] 0 Conv2d-6 [-1, 256, 8, 8] 524,288 BatchNorm2d-7 [-1, 256, 8, 8] 512 LeakyReLU-8 [-1, 256, 8, 8] 0 Conv2d-9 [-1, 512, 4, 4] 2,097,152 BatchNorm2d-10 [-1, 512, 4, 4] 1,024 LeakyReLU-11 [-1, 512, 4, 4] 0 Conv2d-12 [-1, 1, 1, 1] 8,193 Sigmoid-13 [-1, 1, 1, 1] 0 ================================================================ Total params: 2,765,633 Trainable params: 2,765,633 Non-trainable params: 0 ---------------------------------------------------------------- Input size (MB): 0.05 Forward/backward pass size (MB): 2.31 Params size (MB): 10.55 Estimated Total Size (MB): 12.91 ----------------------------------------------------------------
optim_gener = torch.optim.Adam(model.generator.parameters(),
betas=(0.5, 0.999),
lr=GENERATOR_LEARNING_RATE)
optim_discr = torch.optim.Adam(model.discriminator.parameters(),
betas=(0.5, 0.999),
lr=DISCRIMINATOR_LEARNING_RATE)
start_time = time.time()
discr_costs = []
gener_costs = []
images_from_noise = []
for epoch in range(NUM_EPOCHS):
model = model.train()
for batch_idx, (features, targets) in enumerate(train_loader):
# --------------------------
# Train Discriminator
# --------------------------
optim_discr.zero_grad()
real_images = features.to(DEVICE)
num_real = real_images.size(0)
real_label_vec = torch.full((num_real,), real_label, device=DEVICE)
# get discriminator loss on real images
discr_pred_real = model.discriminator_forward(real_images).view(-1)
real_loss = loss_function(discr_pred_real, real_label_vec)
#real_loss.backward()
# get discriminator loss on fake images
random_vec = torch.randn(BATCH_SIZE, LATENT_DIM, 1, 1, device=DEVICE)
fake_images = model.generator_forward(random_vec)
fake_label_vec = torch.full((num_real,), fake_label, device=DEVICE)
discr_pred_fake = model.discriminator_forward(fake_images.detach()).view(-1)
fake_loss = loss_function(discr_pred_fake, fake_label_vec)
#fake_loss.backward()
# combined loss
discr_loss = 0.5*(real_loss + fake_loss)
discr_loss.backward()
optim_discr.step()
# --------------------------
# Train Generator
# --------------------------
optim_gener.zero_grad()
discr_pred_fake = model.discriminator_forward(fake_images).view(-1)
gener_loss = loss_function(discr_pred_fake, real_label_vec)
gener_loss.backward()
optim_gener.step()
# --------------------------
# Logging
# --------------------------
discr_costs.append(discr_loss.item())
gener_costs.append(gener_loss.item())
### LOGGING
if not batch_idx % 100:
print ('Epoch: %03d/%03d | Batch %03d/%03d | Gen/Dis Loss: %.4f/%.4f'
%(epoch+1, NUM_EPOCHS, batch_idx,
len(train_loader), gener_loss, discr_loss))
### Save images for evaluation
with torch.no_grad():
fake_images = model.generator_forward(fixed_noise).detach().cpu()
images_from_noise.append(
vutils.make_grid(fake_images, padding=2, normalize=True))
print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))
print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))
Epoch: 001/100 | Batch 000/156 | Gen/Dis Loss: 5.6721/0.9069 Epoch: 001/100 | Batch 100/156 | Gen/Dis Loss: 4.3278/0.1863 Time elapsed: 0.31 min Epoch: 002/100 | Batch 000/156 | Gen/Dis Loss: 8.5901/0.2524 Epoch: 002/100 | Batch 100/156 | Gen/Dis Loss: 4.8214/0.2331 Time elapsed: 0.61 min Epoch: 003/100 | Batch 000/156 | Gen/Dis Loss: 2.9616/0.2041 Epoch: 003/100 | Batch 100/156 | Gen/Dis Loss: 2.1460/0.4449 Time elapsed: 0.91 min Epoch: 004/100 | Batch 000/156 | Gen/Dis Loss: 2.7284/0.3312 Epoch: 004/100 | Batch 100/156 | Gen/Dis Loss: 6.1412/0.3749 Time elapsed: 1.22 min Epoch: 005/100 | Batch 000/156 | Gen/Dis Loss: 6.1561/0.4841 Epoch: 005/100 | Batch 100/156 | Gen/Dis Loss: 4.2732/0.2204 Time elapsed: 1.53 min Epoch: 006/100 | Batch 000/156 | Gen/Dis Loss: 4.7866/0.3029 Epoch: 006/100 | Batch 100/156 | Gen/Dis Loss: 3.3484/0.2923 Time elapsed: 1.83 min Epoch: 007/100 | Batch 000/156 | Gen/Dis Loss: 4.8264/0.2859 Epoch: 007/100 | Batch 100/156 | Gen/Dis Loss: 4.5199/0.3443 Time elapsed: 2.14 min Epoch: 008/100 | Batch 000/156 | Gen/Dis Loss: 3.5742/0.2299 Epoch: 008/100 | Batch 100/156 | Gen/Dis Loss: 2.3944/0.2886 Time elapsed: 2.44 min Epoch: 009/100 | Batch 000/156 | Gen/Dis Loss: 5.7034/0.6672 Epoch: 009/100 | Batch 100/156 | Gen/Dis Loss: 3.0610/0.2283 Time elapsed: 2.75 min Epoch: 010/100 | Batch 000/156 | Gen/Dis Loss: 3.7279/0.1488 Epoch: 010/100 | Batch 100/156 | Gen/Dis Loss: 4.0779/0.2247 Time elapsed: 3.05 min Epoch: 011/100 | Batch 000/156 | Gen/Dis Loss: 5.0260/0.4252 Epoch: 011/100 | Batch 100/156 | Gen/Dis Loss: 4.0253/0.2887 Time elapsed: 3.36 min Epoch: 012/100 | Batch 000/156 | Gen/Dis Loss: 2.7586/0.4184 Epoch: 012/100 | Batch 100/156 | Gen/Dis Loss: 2.7669/0.3326 Time elapsed: 3.67 min Epoch: 013/100 | Batch 000/156 | Gen/Dis Loss: 3.0632/0.3356 Epoch: 013/100 | Batch 100/156 | Gen/Dis Loss: 1.8059/0.3413 Time elapsed: 3.98 min Epoch: 014/100 | Batch 000/156 | Gen/Dis Loss: 4.8230/0.8299 Epoch: 014/100 | Batch 100/156 | Gen/Dis Loss: 3.0577/0.2651 Time elapsed: 4.28 min Epoch: 015/100 | Batch 000/156 | Gen/Dis Loss: 0.9512/0.7387 Epoch: 015/100 | Batch 100/156 | Gen/Dis Loss: 3.5348/0.3110 Time elapsed: 4.59 min Epoch: 016/100 | Batch 000/156 | Gen/Dis Loss: 1.4383/0.3949 Epoch: 016/100 | Batch 100/156 | Gen/Dis Loss: 2.6647/0.3474 Time elapsed: 4.89 min Epoch: 017/100 | Batch 000/156 | Gen/Dis Loss: 4.0802/0.3814 Epoch: 017/100 | Batch 100/156 | Gen/Dis Loss: 1.2604/0.4630 Time elapsed: 5.20 min Epoch: 018/100 | Batch 000/156 | Gen/Dis Loss: 1.9800/0.2677 Epoch: 018/100 | Batch 100/156 | Gen/Dis Loss: 1.3349/0.5785 Time elapsed: 5.51 min Epoch: 019/100 | Batch 000/156 | Gen/Dis Loss: 1.4303/0.3838 Epoch: 019/100 | Batch 100/156 | Gen/Dis Loss: 4.4391/0.5851 Time elapsed: 5.82 min Epoch: 020/100 | Batch 000/156 | Gen/Dis Loss: 2.7683/0.2513 Epoch: 020/100 | Batch 100/156 | Gen/Dis Loss: 2.1813/0.3578 Time elapsed: 6.12 min Epoch: 021/100 | Batch 000/156 | Gen/Dis Loss: 3.2441/0.1968 Epoch: 021/100 | Batch 100/156 | Gen/Dis Loss: 2.8597/0.2032 Time elapsed: 6.43 min Epoch: 022/100 | Batch 000/156 | Gen/Dis Loss: 3.4742/0.3404 Epoch: 022/100 | Batch 100/156 | Gen/Dis Loss: 2.4937/0.2504 Time elapsed: 6.74 min Epoch: 023/100 | Batch 000/156 | Gen/Dis Loss: 1.9667/0.2644 Epoch: 023/100 | Batch 100/156 | Gen/Dis Loss: 1.5939/0.3413 Time elapsed: 7.04 min Epoch: 024/100 | Batch 000/156 | Gen/Dis Loss: 2.5891/0.3194 Epoch: 024/100 | Batch 100/156 | Gen/Dis Loss: 2.7795/0.3538 Time elapsed: 7.35 min Epoch: 025/100 | Batch 000/156 | Gen/Dis Loss: 3.0261/0.2235 Epoch: 025/100 | Batch 100/156 | Gen/Dis Loss: 4.0539/0.4257 Time elapsed: 7.65 min Epoch: 026/100 | Batch 000/156 | Gen/Dis Loss: 3.0815/0.2805 Epoch: 026/100 | Batch 100/156 | Gen/Dis Loss: 2.6320/0.2598 Time elapsed: 7.96 min Epoch: 027/100 | Batch 000/156 | Gen/Dis Loss: 3.3634/0.2299 Epoch: 027/100 | Batch 100/156 | Gen/Dis Loss: 2.9601/0.3076 Time elapsed: 8.27 min Epoch: 028/100 | Batch 000/156 | Gen/Dis Loss: 3.4327/0.1964 Epoch: 028/100 | Batch 100/156 | Gen/Dis Loss: 3.7440/0.3206 Time elapsed: 8.57 min Epoch: 029/100 | Batch 000/156 | Gen/Dis Loss: 3.4061/0.3449 Epoch: 029/100 | Batch 100/156 | Gen/Dis Loss: 2.5216/0.3170 Time elapsed: 8.88 min Epoch: 030/100 | Batch 000/156 | Gen/Dis Loss: 2.1389/0.2486 Epoch: 030/100 | Batch 100/156 | Gen/Dis Loss: 2.6346/0.3278 Time elapsed: 9.18 min Epoch: 031/100 | Batch 000/156 | Gen/Dis Loss: 2.9123/0.2027 Epoch: 031/100 | Batch 100/156 | Gen/Dis Loss: 2.6842/0.1727 Time elapsed: 9.49 min Epoch: 032/100 | Batch 000/156 | Gen/Dis Loss: 2.9600/0.2742 Epoch: 032/100 | Batch 100/156 | Gen/Dis Loss: 2.2887/0.3029 Time elapsed: 9.80 min Epoch: 033/100 | Batch 000/156 | Gen/Dis Loss: 4.6838/0.3094 Epoch: 033/100 | Batch 100/156 | Gen/Dis Loss: 5.3511/0.5146 Time elapsed: 10.11 min Epoch: 034/100 | Batch 000/156 | Gen/Dis Loss: 4.4270/0.2960 Epoch: 034/100 | Batch 100/156 | Gen/Dis Loss: 2.1736/0.2605 Time elapsed: 10.41 min Epoch: 035/100 | Batch 000/156 | Gen/Dis Loss: 3.2069/0.2123 Epoch: 035/100 | Batch 100/156 | Gen/Dis Loss: 2.8760/0.1864 Time elapsed: 10.72 min Epoch: 036/100 | Batch 000/156 | Gen/Dis Loss: 3.6395/0.2381 Epoch: 036/100 | Batch 100/156 | Gen/Dis Loss: 3.5067/0.2725 Time elapsed: 11.03 min Epoch: 037/100 | Batch 000/156 | Gen/Dis Loss: 3.0389/0.3297 Epoch: 037/100 | Batch 100/156 | Gen/Dis Loss: 2.6393/0.2121 Time elapsed: 11.33 min Epoch: 038/100 | Batch 000/156 | Gen/Dis Loss: 3.2688/0.3955 Epoch: 038/100 | Batch 100/156 | Gen/Dis Loss: 2.0117/0.2390 Time elapsed: 11.64 min Epoch: 039/100 | Batch 000/156 | Gen/Dis Loss: 1.4859/0.2720 Epoch: 039/100 | Batch 100/156 | Gen/Dis Loss: 2.0843/0.2558 Time elapsed: 11.95 min Epoch: 040/100 | Batch 000/156 | Gen/Dis Loss: 3.3301/0.2823 Epoch: 040/100 | Batch 100/156 | Gen/Dis Loss: 2.9919/0.3162 Time elapsed: 12.26 min Epoch: 041/100 | Batch 000/156 | Gen/Dis Loss: 3.4604/0.1882 Epoch: 041/100 | Batch 100/156 | Gen/Dis Loss: 3.1851/0.2583 Time elapsed: 12.56 min Epoch: 042/100 | Batch 000/156 | Gen/Dis Loss: 3.0730/0.1830 Epoch: 042/100 | Batch 100/156 | Gen/Dis Loss: 1.3499/0.5049 Time elapsed: 12.87 min Epoch: 043/100 | Batch 000/156 | Gen/Dis Loss: 3.2583/0.2386 Epoch: 043/100 | Batch 100/156 | Gen/Dis Loss: 6.0671/0.9041 Time elapsed: 13.18 min Epoch: 044/100 | Batch 000/156 | Gen/Dis Loss: 3.3314/0.1230 Epoch: 044/100 | Batch 100/156 | Gen/Dis Loss: 2.2566/0.2196 Time elapsed: 13.49 min Epoch: 045/100 | Batch 000/156 | Gen/Dis Loss: 2.6622/0.2602 Epoch: 045/100 | Batch 100/156 | Gen/Dis Loss: 1.1283/0.5150 Time elapsed: 13.80 min Epoch: 046/100 | Batch 000/156 | Gen/Dis Loss: 2.6093/0.2086 Epoch: 046/100 | Batch 100/156 | Gen/Dis Loss: 2.7870/0.1554 Time elapsed: 14.11 min Epoch: 047/100 | Batch 000/156 | Gen/Dis Loss: 3.9466/0.2605 Epoch: 047/100 | Batch 100/156 | Gen/Dis Loss: 2.2170/0.1837 Time elapsed: 14.41 min Epoch: 048/100 | Batch 000/156 | Gen/Dis Loss: 2.8530/0.1605 Epoch: 048/100 | Batch 100/156 | Gen/Dis Loss: 0.9577/0.6123 Time elapsed: 14.72 min Epoch: 049/100 | Batch 000/156 | Gen/Dis Loss: 2.7477/0.2290 Epoch: 049/100 | Batch 100/156 | Gen/Dis Loss: 5.7200/0.3954 Time elapsed: 15.03 min Epoch: 050/100 | Batch 000/156 | Gen/Dis Loss: 4.2994/0.2462 Epoch: 050/100 | Batch 100/156 | Gen/Dis Loss: 3.3307/0.1276 Time elapsed: 15.33 min Epoch: 051/100 | Batch 000/156 | Gen/Dis Loss: 2.6723/0.2995 Epoch: 051/100 | Batch 100/156 | Gen/Dis Loss: 3.1913/0.1793 Time elapsed: 15.64 min Epoch: 052/100 | Batch 000/156 | Gen/Dis Loss: 3.7864/0.1874 Epoch: 052/100 | Batch 100/156 | Gen/Dis Loss: 1.8820/0.2316 Time elapsed: 15.95 min Epoch: 053/100 | Batch 000/156 | Gen/Dis Loss: 2.6927/0.2020 Epoch: 053/100 | Batch 100/156 | Gen/Dis Loss: 2.6530/0.1848 Time elapsed: 16.25 min Epoch: 054/100 | Batch 000/156 | Gen/Dis Loss: 2.9418/0.1690 Epoch: 054/100 | Batch 100/156 | Gen/Dis Loss: 3.7257/0.2603 Time elapsed: 16.56 min Epoch: 055/100 | Batch 000/156 | Gen/Dis Loss: 4.7709/0.2005 Epoch: 055/100 | Batch 100/156 | Gen/Dis Loss: 2.4623/0.1317 Time elapsed: 16.87 min Epoch: 056/100 | Batch 000/156 | Gen/Dis Loss: 4.0949/0.1041 Epoch: 056/100 | Batch 100/156 | Gen/Dis Loss: 3.1125/0.1175 Time elapsed: 17.17 min Epoch: 057/100 | Batch 000/156 | Gen/Dis Loss: 1.5920/1.2699 Epoch: 057/100 | Batch 100/156 | Gen/Dis Loss: 7.6799/0.7797 Time elapsed: 17.48 min Epoch: 058/100 | Batch 000/156 | Gen/Dis Loss: 3.8176/0.2780 Epoch: 058/100 | Batch 100/156 | Gen/Dis Loss: 3.1603/0.2012 Time elapsed: 17.79 min Epoch: 059/100 | Batch 000/156 | Gen/Dis Loss: 2.3409/0.2488 Epoch: 059/100 | Batch 100/156 | Gen/Dis Loss: 3.7527/0.1560 Time elapsed: 18.10 min Epoch: 060/100 | Batch 000/156 | Gen/Dis Loss: 2.7943/0.6130 Epoch: 060/100 | Batch 100/156 | Gen/Dis Loss: 2.6994/0.2059 Time elapsed: 18.40 min Epoch: 061/100 | Batch 000/156 | Gen/Dis Loss: 3.1508/0.1310 Epoch: 061/100 | Batch 100/156 | Gen/Dis Loss: 0.7574/0.4549 Time elapsed: 18.71 min Epoch: 062/100 | Batch 000/156 | Gen/Dis Loss: 2.5987/0.1779 Epoch: 062/100 | Batch 100/156 | Gen/Dis Loss: 2.5146/0.1761 Time elapsed: 19.01 min Epoch: 063/100 | Batch 000/156 | Gen/Dis Loss: 2.8739/0.1317 Epoch: 063/100 | Batch 100/156 | Gen/Dis Loss: 2.9416/0.1502 Time elapsed: 19.32 min Epoch: 064/100 | Batch 000/156 | Gen/Dis Loss: 4.2328/0.1938 Epoch: 064/100 | Batch 100/156 | Gen/Dis Loss: 3.7243/0.1294 Time elapsed: 19.63 min Epoch: 065/100 | Batch 000/156 | Gen/Dis Loss: 2.5783/0.1959 Epoch: 065/100 | Batch 100/156 | Gen/Dis Loss: 1.9980/0.1570 Time elapsed: 19.93 min Epoch: 066/100 | Batch 000/156 | Gen/Dis Loss: 4.1402/0.1897 Epoch: 066/100 | Batch 100/156 | Gen/Dis Loss: 2.4067/0.2341 Time elapsed: 20.24 min Epoch: 067/100 | Batch 000/156 | Gen/Dis Loss: 0.1102/1.7427 Epoch: 067/100 | Batch 100/156 | Gen/Dis Loss: 2.1101/0.1876 Time elapsed: 20.55 min Epoch: 068/100 | Batch 000/156 | Gen/Dis Loss: 2.6289/0.1745 Epoch: 068/100 | Batch 100/156 | Gen/Dis Loss: 0.8209/0.6411 Time elapsed: 20.85 min Epoch: 069/100 | Batch 000/156 | Gen/Dis Loss: 1.7605/0.2849 Epoch: 069/100 | Batch 100/156 | Gen/Dis Loss: 2.0866/0.2438 Time elapsed: 21.16 min Epoch: 070/100 | Batch 000/156 | Gen/Dis Loss: 5.1044/0.1987 Epoch: 070/100 | Batch 100/156 | Gen/Dis Loss: 3.8816/0.1212 Time elapsed: 21.47 min Epoch: 071/100 | Batch 000/156 | Gen/Dis Loss: 2.6012/0.2113 Epoch: 071/100 | Batch 100/156 | Gen/Dis Loss: 3.4299/4.3089 Time elapsed: 21.78 min Epoch: 072/100 | Batch 000/156 | Gen/Dis Loss: 4.7329/0.5443 Epoch: 072/100 | Batch 100/156 | Gen/Dis Loss: 3.0962/0.1495 Time elapsed: 22.09 min Epoch: 073/100 | Batch 000/156 | Gen/Dis Loss: 3.8376/0.1262 Epoch: 073/100 | Batch 100/156 | Gen/Dis Loss: 1.6284/0.2547 Time elapsed: 22.39 min Epoch: 074/100 | Batch 000/156 | Gen/Dis Loss: 3.7829/0.1239 Epoch: 074/100 | Batch 100/156 | Gen/Dis Loss: 2.5319/0.1236 Time elapsed: 22.70 min Epoch: 075/100 | Batch 000/156 | Gen/Dis Loss: 4.3512/0.1529 Epoch: 075/100 | Batch 100/156 | Gen/Dis Loss: 2.2636/0.1287 Time elapsed: 23.01 min Epoch: 076/100 | Batch 000/156 | Gen/Dis Loss: 2.3775/0.4868 Epoch: 076/100 | Batch 100/156 | Gen/Dis Loss: 3.6431/0.0947 Time elapsed: 23.32 min Epoch: 077/100 | Batch 000/156 | Gen/Dis Loss: 3.3066/0.0844 Epoch: 077/100 | Batch 100/156 | Gen/Dis Loss: 2.6288/0.1279 Time elapsed: 23.63 min Epoch: 078/100 | Batch 000/156 | Gen/Dis Loss: 3.0262/0.1441 Epoch: 078/100 | Batch 100/156 | Gen/Dis Loss: 0.3917/0.3608 Time elapsed: 23.94 min Epoch: 079/100 | Batch 000/156 | Gen/Dis Loss: 4.0185/0.1878 Epoch: 079/100 | Batch 100/156 | Gen/Dis Loss: 3.6610/0.1182 Time elapsed: 24.24 min Epoch: 080/100 | Batch 000/156 | Gen/Dis Loss: 5.0186/0.1827 Epoch: 080/100 | Batch 100/156 | Gen/Dis Loss: 3.3670/0.1525 Time elapsed: 24.55 min Epoch: 081/100 | Batch 000/156 | Gen/Dis Loss: 4.3158/0.0521 Epoch: 081/100 | Batch 100/156 | Gen/Dis Loss: 4.2761/0.0644 Time elapsed: 24.86 min Epoch: 082/100 | Batch 000/156 | Gen/Dis Loss: 2.3813/0.5657 Epoch: 082/100 | Batch 100/156 | Gen/Dis Loss: 1.9467/0.1995 Time elapsed: 25.16 min Epoch: 083/100 | Batch 000/156 | Gen/Dis Loss: 3.8486/0.1068 Epoch: 083/100 | Batch 100/156 | Gen/Dis Loss: 3.8631/0.1093 Time elapsed: 25.47 min Epoch: 084/100 | Batch 000/156 | Gen/Dis Loss: 3.4586/0.0777 Epoch: 084/100 | Batch 100/156 | Gen/Dis Loss: 2.8999/0.1149 Time elapsed: 25.78 min Epoch: 085/100 | Batch 000/156 | Gen/Dis Loss: 3.8060/0.1013 Epoch: 085/100 | Batch 100/156 | Gen/Dis Loss: 10.5981/4.2818 Time elapsed: 26.09 min Epoch: 086/100 | Batch 000/156 | Gen/Dis Loss: 2.5460/0.2426 Epoch: 086/100 | Batch 100/156 | Gen/Dis Loss: 2.8276/0.1161 Time elapsed: 26.39 min Epoch: 087/100 | Batch 000/156 | Gen/Dis Loss: 4.2669/0.1272 Epoch: 087/100 | Batch 100/156 | Gen/Dis Loss: 3.6149/0.0724 Time elapsed: 26.70 min Epoch: 088/100 | Batch 000/156 | Gen/Dis Loss: 2.8210/0.3012 Epoch: 088/100 | Batch 100/156 | Gen/Dis Loss: 4.2154/0.0838 Time elapsed: 27.01 min Epoch: 089/100 | Batch 000/156 | Gen/Dis Loss: 3.5050/0.0905 Epoch: 089/100 | Batch 100/156 | Gen/Dis Loss: 2.7687/0.1155 Time elapsed: 27.31 min Epoch: 090/100 | Batch 000/156 | Gen/Dis Loss: 3.6506/0.0640 Epoch: 090/100 | Batch 100/156 | Gen/Dis Loss: 0.1520/1.6858 Time elapsed: 27.62 min Epoch: 091/100 | Batch 000/156 | Gen/Dis Loss: 4.2384/0.1773 Epoch: 091/100 | Batch 100/156 | Gen/Dis Loss: 3.3331/0.1184 Time elapsed: 27.93 min Epoch: 092/100 | Batch 000/156 | Gen/Dis Loss: 3.9736/0.0747 Epoch: 092/100 | Batch 100/156 | Gen/Dis Loss: 2.7988/0.0946 Time elapsed: 28.23 min Epoch: 093/100 | Batch 000/156 | Gen/Dis Loss: 2.6911/0.1160 Epoch: 093/100 | Batch 100/156 | Gen/Dis Loss: 3.5433/0.0685 Time elapsed: 28.54 min Epoch: 094/100 | Batch 000/156 | Gen/Dis Loss: 6.6064/0.4917 Epoch: 094/100 | Batch 100/156 | Gen/Dis Loss: 2.8160/0.1301 Time elapsed: 28.85 min Epoch: 095/100 | Batch 000/156 | Gen/Dis Loss: 4.4914/0.0762 Epoch: 095/100 | Batch 100/156 | Gen/Dis Loss: 4.9280/0.0877 Time elapsed: 29.15 min Epoch: 096/100 | Batch 000/156 | Gen/Dis Loss: 4.7079/0.0516 Epoch: 096/100 | Batch 100/156 | Gen/Dis Loss: 4.0562/0.2307 Time elapsed: 29.48 min Epoch: 097/100 | Batch 000/156 | Gen/Dis Loss: 4.0915/0.0834 Epoch: 097/100 | Batch 100/156 | Gen/Dis Loss: 4.2067/0.1276 Time elapsed: 29.79 min Epoch: 098/100 | Batch 000/156 | Gen/Dis Loss: 4.3396/0.0750 Epoch: 098/100 | Batch 100/156 | Gen/Dis Loss: 3.9746/0.0395 Time elapsed: 30.10 min Epoch: 099/100 | Batch 000/156 | Gen/Dis Loss: 5.8522/0.0290 Epoch: 099/100 | Batch 100/156 | Gen/Dis Loss: 4.1718/0.2145 Time elapsed: 30.41 min Epoch: 100/100 | Batch 000/156 | Gen/Dis Loss: 3.9022/0.1663 Epoch: 100/100 | Batch 100/156 | Gen/Dis Loss: 3.1544/0.0956 Time elapsed: 30.72 min Total Training Time: 30.72 min
%matplotlib inline
import matplotlib.pyplot as plt
ax1 = plt.subplot(1, 1, 1)
ax1.plot(range(len(gener_costs)), gener_costs, label='Generator loss')
ax1.plot(range(len(discr_costs)), discr_costs, label='Discriminator loss')
ax1.set_xlabel('Iterations')
ax1.set_ylabel('Loss')
ax1.legend()
###################
# Set scond x-axis
ax2 = ax1.twiny()
newlabel = list(range(NUM_EPOCHS+1))
iter_per_epoch = len(train_loader)
newpos = [e*iter_per_epoch for e in newlabel]
ax2.set_xticklabels(newlabel[::10])
ax2.set_xticks(newpos[::10])
ax2.xaxis.set_ticks_position('bottom')
ax2.xaxis.set_label_position('bottom')
ax2.spines['bottom'].set_position(('outward', 45))
ax2.set_xlabel('Epochs')
ax2.set_xlim(ax1.get_xlim())
###################
plt.show()
##########################
### VISUALIZATION
##########################
for i in range(0, NUM_EPOCHS, 5):
plt.imshow(np.transpose(images_from_noise[i], (1, 2, 0)))
plt.show()
plt.figure(figsize=(10, 10))
plt.imshow(np.transpose(images_from_noise[-1], (1, 2, 0)))
plt.show()