Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.
%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -p torch
Sebastian Raschka CPython 3.7.3 IPython 7.6.1 torch 1.3.0
import time
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import datasets
from torchvision import transforms
from torch.utils.data import DataLoader
if torch.cuda.is_available():
torch.backends.cudnn.deterministic = True
##########################
### SETTINGS
##########################
# Device
DEVICE = torch.device("cuda:3" if torch.cuda.is_available() else "cpu")
print('Device:', DEVICE)
# Hyperparameters
random_seed = 1
learning_rate = 0.001
num_epochs = 10
batch_size = 128
# Architecture
num_features = 784
num_classes = 10
##########################
### MNIST DATASET
##########################
# Note transforms.ToTensor() scales input images
# to 0-1 range
train_dataset = datasets.CIFAR10(root='data',
train=True,
transform=transforms.ToTensor(),
download=True)
test_dataset = datasets.CIFAR10(root='data',
train=False,
transform=transforms.ToTensor())
train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
test_loader = DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)
# Checking the dataset
for images, labels in train_loader:
print('Image batch dimensions:', images.shape)
print('Image label dimensions:', labels.shape)
break
Device: cuda:3 Files already downloaded and verified Image batch dimensions: torch.Size([128, 3, 32, 32]) Image label dimensions: torch.Size([128])
##########################
### MODEL
##########################
class VGG16(torch.nn.Module):
def __init__(self, num_features, num_classes):
super(VGG16, self).__init__()
# calculate same padding:
# (w - k + 2*p)/s + 1 = o
# => p = (s(o-1) - w + k)/2
self.block_1 = nn.Sequential(
nn.Conv2d(in_channels=3,
out_channels=64,
kernel_size=(3, 3),
stride=(1, 1),
# (1(32-1)- 32 + 3)/2 = 1
padding=1),
nn.ReLU(),
nn.Conv2d(in_channels=64,
out_channels=64,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.MaxPool2d(kernel_size=(2, 2),
stride=(2, 2))
)
self.block_2 = nn.Sequential(
nn.Conv2d(in_channels=64,
out_channels=128,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.Conv2d(in_channels=128,
out_channels=128,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.MaxPool2d(kernel_size=(2, 2),
stride=(2, 2))
)
self.block_3 = nn.Sequential(
nn.Conv2d(in_channels=128,
out_channels=256,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.Conv2d(in_channels=256,
out_channels=256,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.Conv2d(in_channels=256,
out_channels=256,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.MaxPool2d(kernel_size=(2, 2),
stride=(2, 2))
)
self.block_4 = nn.Sequential(
nn.Conv2d(in_channels=256,
out_channels=512,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.Conv2d(in_channels=512,
out_channels=512,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.Conv2d(in_channels=512,
out_channels=512,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.MaxPool2d(kernel_size=(2, 2),
stride=(2, 2))
)
self.block_5 = nn.Sequential(
nn.Conv2d(in_channels=512,
out_channels=512,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.Conv2d(in_channels=512,
out_channels=512,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.Conv2d(in_channels=512,
out_channels=512,
kernel_size=(3, 3),
stride=(1, 1),
padding=1),
nn.ReLU(),
nn.MaxPool2d(kernel_size=(2, 2),
stride=(2, 2))
)
self.classifier = nn.Sequential(
nn.Linear(512, 4096),
nn.ReLU(True),
#nn.Dropout(p=0.5),
nn.Linear(4096, 4096),
nn.ReLU(True),
#nn.Dropout(p=0.5),
nn.Linear(4096, num_classes),
)
for m in self.modules():
if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):
nn.init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='relu')
if m.bias is not None:
m.bias.detach().zero_()
#self.avgpool = nn.AdaptiveAvgPool2d((7, 7))
def forward(self, x):
x = self.block_1(x)
x = self.block_2(x)
x = self.block_3(x)
x = self.block_4(x)
x = self.block_5(x)
#x = self.avgpool(x)
x = x.view(x.size(0), -1)
logits = self.classifier(x)
probas = F.softmax(logits, dim=1)
return logits, probas
torch.manual_seed(random_seed)
model = VGG16(num_features=num_features,
num_classes=num_classes)
model = model.to(DEVICE)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
def compute_accuracy(model, data_loader):
model.eval()
correct_pred, num_examples = 0, 0
for i, (features, targets) in enumerate(data_loader):
features = features.to(DEVICE)
targets = targets.to(DEVICE)
logits, probas = model(features)
_, predicted_labels = torch.max(probas, 1)
num_examples += targets.size(0)
correct_pred += (predicted_labels == targets).sum()
return correct_pred.float()/num_examples * 100
def compute_epoch_loss(model, data_loader):
model.eval()
curr_loss, num_examples = 0., 0
with torch.no_grad():
for features, targets in data_loader:
features = features.to(DEVICE)
targets = targets.to(DEVICE)
logits, probas = model(features)
loss = F.cross_entropy(logits, targets, reduction='sum')
num_examples += targets.size(0)
curr_loss += loss
curr_loss = curr_loss / num_examples
return curr_loss
start_time = time.time()
for epoch in range(num_epochs):
model.train()
for batch_idx, (features, targets) in enumerate(train_loader):
features = features.to(DEVICE)
targets = targets.to(DEVICE)
### FORWARD AND BACK PROP
logits, probas = model(features)
cost = F.cross_entropy(logits, targets)
optimizer.zero_grad()
cost.backward()
### UPDATE MODEL PARAMETERS
optimizer.step()
### LOGGING
if not batch_idx % 50:
print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f'
%(epoch+1, num_epochs, batch_idx,
len(train_loader), cost))
model.eval()
with torch.set_grad_enabled(False): # save memory during inference
print('Epoch: %03d/%03d | Train: %.3f%% | Loss: %.3f' % (
epoch+1, num_epochs,
compute_accuracy(model, train_loader),
compute_epoch_loss(model, train_loader)))
print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))
print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))
Epoch: 001/010 | Batch 0000/0391 | Cost: 2.4443 Epoch: 001/010 | Batch 0050/0391 | Cost: 2.2800 Epoch: 001/010 | Batch 0100/0391 | Cost: 2.1115 Epoch: 001/010 | Batch 0150/0391 | Cost: 1.9596 Epoch: 001/010 | Batch 0200/0391 | Cost: 2.0510 Epoch: 001/010 | Batch 0250/0391 | Cost: 1.8116 Epoch: 001/010 | Batch 0300/0391 | Cost: 1.8209 Epoch: 001/010 | Batch 0350/0391 | Cost: 1.7161 Epoch: 001/010 | Train: 36.232% | Loss: 1.600 Time elapsed: 0.59 min Epoch: 002/010 | Batch 0000/0391 | Cost: 1.6484 Epoch: 002/010 | Batch 0050/0391 | Cost: 1.8022 Epoch: 002/010 | Batch 0100/0391 | Cost: 1.6488 Epoch: 002/010 | Batch 0150/0391 | Cost: 1.5652 Epoch: 002/010 | Batch 0200/0391 | Cost: 1.3576 Epoch: 002/010 | Batch 0250/0391 | Cost: 1.4434 Epoch: 002/010 | Batch 0300/0391 | Cost: 1.3092 Epoch: 002/010 | Batch 0350/0391 | Cost: 1.1899 Epoch: 002/010 | Train: 55.148% | Loss: 1.229 Time elapsed: 1.17 min Epoch: 003/010 | Batch 0000/0391 | Cost: 1.1967 Epoch: 003/010 | Batch 0050/0391 | Cost: 1.1714 Epoch: 003/010 | Batch 0100/0391 | Cost: 1.3094 Epoch: 003/010 | Batch 0150/0391 | Cost: 1.2088 Epoch: 003/010 | Batch 0200/0391 | Cost: 1.2108 Epoch: 003/010 | Batch 0250/0391 | Cost: 0.9571 Epoch: 003/010 | Batch 0300/0391 | Cost: 1.0708 Epoch: 003/010 | Batch 0350/0391 | Cost: 0.9150 Epoch: 003/010 | Train: 67.218% | Loss: 0.938 Time elapsed: 1.77 min Epoch: 004/010 | Batch 0000/0391 | Cost: 0.8263 Epoch: 004/010 | Batch 0050/0391 | Cost: 0.9029 Epoch: 004/010 | Batch 0100/0391 | Cost: 1.1502 Epoch: 004/010 | Batch 0150/0391 | Cost: 0.9787 Epoch: 004/010 | Batch 0200/0391 | Cost: 0.8300 Epoch: 004/010 | Batch 0250/0391 | Cost: 1.0111 Epoch: 004/010 | Batch 0300/0391 | Cost: 0.9322 Epoch: 004/010 | Batch 0350/0391 | Cost: 0.9228 Epoch: 004/010 | Train: 70.676% | Loss: 0.852 Time elapsed: 2.36 min Epoch: 005/010 | Batch 0000/0391 | Cost: 0.8467 Epoch: 005/010 | Batch 0050/0391 | Cost: 0.6927 Epoch: 005/010 | Batch 0100/0391 | Cost: 0.8755 Epoch: 005/010 | Batch 0150/0391 | Cost: 0.8654 Epoch: 005/010 | Batch 0200/0391 | Cost: 0.8372 Epoch: 005/010 | Batch 0250/0391 | Cost: 0.6731 Epoch: 005/010 | Batch 0300/0391 | Cost: 0.6184 Epoch: 005/010 | Batch 0350/0391 | Cost: 0.6990 Epoch: 005/010 | Train: 73.508% | Loss: 0.764 Time elapsed: 2.96 min Epoch: 006/010 | Batch 0000/0391 | Cost: 0.7755 Epoch: 006/010 | Batch 0050/0391 | Cost: 0.5677 Epoch: 006/010 | Batch 0100/0391 | Cost: 0.8109 Epoch: 006/010 | Batch 0150/0391 | Cost: 0.5656 Epoch: 006/010 | Batch 0200/0391 | Cost: 0.5641 Epoch: 006/010 | Batch 0250/0391 | Cost: 0.8524 Epoch: 006/010 | Batch 0300/0391 | Cost: 0.8494 Epoch: 006/010 | Batch 0350/0391 | Cost: 0.6887 Epoch: 006/010 | Train: 81.112% | Loss: 0.559 Time elapsed: 3.56 min Epoch: 007/010 | Batch 0000/0391 | Cost: 0.5640 Epoch: 007/010 | Batch 0050/0391 | Cost: 0.5809 Epoch: 007/010 | Batch 0100/0391 | Cost: 0.7346 Epoch: 007/010 | Batch 0150/0391 | Cost: 0.6881 Epoch: 007/010 | Batch 0200/0391 | Cost: 0.4644 Epoch: 007/010 | Batch 0250/0391 | Cost: 0.5216 Epoch: 007/010 | Batch 0300/0391 | Cost: 0.6380 Epoch: 007/010 | Batch 0350/0391 | Cost: 0.5521 Epoch: 007/010 | Train: 84.438% | Loss: 0.460 Time elapsed: 4.15 min Epoch: 008/010 | Batch 0000/0391 | Cost: 0.4843 Epoch: 008/010 | Batch 0050/0391 | Cost: 0.5226 Epoch: 008/010 | Batch 0100/0391 | Cost: 0.3682 Epoch: 008/010 | Batch 0150/0391 | Cost: 0.4941 Epoch: 008/010 | Batch 0200/0391 | Cost: 0.5376 Epoch: 008/010 | Batch 0250/0391 | Cost: 0.4707 Epoch: 008/010 | Batch 0300/0391 | Cost: 0.5882 Epoch: 008/010 | Batch 0350/0391 | Cost: 0.5355 Epoch: 008/010 | Train: 85.186% | Loss: 0.442 Time elapsed: 4.76 min Epoch: 009/010 | Batch 0000/0391 | Cost: 0.4174 Epoch: 009/010 | Batch 0050/0391 | Cost: 0.4007 Epoch: 009/010 | Batch 0100/0391 | Cost: 0.4133 Epoch: 009/010 | Batch 0150/0391 | Cost: 0.6204 Epoch: 009/010 | Batch 0200/0391 | Cost: 0.3827 Epoch: 009/010 | Batch 0250/0391 | Cost: 0.4509 Epoch: 009/010 | Batch 0300/0391 | Cost: 0.4461 Epoch: 009/010 | Batch 0350/0391 | Cost: 0.4692 Epoch: 009/010 | Train: 89.038% | Loss: 0.328 Time elapsed: 5.36 min Epoch: 010/010 | Batch 0000/0391 | Cost: 0.2964 Epoch: 010/010 | Batch 0050/0391 | Cost: 0.2739 Epoch: 010/010 | Batch 0100/0391 | Cost: 0.3803 Epoch: 010/010 | Batch 0150/0391 | Cost: 0.3981 Epoch: 010/010 | Batch 0200/0391 | Cost: 0.3877 Epoch: 010/010 | Batch 0250/0391 | Cost: 0.4213 Epoch: 010/010 | Batch 0300/0391 | Cost: 0.5088 Epoch: 010/010 | Batch 0350/0391 | Cost: 0.4598 Epoch: 010/010 | Train: 89.668% | Loss: 0.321 Time elapsed: 5.96 min Total Training Time: 5.96 min
with torch.set_grad_enabled(False): # save memory during inference
print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))
Test accuracy: 76.31%
%watermark -iv
torchvision 0.4.1a0+d94043a numpy 1.16.4 torch 1.3.0