Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.
%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -p torch
Sebastian Raschka CPython 3.6.8 IPython 7.2.0 torch 1.0.1.post2
Simple convolutional neural network that uses stride=2 every 2nd convolutional layer, instead of max pooling, to reduce the feature maps. Loosely based on
import time
import numpy as np
import torch
import torch.nn.functional as F
from torchvision import datasets
from torchvision import transforms
from torch.utils.data import DataLoader
if torch.cuda.is_available():
torch.backends.cudnn.deterministic = True
##########################
### SETTINGS
##########################
# Device
device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")
# Hyperparameters
random_seed = 1
learning_rate = 0.001
num_epochs = 15
batch_size = 256
# Architecture
num_classes = 10
##########################
### MNIST DATASET
##########################
# Note transforms.ToTensor() scales input images
# to 0-1 range
train_dataset = datasets.MNIST(root='data',
train=True,
transform=transforms.ToTensor(),
download=True)
test_dataset = datasets.MNIST(root='data',
train=False,
transform=transforms.ToTensor())
train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
test_loader = DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)
# Checking the dataset
for images, labels in train_loader:
print('Image batch dimensions:', images.shape)
print('Image label dimensions:', labels.shape)
break
Image batch dimensions: torch.Size([256, 1, 28, 28]) Image label dimensions: torch.Size([256])
##########################
### MODEL
##########################
class ConvNet(torch.nn.Module):
def __init__(self, num_classes):
super(ConvNet, self).__init__()
self.num_classes = num_classes
# calculate same padding:
# (w - k + 2*p)/s + 1 = o
# => p = (s(o-1) - w + k)/2
# 28x28x1 => 28x28x4
self.conv_1 = torch.nn.Conv2d(in_channels=1,
out_channels=4,
kernel_size=(3, 3),
stride=(1, 1),
padding=1) # (1(28-1) - 28 + 3) / 2 = 1
# 28x28x4 => 14x14x4
self.conv_2 = torch.nn.Conv2d(in_channels=4,
out_channels=4,
kernel_size=(3, 3),
stride=(2, 2),
padding=1)
# 14x14x4 => 14x14x8
self.conv_3 = torch.nn.Conv2d(in_channels=4,
out_channels=8,
kernel_size=(3, 3),
stride=(1, 1),
padding=1) # (1(14-1) - 14 + 3) / 2 = 1
# 14x14x8 => 7x7x8
self.conv_4 = torch.nn.Conv2d(in_channels=8,
out_channels=8,
kernel_size=(3, 3),
stride=(2, 2),
padding=1)
# 7x7x8 => 7x7x16
self.conv_5 = torch.nn.Conv2d(in_channels=8,
out_channels=16,
kernel_size=(3, 3),
stride=(1, 1),
padding=1) # (1(7-1) - 7 + 3) / 2 = 1
# 7x7x16 => 4x4x16
self.conv_6 = torch.nn.Conv2d(in_channels=16,
out_channels=16,
kernel_size=(3, 3),
stride=(2, 2),
padding=1)
# 4x4x16 => 4x4xnum_classes
self.conv_7 = torch.nn.Conv2d(in_channels=16,
out_channels=self.num_classes,
kernel_size=(3, 3),
stride=(1, 1),
padding=1) # (1(7-1) - 7 + 3) / 2 = 1
def forward(self, x):
out = self.conv_1(x)
out = F.relu(out)
out = self.conv_2(out)
out = F.relu(out)
out = self.conv_3(out)
out = F.relu(out)
out = self.conv_4(out)
out = F.relu(out)
out = self.conv_5(out)
out = F.relu(out)
out = self.conv_6(out)
out = F.relu(out)
out = self.conv_7(out)
out = F.relu(out)
logits = F.adaptive_avg_pool2d(out, 1)
# drop width
logits.squeeze_(-1)
# drop height
logits.squeeze_(-1)
probas = torch.softmax(logits, dim=1)
return logits, probas
torch.manual_seed(random_seed)
model = ConvNet(num_classes=num_classes)
model = model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
def compute_accuracy(model, data_loader):
correct_pred, num_examples = 0, 0
for features, targets in data_loader:
features = features.to(device)
targets = targets.to(device)
logits, probas = model(features)
_, predicted_labels = torch.max(probas, 1)
num_examples += targets.size(0)
correct_pred += (predicted_labels == targets).sum()
return correct_pred.float()/num_examples * 100
start_time = time.time()
for epoch in range(num_epochs):
model = model.train()
for batch_idx, (features, targets) in enumerate(train_loader):
features = features.to(device)
targets = targets.to(device)
### FORWARD AND BACK PROP
logits, probas = model(features)
cost = F.cross_entropy(logits, targets)
optimizer.zero_grad()
cost.backward()
### UPDATE MODEL PARAMETERS
optimizer.step()
### LOGGING
if not batch_idx % 50:
print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f'
%(epoch+1, num_epochs, batch_idx,
len(train_loader), cost))
model = model.eval()
print('Epoch: %03d/%03d training accuracy: %.2f%%' % (
epoch+1, num_epochs,
compute_accuracy(model, train_loader)))
print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))
print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))
Epoch: 001/015 | Batch 000/235 | Cost: 2.3051 Epoch: 001/015 | Batch 050/235 | Cost: 2.2926 Epoch: 001/015 | Batch 100/235 | Cost: 2.0812 Epoch: 001/015 | Batch 150/235 | Cost: 1.4435 Epoch: 001/015 | Batch 200/235 | Cost: 0.9232 Epoch: 001/015 training accuracy: 76.06% Time elapsed: 0.23 min Epoch: 002/015 | Batch 000/235 | Cost: 0.7001 Epoch: 002/015 | Batch 050/235 | Cost: 0.5710 Epoch: 002/015 | Batch 100/235 | Cost: 0.5925 Epoch: 002/015 | Batch 150/235 | Cost: 0.4022 Epoch: 002/015 | Batch 200/235 | Cost: 0.4663 Epoch: 002/015 training accuracy: 85.68% Time elapsed: 0.45 min Epoch: 003/015 | Batch 000/235 | Cost: 0.4332 Epoch: 003/015 | Batch 050/235 | Cost: 0.3523 Epoch: 003/015 | Batch 100/235 | Cost: 0.4114 Epoch: 003/015 | Batch 150/235 | Cost: 0.4587 Epoch: 003/015 | Batch 200/235 | Cost: 0.4517 Epoch: 003/015 training accuracy: 89.33% Time elapsed: 0.68 min Epoch: 004/015 | Batch 000/235 | Cost: 0.4083 Epoch: 004/015 | Batch 050/235 | Cost: 0.3158 Epoch: 004/015 | Batch 100/235 | Cost: 0.2728 Epoch: 004/015 | Batch 150/235 | Cost: 0.3023 Epoch: 004/015 | Batch 200/235 | Cost: 0.2709 Epoch: 004/015 training accuracy: 90.40% Time elapsed: 0.90 min Epoch: 005/015 | Batch 000/235 | Cost: 0.2514 Epoch: 005/015 | Batch 050/235 | Cost: 0.3704 Epoch: 005/015 | Batch 100/235 | Cost: 0.2972 Epoch: 005/015 | Batch 150/235 | Cost: 0.2335 Epoch: 005/015 | Batch 200/235 | Cost: 0.3242 Epoch: 005/015 training accuracy: 91.36% Time elapsed: 1.13 min Epoch: 006/015 | Batch 000/235 | Cost: 0.3255 Epoch: 006/015 | Batch 050/235 | Cost: 0.2985 Epoch: 006/015 | Batch 100/235 | Cost: 0.3501 Epoch: 006/015 | Batch 150/235 | Cost: 0.2415 Epoch: 006/015 | Batch 200/235 | Cost: 0.1978 Epoch: 006/015 training accuracy: 92.82% Time elapsed: 1.35 min Epoch: 007/015 | Batch 000/235 | Cost: 0.1925 Epoch: 007/015 | Batch 050/235 | Cost: 0.2179 Epoch: 007/015 | Batch 100/235 | Cost: 0.3337 Epoch: 007/015 | Batch 150/235 | Cost: 0.1856 Epoch: 007/015 | Batch 200/235 | Cost: 0.1333 Epoch: 007/015 training accuracy: 93.68% Time elapsed: 1.58 min Epoch: 008/015 | Batch 000/235 | Cost: 0.1776 Epoch: 008/015 | Batch 050/235 | Cost: 0.2973 Epoch: 008/015 | Batch 100/235 | Cost: 0.1685 Epoch: 008/015 | Batch 150/235 | Cost: 0.2062 Epoch: 008/015 | Batch 200/235 | Cost: 0.2165 Epoch: 008/015 training accuracy: 94.42% Time elapsed: 1.80 min Epoch: 009/015 | Batch 000/235 | Cost: 0.2038 Epoch: 009/015 | Batch 050/235 | Cost: 0.1301 Epoch: 009/015 | Batch 100/235 | Cost: 0.1977 Epoch: 009/015 | Batch 150/235 | Cost: 0.2160 Epoch: 009/015 | Batch 200/235 | Cost: 0.1772 Epoch: 009/015 training accuracy: 94.61% Time elapsed: 2.02 min Epoch: 010/015 | Batch 000/235 | Cost: 0.1709 Epoch: 010/015 | Batch 050/235 | Cost: 0.1695 Epoch: 010/015 | Batch 100/235 | Cost: 0.2144 Epoch: 010/015 | Batch 150/235 | Cost: 0.1548 Epoch: 010/015 | Batch 200/235 | Cost: 0.1033 Epoch: 010/015 training accuracy: 94.90% Time elapsed: 2.25 min Epoch: 011/015 | Batch 000/235 | Cost: 0.1651 Epoch: 011/015 | Batch 050/235 | Cost: 0.1899 Epoch: 011/015 | Batch 100/235 | Cost: 0.1727 Epoch: 011/015 | Batch 150/235 | Cost: 0.1216 Epoch: 011/015 | Batch 200/235 | Cost: 0.1859 Epoch: 011/015 training accuracy: 94.82% Time elapsed: 2.47 min Epoch: 012/015 | Batch 000/235 | Cost: 0.2490 Epoch: 012/015 | Batch 050/235 | Cost: 0.1022 Epoch: 012/015 | Batch 100/235 | Cost: 0.0793 Epoch: 012/015 | Batch 150/235 | Cost: 0.2258 Epoch: 012/015 | Batch 200/235 | Cost: 0.1356 Epoch: 012/015 training accuracy: 95.35% Time elapsed: 2.70 min Epoch: 013/015 | Batch 000/235 | Cost: 0.1512 Epoch: 013/015 | Batch 050/235 | Cost: 0.1758 Epoch: 013/015 | Batch 100/235 | Cost: 0.1349 Epoch: 013/015 | Batch 150/235 | Cost: 0.1838 Epoch: 013/015 | Batch 200/235 | Cost: 0.1166 Epoch: 013/015 training accuracy: 95.61% Time elapsed: 2.92 min Epoch: 014/015 | Batch 000/235 | Cost: 0.1210 Epoch: 014/015 | Batch 050/235 | Cost: 0.1511 Epoch: 014/015 | Batch 100/235 | Cost: 0.1331 Epoch: 014/015 | Batch 150/235 | Cost: 0.1058 Epoch: 014/015 | Batch 200/235 | Cost: 0.1340 Epoch: 014/015 training accuracy: 95.53% Time elapsed: 3.15 min Epoch: 015/015 | Batch 000/235 | Cost: 0.2342 Epoch: 015/015 | Batch 050/235 | Cost: 0.1371 Epoch: 015/015 | Batch 100/235 | Cost: 0.0944 Epoch: 015/015 | Batch 150/235 | Cost: 0.1102 Epoch: 015/015 | Batch 200/235 | Cost: 0.1259 Epoch: 015/015 training accuracy: 96.36% Time elapsed: 3.37 min Total Training Time: 3.37 min
print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))
Test accuracy: 96.42%
%watermark -iv
numpy 1.15.4 torch 1.0.1.post2