Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.

In [1]:
%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -p torch
Sebastian Raschka 

CPython 3.6.8
IPython 7.2.0

torch 1.0.0
  • Runs on CPU or GPU (if available)

Model Zoo -- Softmax Regression

Implementation of softmax regression (multinomial logistic regression).

Imports

In [2]:
from torchvision import datasets
from torchvision import transforms
from torch.utils.data import DataLoader
import torch.nn.functional as F
import torch

Settings and Dataset

In [3]:
##########################
### SETTINGS
##########################

# Device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# Hyperparameters
random_seed = 123
learning_rate = 0.1
num_epochs = 10
batch_size = 256

# Architecture
num_features = 784
num_classes = 10


##########################
### MNIST DATASET
##########################

train_dataset = datasets.MNIST(root='data', 
                               train=True, 
                               transform=transforms.ToTensor(),  
                               download=True)

test_dataset = datasets.MNIST(root='data', 
                              train=False, 
                              transform=transforms.ToTensor())


train_loader = DataLoader(dataset=train_dataset, 
                          batch_size=batch_size, 
                          shuffle=True)

test_loader = DataLoader(dataset=test_dataset, 
                         batch_size=batch_size, 
                         shuffle=False)


# Checking the dataset
for images, labels in train_loader:  
    print('Image batch dimensions:', images.shape)
    print('Image label dimensions:', labels.shape)
    break
Image batch dimensions: torch.Size([256, 1, 28, 28])
Image label dimensions: torch.Size([256])
In [4]:
##########################
### MODEL
##########################

class SoftmaxRegression(torch.nn.Module):

    def __init__(self, num_features, num_classes):
        super(SoftmaxRegression, self).__init__()
        self.linear = torch.nn.Linear(num_features, num_classes)
        
        self.linear.weight.detach().zero_()
        self.linear.bias.detach().zero_()
        
    def forward(self, x):
        logits = self.linear(x)
        probas = F.softmax(logits, dim=1)
        return logits, probas

model = SoftmaxRegression(num_features=num_features,
                          num_classes=num_classes)

model.to(device)

##########################
### COST AND OPTIMIZER
##########################

optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  
In [5]:
# Manual seed for deterministic data loader
torch.manual_seed(random_seed)


def compute_accuracy(model, data_loader):
    correct_pred, num_examples = 0, 0
    
    for features, targets in data_loader:
        features = features.view(-1, 28*28).to(device)
        targets = targets.to(device)
        logits, probas = model(features)
        _, predicted_labels = torch.max(probas, 1)
        num_examples += targets.size(0)
        correct_pred += (predicted_labels == targets).sum()
        
    return correct_pred.float() / num_examples * 100
    

for epoch in range(num_epochs):
    for batch_idx, (features, targets) in enumerate(train_loader):
        
        features = features.view(-1, 28*28).to(device)
        targets = targets.to(device)
            
        ### FORWARD AND BACK PROP
        logits, probas = model(features)
        
        # note that the PyTorch implementation of
        # CrossEntropyLoss works with logits, not
        # probabilities
        cost = F.cross_entropy(logits, targets)
        optimizer.zero_grad()
        cost.backward()
        
        ### UPDATE MODEL PARAMETERS
        optimizer.step()
        
        ### LOGGING
        if not batch_idx % 50:
            print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' 
                   %(epoch+1, num_epochs, batch_idx, 
                     len(train_dataset)//batch_size, cost))
            
    with torch.set_grad_enabled(False):
        print('Epoch: %03d/%03d training accuracy: %.2f%%' % (
              epoch+1, num_epochs, 
              compute_accuracy(model, train_loader)))
Epoch: 001/010 | Batch 000/234 | Cost: 2.3026
Epoch: 001/010 | Batch 050/234 | Cost: 0.7941
Epoch: 001/010 | Batch 100/234 | Cost: 0.5651
Epoch: 001/010 | Batch 150/234 | Cost: 0.4603
Epoch: 001/010 | Batch 200/234 | Cost: 0.4822
Epoch: 001/010 training accuracy: 88.04%
Epoch: 002/010 | Batch 000/234 | Cost: 0.4105
Epoch: 002/010 | Batch 050/234 | Cost: 0.4415
Epoch: 002/010 | Batch 100/234 | Cost: 0.4367
Epoch: 002/010 | Batch 150/234 | Cost: 0.4289
Epoch: 002/010 | Batch 200/234 | Cost: 0.3926
Epoch: 002/010 training accuracy: 89.37%
Epoch: 003/010 | Batch 000/234 | Cost: 0.4112
Epoch: 003/010 | Batch 050/234 | Cost: 0.3579
Epoch: 003/010 | Batch 100/234 | Cost: 0.3013
Epoch: 003/010 | Batch 150/234 | Cost: 0.3258
Epoch: 003/010 | Batch 200/234 | Cost: 0.4254
Epoch: 003/010 training accuracy: 89.98%
Epoch: 004/010 | Batch 000/234 | Cost: 0.3988
Epoch: 004/010 | Batch 050/234 | Cost: 0.3690
Epoch: 004/010 | Batch 100/234 | Cost: 0.3459
Epoch: 004/010 | Batch 150/234 | Cost: 0.4030
Epoch: 004/010 | Batch 200/234 | Cost: 0.3240
Epoch: 004/010 training accuracy: 90.35%
Epoch: 005/010 | Batch 000/234 | Cost: 0.3265
Epoch: 005/010 | Batch 050/234 | Cost: 0.3673
Epoch: 005/010 | Batch 100/234 | Cost: 0.3085
Epoch: 005/010 | Batch 150/234 | Cost: 0.3183
Epoch: 005/010 | Batch 200/234 | Cost: 0.3316
Epoch: 005/010 training accuracy: 90.64%
Epoch: 006/010 | Batch 000/234 | Cost: 0.4518
Epoch: 006/010 | Batch 050/234 | Cost: 0.3863
Epoch: 006/010 | Batch 100/234 | Cost: 0.3620
Epoch: 006/010 | Batch 150/234 | Cost: 0.3733
Epoch: 006/010 | Batch 200/234 | Cost: 0.3289
Epoch: 006/010 training accuracy: 90.86%
Epoch: 007/010 | Batch 000/234 | Cost: 0.3450
Epoch: 007/010 | Batch 050/234 | Cost: 0.2289
Epoch: 007/010 | Batch 100/234 | Cost: 0.3073
Epoch: 007/010 | Batch 150/234 | Cost: 0.2750
Epoch: 007/010 | Batch 200/234 | Cost: 0.3456
Epoch: 007/010 training accuracy: 91.00%
Epoch: 008/010 | Batch 000/234 | Cost: 0.4900
Epoch: 008/010 | Batch 050/234 | Cost: 0.3479
Epoch: 008/010 | Batch 100/234 | Cost: 0.2343
Epoch: 008/010 | Batch 150/234 | Cost: 0.3059
Epoch: 008/010 | Batch 200/234 | Cost: 0.3684
Epoch: 008/010 training accuracy: 91.22%
Epoch: 009/010 | Batch 000/234 | Cost: 0.3762
Epoch: 009/010 | Batch 050/234 | Cost: 0.2976
Epoch: 009/010 | Batch 100/234 | Cost: 0.2690
Epoch: 009/010 | Batch 150/234 | Cost: 0.2610
Epoch: 009/010 | Batch 200/234 | Cost: 0.3140
Epoch: 009/010 training accuracy: 91.34%
Epoch: 010/010 | Batch 000/234 | Cost: 0.2790
Epoch: 010/010 | Batch 050/234 | Cost: 0.3070
Epoch: 010/010 | Batch 100/234 | Cost: 0.3300
Epoch: 010/010 | Batch 150/234 | Cost: 0.2520
Epoch: 010/010 | Batch 200/234 | Cost: 0.3301
Epoch: 010/010 training accuracy: 91.40%
In [6]:
print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))
Test accuracy: 91.77%
In [7]:
%watermark -iv
torch       1.0.0