MNIST-Neural Network-Two Hidden Layers with Variable Optimziers

In [1]:
# coding: utf-8
import sys, os
import numpy as np
import matplotlib.pyplot as plt
import math

sys.path.append(os.pardir)
from deeplink.mnist import *
from deeplink.functions import *
from deeplink.layers import *
from deeplink.util import *
from deeplink.optimizers import *

Multilayer Neural Network Model

In [2]:
import sys, os
from collections import OrderedDict
from scipy import stats
from pandas import DataFrame

class TwoLayerNet:
    def __init__(self, input_size, hidden_layer1_size, hidden_layer2_size, output_size, weight_init_std = 0.01):
        self.params = {}
        self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_layer1_size)
        self.params['b1'] = np.zeros(hidden_layer1_size)
        self.params['W2'] = weight_init_std * np.random.randn(hidden_layer1_size, hidden_layer2_size)
        self.params['b2'] = np.zeros(hidden_layer2_size)
        self.params['W3'] = weight_init_std * np.random.randn(hidden_layer2_size, output_size)
        self.params['b3'] = np.zeros(output_size)

        self.layers = OrderedDict()
        self.layers['Affine1'] = Affine(self.params['W1'], self.params['b1'])
        self.layers['Relu1'] = ReLU()
        self.layers['Affine2'] = Affine(self.params['W2'], self.params['b2'])
        self.layers['Relu2'] = ReLU()
        self.layers['Affine3'] = Affine(self.params['W3'], self.params['b3'])

        self.lastLayer = SoftmaxWithCrossEntropyLoss()
            
    def predict(self, x):
        for layer in self.layers.values():
            x = layer.forward(x)
        
        return x
        
    def loss(self, x, t):
        y = self.predict(x)
        return self.lastLayer.forward(y, t)
    
    def accuracy(self, x, t):
        y = self.predict(x)
        y = np.argmax(y, axis=1)
        if t.ndim != 1 : t = np.argmax(t, axis=1)
        
        accuracy = np.sum(y == t) / float(x.shape[0])
        return accuracy
        
    def backpropagation_gradient(self, x, t):
        # forward
        self.loss(x, t)

        # backward
        din = 1
        din = self.lastLayer.backward(din)
        
        layers = list(self.layers.values())
        layers.reverse()
        for layer in layers:
            din = layer.backward(din)

        grads = {}
        grads['W1'], grads['b1'] = self.layers['Affine1'].dW, self.layers['Affine1'].db
        grads['W2'], grads['b2'] = self.layers['Affine2'].dW, self.layers['Affine2'].db
        grads['W3'], grads['b3'] = self.layers['Affine3'].dW, self.layers['Affine3'].db

        return grads
    
    def learning(self, learning_rate, x_batch, t_batch, optimizer):
        grads = self.backpropagation_gradient(x_batch, t_batch)
        optimizer.update(self.params, grads)

Learning and Validation

In [3]:
data = mnist_data("/Users/yhhan/git/aiclass/0.Professor/data/MNIST_data/.")
(img_train, label_train), (img_validation, label_validation), (img_test, label_test) = data.load_mnist(flatten=True, normalize=True, one_hot_label=True)

optimizers = {}
optimizers['SGD'] = SGD()
optimizers['Momentum'] = Momentum()
optimizers['Nesterov'] = Nesterov()
optimizers['AdaGrad'] = AdaGrad()
optimizers['RMSprop'] = RMSprop()
optimizers['Adam'] = Adam()

num_epochs = 50
train_size = img_train.shape[0]
batch_size = 1000
learning_rate = 0.1

networks = {}
train_errors = {}
validation_errors = {}
test_accuracy_values = {}
max_test_accuracy_epoch = {}
max_test_accuracy_value = {}

for key in optimizers.keys():
    networks[key] = TwoLayerNet(input_size=784, hidden_layer1_size=128, hidden_layer2_size=128, output_size=10)
    train_errors[key] = [] 
    validation_errors[key] = []
    test_accuracy_values[key] = []
    max_test_accuracy_epoch[key] = 0
    max_test_accuracy_value[key] = 0.0

num_batch = math.ceil(train_size / batch_size)
epoch_list = []

for i in range(num_epochs):
    epoch_list.append(i)
    for key in optimizers.keys():
        for k in range(num_batch):
            x_batch = img_train[k * batch_size : k * batch_size + batch_size]
            t_batch = label_train[k * batch_size : k * batch_size + batch_size]
            networks[key].learning(learning_rate, x_batch, t_batch, optimizers[key])

        train_loss = networks[key].loss(x_batch, t_batch)
        train_errors[key].append(train_loss)

        validation_loss = networks[key].loss(img_validation, label_validation)
        validation_errors[key].append(validation_loss)    

        test_accuracy = networks[key].accuracy(img_test, label_test)
        test_accuracy_values[key].append(test_accuracy)
        if test_accuracy > max_test_accuracy_value[key]:
            max_test_accuracy_epoch[key] = i            
            max_test_accuracy_value[key] = test_accuracy
        print("{0:8s}-Epoch:{1:3d}, Train Err.:{2:7.5f}, Validation Err.:{3:7.5f}, Test Accuracy:{4:7.5f}, Max Test Accuracy:{5:7.5f}".format(
            key,
            i,
            train_loss,
            validation_loss,
            test_accuracy,
            max_test_accuracy_value[key]
        ))
    print()
SGD     -Epoch:  0, Train Err.:2.30247, Validation Err.:2.30244, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  0, Train Err.:2.30181, Validation Err.:2.30151, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Nesterov-Epoch:  0, Train Err.:2.30159, Validation Err.:2.30133, Test Accuracy:0.11350, Max Test Accuracy:0.11350
AdaGrad -Epoch:  0, Train Err.:0.42933, Validation Err.:0.37621, Test Accuracy:0.87570, Max Test Accuracy:0.87570
RMSprop -Epoch:  0, Train Err.:0.31946, Validation Err.:0.29008, Test Accuracy:0.89900, Max Test Accuracy:0.89900
Adam    -Epoch:  0, Train Err.:0.59923, Validation Err.:0.53937, Test Accuracy:0.81020, Max Test Accuracy:0.81020

SGD     -Epoch:  1, Train Err.:2.30234, Validation Err.:2.30228, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  1, Train Err.:2.30104, Validation Err.:2.30057, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Nesterov-Epoch:  1, Train Err.:2.30056, Validation Err.:2.30016, Test Accuracy:0.11350, Max Test Accuracy:0.11350
AdaGrad -Epoch:  1, Train Err.:0.32972, Validation Err.:0.27351, Test Accuracy:0.90430, Max Test Accuracy:0.90430
RMSprop -Epoch:  1, Train Err.:0.24382, Validation Err.:0.22124, Test Accuracy:0.92090, Max Test Accuracy:0.92090
Adam    -Epoch:  1, Train Err.:0.38069, Validation Err.:0.31509, Test Accuracy:0.88850, Max Test Accuracy:0.88850

SGD     -Epoch:  2, Train Err.:2.30223, Validation Err.:2.30213, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  2, Train Err.:2.29943, Validation Err.:2.29885, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Nesterov-Epoch:  2, Train Err.:2.29815, Validation Err.:2.29765, Test Accuracy:0.11350, Max Test Accuracy:0.11350
AdaGrad -Epoch:  2, Train Err.:0.28053, Validation Err.:0.23047, Test Accuracy:0.91800, Max Test Accuracy:0.91800
RMSprop -Epoch:  2, Train Err.:0.16622, Validation Err.:0.16929, Test Accuracy:0.94030, Max Test Accuracy:0.94030
Adam    -Epoch:  2, Train Err.:0.32200, Validation Err.:0.26421, Test Accuracy:0.91080, Max Test Accuracy:0.91080

SGD     -Epoch:  3, Train Err.:2.30212, Validation Err.:2.30200, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  3, Train Err.:2.29418, Validation Err.:2.29345, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Nesterov-Epoch:  3, Train Err.:2.28885, Validation Err.:2.28833, Test Accuracy:0.14390, Max Test Accuracy:0.14390
AdaGrad -Epoch:  3, Train Err.:0.24584, Validation Err.:0.20466, Test Accuracy:0.92790, Max Test Accuracy:0.92790
RMSprop -Epoch:  3, Train Err.:0.23688, Validation Err.:0.22960, Test Accuracy:0.91110, Max Test Accuracy:0.94030
Adam    -Epoch:  3, Train Err.:0.28410, Validation Err.:0.23280, Test Accuracy:0.91940, Max Test Accuracy:0.91940

SGD     -Epoch:  4, Train Err.:2.30201, Validation Err.:2.30187, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  4, Train Err.:2.26443, Validation Err.:2.26338, Test Accuracy:0.21050, Max Test Accuracy:0.21050
Nesterov-Epoch:  4, Train Err.:2.21612, Validation Err.:2.21657, Test Accuracy:0.22940, Max Test Accuracy:0.22940
AdaGrad -Epoch:  4, Train Err.:0.21909, Validation Err.:0.18696, Test Accuracy:0.93260, Max Test Accuracy:0.93260
RMSprop -Epoch:  4, Train Err.:0.09962, Validation Err.:0.15052, Test Accuracy:0.94970, Max Test Accuracy:0.94970
Adam    -Epoch:  4, Train Err.:0.25371, Validation Err.:0.20968, Test Accuracy:0.92550, Max Test Accuracy:0.92550

SGD     -Epoch:  5, Train Err.:2.30192, Validation Err.:2.30175, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  5, Train Err.:2.05104, Validation Err.:2.04930, Test Accuracy:0.28610, Max Test Accuracy:0.28610
Nesterov-Epoch:  5, Train Err.:1.88057, Validation Err.:1.88027, Test Accuracy:0.37970, Max Test Accuracy:0.37970
AdaGrad -Epoch:  5, Train Err.:0.19800, Validation Err.:0.17408, Test Accuracy:0.93650, Max Test Accuracy:0.93650
RMSprop -Epoch:  5, Train Err.:0.10011, Validation Err.:0.14826, Test Accuracy:0.94520, Max Test Accuracy:0.94970
Adam    -Epoch:  5, Train Err.:0.22606, Validation Err.:0.19102, Test Accuracy:0.93270, Max Test Accuracy:0.93270

SGD     -Epoch:  6, Train Err.:2.30183, Validation Err.:2.30163, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  6, Train Err.:1.39538, Validation Err.:1.36707, Test Accuracy:0.57280, Max Test Accuracy:0.57280
Nesterov-Epoch:  6, Train Err.:1.04205, Validation Err.:1.00803, Test Accuracy:0.68280, Max Test Accuracy:0.68280
AdaGrad -Epoch:  6, Train Err.:0.18097, Validation Err.:0.16429, Test Accuracy:0.93940, Max Test Accuracy:0.93940
RMSprop -Epoch:  6, Train Err.:0.08246, Validation Err.:0.13258, Test Accuracy:0.95160, Max Test Accuracy:0.95160
Adam    -Epoch:  6, Train Err.:0.19867, Validation Err.:0.17376, Test Accuracy:0.93830, Max Test Accuracy:0.93830

SGD     -Epoch:  7, Train Err.:2.30174, Validation Err.:2.30153, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  7, Train Err.:0.86912, Validation Err.:0.81786, Test Accuracy:0.71560, Max Test Accuracy:0.71560
Nesterov-Epoch:  7, Train Err.:0.71625, Validation Err.:0.65079, Test Accuracy:0.77400, Max Test Accuracy:0.77400
AdaGrad -Epoch:  7, Train Err.:0.16689, Validation Err.:0.15640, Test Accuracy:0.94150, Max Test Accuracy:0.94150
RMSprop -Epoch:  7, Train Err.:0.07401, Validation Err.:0.13384, Test Accuracy:0.95320, Max Test Accuracy:0.95320
Adam    -Epoch:  7, Train Err.:0.17460, Validation Err.:0.15896, Test Accuracy:0.94170, Max Test Accuracy:0.94170

SGD     -Epoch:  8, Train Err.:2.30166, Validation Err.:2.30142, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  8, Train Err.:0.68047, Validation Err.:0.61326, Test Accuracy:0.78600, Max Test Accuracy:0.78600
Nesterov-Epoch:  8, Train Err.:0.62916, Validation Err.:0.55671, Test Accuracy:0.80720, Max Test Accuracy:0.80720
AdaGrad -Epoch:  8, Train Err.:0.15531, Validation Err.:0.14993, Test Accuracy:0.94400, Max Test Accuracy:0.94400
RMSprop -Epoch:  8, Train Err.:0.06335, Validation Err.:0.12631, Test Accuracy:0.95580, Max Test Accuracy:0.95580
Adam    -Epoch:  8, Train Err.:0.15562, Validation Err.:0.14709, Test Accuracy:0.94590, Max Test Accuracy:0.94590

SGD     -Epoch:  9, Train Err.:2.30158, Validation Err.:2.30132, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch:  9, Train Err.:0.60429, Validation Err.:0.53441, Test Accuracy:0.81610, Max Test Accuracy:0.81610
Nesterov-Epoch:  9, Train Err.:0.58002, Validation Err.:0.50528, Test Accuracy:0.82710, Max Test Accuracy:0.82710
AdaGrad -Epoch:  9, Train Err.:0.14571, Validation Err.:0.14456, Test Accuracy:0.94580, Max Test Accuracy:0.94580
RMSprop -Epoch:  9, Train Err.:0.08532, Validation Err.:0.13603, Test Accuracy:0.95320, Max Test Accuracy:0.95580
Adam    -Epoch:  9, Train Err.:0.13979, Validation Err.:0.13698, Test Accuracy:0.94910, Max Test Accuracy:0.94910

SGD     -Epoch: 10, Train Err.:2.30149, Validation Err.:2.30122, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 10, Train Err.:0.55084, Validation Err.:0.47956, Test Accuracy:0.83440, Max Test Accuracy:0.83440
Nesterov-Epoch: 10, Train Err.:0.53941, Validation Err.:0.46344, Test Accuracy:0.83970, Max Test Accuracy:0.83970
AdaGrad -Epoch: 10, Train Err.:0.13767, Validation Err.:0.13995, Test Accuracy:0.94750, Max Test Accuracy:0.94750
RMSprop -Epoch: 10, Train Err.:0.06467, Validation Err.:0.12934, Test Accuracy:0.95700, Max Test Accuracy:0.95700
Adam    -Epoch: 10, Train Err.:0.12606, Validation Err.:0.12829, Test Accuracy:0.95450, Max Test Accuracy:0.95450

SGD     -Epoch: 11, Train Err.:2.30141, Validation Err.:2.30112, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 11, Train Err.:0.50678, Validation Err.:0.43303, Test Accuracy:0.85030, Max Test Accuracy:0.85030
Nesterov-Epoch: 11, Train Err.:0.50059, Validation Err.:0.42296, Test Accuracy:0.85270, Max Test Accuracy:0.85270
AdaGrad -Epoch: 11, Train Err.:0.13069, Validation Err.:0.13588, Test Accuracy:0.94910, Max Test Accuracy:0.94910
RMSprop -Epoch: 11, Train Err.:0.07355, Validation Err.:0.14455, Test Accuracy:0.95300, Max Test Accuracy:0.95700
Adam    -Epoch: 11, Train Err.:0.11478, Validation Err.:0.12094, Test Accuracy:0.95760, Max Test Accuracy:0.95760

SGD     -Epoch: 12, Train Err.:2.30132, Validation Err.:2.30101, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 12, Train Err.:0.47396, Validation Err.:0.39680, Test Accuracy:0.86210, Max Test Accuracy:0.86210
Nesterov-Epoch: 12, Train Err.:0.46651, Validation Err.:0.38704, Test Accuracy:0.86430, Max Test Accuracy:0.86430
AdaGrad -Epoch: 12, Train Err.:0.12445, Validation Err.:0.13215, Test Accuracy:0.95060, Max Test Accuracy:0.95060
RMSprop -Epoch: 12, Train Err.:0.05375, Validation Err.:0.13627, Test Accuracy:0.95540, Max Test Accuracy:0.95700
Adam    -Epoch: 12, Train Err.:0.10467, Validation Err.:0.11469, Test Accuracy:0.96040, Max Test Accuracy:0.96040

SGD     -Epoch: 13, Train Err.:2.30123, Validation Err.:2.30090, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 13, Train Err.:0.44863, Validation Err.:0.36989, Test Accuracy:0.87000, Max Test Accuracy:0.87000
Nesterov-Epoch: 13, Train Err.:0.43954, Validation Err.:0.35940, Test Accuracy:0.87380, Max Test Accuracy:0.87380
AdaGrad -Epoch: 13, Train Err.:0.11853, Validation Err.:0.12905, Test Accuracy:0.95210, Max Test Accuracy:0.95210
RMSprop -Epoch: 13, Train Err.:0.06131, Validation Err.:0.13693, Test Accuracy:0.95860, Max Test Accuracy:0.95860
Adam    -Epoch: 13, Train Err.:0.09671, Validation Err.:0.10963, Test Accuracy:0.96190, Max Test Accuracy:0.96190

SGD     -Epoch: 14, Train Err.:2.30114, Validation Err.:2.30079, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 14, Train Err.:0.42726, Validation Err.:0.34864, Test Accuracy:0.87830, Max Test Accuracy:0.87830
Nesterov-Epoch: 14, Train Err.:0.41657, Validation Err.:0.33721, Test Accuracy:0.88060, Max Test Accuracy:0.88060
AdaGrad -Epoch: 14, Train Err.:0.11360, Validation Err.:0.12599, Test Accuracy:0.95370, Max Test Accuracy:0.95370
RMSprop -Epoch: 14, Train Err.:0.05116, Validation Err.:0.13572, Test Accuracy:0.95810, Max Test Accuracy:0.95860
Adam    -Epoch: 14, Train Err.:0.08908, Validation Err.:0.10528, Test Accuracy:0.96390, Max Test Accuracy:0.96390

SGD     -Epoch: 15, Train Err.:2.30104, Validation Err.:2.30068, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 15, Train Err.:0.40835, Validation Err.:0.33086, Test Accuracy:0.88300, Max Test Accuracy:0.88300
Nesterov-Epoch: 15, Train Err.:0.39500, Validation Err.:0.31848, Test Accuracy:0.88630, Max Test Accuracy:0.88630
AdaGrad -Epoch: 15, Train Err.:0.10891, Validation Err.:0.12331, Test Accuracy:0.95540, Max Test Accuracy:0.95540
RMSprop -Epoch: 15, Train Err.:0.04457, Validation Err.:0.12839, Test Accuracy:0.96020, Max Test Accuracy:0.96020
Adam    -Epoch: 15, Train Err.:0.08205, Validation Err.:0.10177, Test Accuracy:0.96540, Max Test Accuracy:0.96540

SGD     -Epoch: 16, Train Err.:2.30093, Validation Err.:2.30055, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 16, Train Err.:0.39084, Validation Err.:0.31534, Test Accuracy:0.88820, Max Test Accuracy:0.88820
Nesterov-Epoch: 16, Train Err.:0.37549, Validation Err.:0.30217, Test Accuracy:0.89150, Max Test Accuracy:0.89150
AdaGrad -Epoch: 16, Train Err.:0.10476, Validation Err.:0.12082, Test Accuracy:0.95660, Max Test Accuracy:0.95660
RMSprop -Epoch: 16, Train Err.:0.05032, Validation Err.:0.13215, Test Accuracy:0.95810, Max Test Accuracy:0.96020
Adam    -Epoch: 16, Train Err.:0.07562, Validation Err.:0.09847, Test Accuracy:0.96620, Max Test Accuracy:0.96620

SGD     -Epoch: 17, Train Err.:2.30081, Validation Err.:2.30042, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 17, Train Err.:0.37443, Validation Err.:0.30125, Test Accuracy:0.89250, Max Test Accuracy:0.89250
Nesterov-Epoch: 17, Train Err.:0.35779, Validation Err.:0.28763, Test Accuracy:0.89800, Max Test Accuracy:0.89800
AdaGrad -Epoch: 17, Train Err.:0.10106, Validation Err.:0.11865, Test Accuracy:0.95800, Max Test Accuracy:0.95800
RMSprop -Epoch: 17, Train Err.:0.04925, Validation Err.:0.15284, Test Accuracy:0.95780, Max Test Accuracy:0.96020
Adam    -Epoch: 17, Train Err.:0.06987, Validation Err.:0.09597, Test Accuracy:0.96810, Max Test Accuracy:0.96810

SGD     -Epoch: 18, Train Err.:2.30068, Validation Err.:2.30028, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 18, Train Err.:0.35884, Validation Err.:0.28849, Test Accuracy:0.89620, Max Test Accuracy:0.89620
Nesterov-Epoch: 18, Train Err.:0.34170, Validation Err.:0.27473, Test Accuracy:0.90220, Max Test Accuracy:0.90220
AdaGrad -Epoch: 18, Train Err.:0.09753, Validation Err.:0.11673, Test Accuracy:0.95840, Max Test Accuracy:0.95840
RMSprop -Epoch: 18, Train Err.:0.03862, Validation Err.:0.14301, Test Accuracy:0.95870, Max Test Accuracy:0.96020
Adam    -Epoch: 18, Train Err.:0.06533, Validation Err.:0.09376, Test Accuracy:0.96930, Max Test Accuracy:0.96930

SGD     -Epoch: 19, Train Err.:2.30055, Validation Err.:2.30013, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 19, Train Err.:0.34389, Validation Err.:0.27676, Test Accuracy:0.90160, Max Test Accuracy:0.90160
Nesterov-Epoch: 19, Train Err.:0.32665, Validation Err.:0.26310, Test Accuracy:0.90600, Max Test Accuracy:0.90600
AdaGrad -Epoch: 19, Train Err.:0.09428, Validation Err.:0.11492, Test Accuracy:0.95960, Max Test Accuracy:0.95960
RMSprop -Epoch: 19, Train Err.:0.22136, Validation Err.:0.26585, Test Accuracy:0.92940, Max Test Accuracy:0.96020
Adam    -Epoch: 19, Train Err.:0.06103, Validation Err.:0.09210, Test Accuracy:0.97000, Max Test Accuracy:0.97000

SGD     -Epoch: 20, Train Err.:2.30040, Validation Err.:2.29996, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 20, Train Err.:0.32962, Validation Err.:0.26594, Test Accuracy:0.90620, Max Test Accuracy:0.90620
Nesterov-Epoch: 20, Train Err.:0.31265, Validation Err.:0.25262, Test Accuracy:0.91080, Max Test Accuracy:0.91080
AdaGrad -Epoch: 20, Train Err.:0.09126, Validation Err.:0.11333, Test Accuracy:0.96050, Max Test Accuracy:0.96050
RMSprop -Epoch: 20, Train Err.:0.03511, Validation Err.:0.14248, Test Accuracy:0.95960, Max Test Accuracy:0.96020
Adam    -Epoch: 20, Train Err.:0.05714, Validation Err.:0.09074, Test Accuracy:0.97060, Max Test Accuracy:0.97060

SGD     -Epoch: 21, Train Err.:2.30023, Validation Err.:2.29978, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 21, Train Err.:0.31613, Validation Err.:0.25574, Test Accuracy:0.90910, Max Test Accuracy:0.90910
Nesterov-Epoch: 21, Train Err.:0.29968, Validation Err.:0.24292, Test Accuracy:0.91420, Max Test Accuracy:0.91420
AdaGrad -Epoch: 21, Train Err.:0.08849, Validation Err.:0.11183, Test Accuracy:0.96070, Max Test Accuracy:0.96070
RMSprop -Epoch: 21, Train Err.:0.03603, Validation Err.:0.15121, Test Accuracy:0.95860, Max Test Accuracy:0.96020
Adam    -Epoch: 21, Train Err.:0.05360, Validation Err.:0.08980, Test Accuracy:0.97070, Max Test Accuracy:0.97070

SGD     -Epoch: 22, Train Err.:2.30005, Validation Err.:2.29959, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 22, Train Err.:0.30322, Validation Err.:0.24610, Test Accuracy:0.91110, Max Test Accuracy:0.91110
Nesterov-Epoch: 22, Train Err.:0.28736, Validation Err.:0.23391, Test Accuracy:0.91720, Max Test Accuracy:0.91720
AdaGrad -Epoch: 22, Train Err.:0.08588, Validation Err.:0.11042, Test Accuracy:0.96120, Max Test Accuracy:0.96120
RMSprop -Epoch: 22, Train Err.:0.05252, Validation Err.:0.14305, Test Accuracy:0.95840, Max Test Accuracy:0.96020
Adam    -Epoch: 22, Train Err.:0.05050, Validation Err.:0.08917, Test Accuracy:0.97150, Max Test Accuracy:0.97150

SGD     -Epoch: 23, Train Err.:2.29984, Validation Err.:2.29937, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 23, Train Err.:0.29050, Validation Err.:0.23703, Test Accuracy:0.91330, Max Test Accuracy:0.91330
Nesterov-Epoch: 23, Train Err.:0.27556, Validation Err.:0.22551, Test Accuracy:0.92060, Max Test Accuracy:0.92060
AdaGrad -Epoch: 23, Train Err.:0.08344, Validation Err.:0.10912, Test Accuracy:0.96180, Max Test Accuracy:0.96180
RMSprop -Epoch: 23, Train Err.:0.03476, Validation Err.:0.12857, Test Accuracy:0.96210, Max Test Accuracy:0.96210
Adam    -Epoch: 23, Train Err.:0.04751, Validation Err.:0.08868, Test Accuracy:0.97250, Max Test Accuracy:0.97250

SGD     -Epoch: 24, Train Err.:2.29961, Validation Err.:2.29913, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 24, Train Err.:0.27857, Validation Err.:0.22851, Test Accuracy:0.91650, Max Test Accuracy:0.91650
Nesterov-Epoch: 24, Train Err.:0.26437, Validation Err.:0.21754, Test Accuracy:0.92370, Max Test Accuracy:0.92370
AdaGrad -Epoch: 24, Train Err.:0.08117, Validation Err.:0.10794, Test Accuracy:0.96260, Max Test Accuracy:0.96260
RMSprop -Epoch: 24, Train Err.:0.02699, Validation Err.:0.13965, Test Accuracy:0.96090, Max Test Accuracy:0.96210
Adam    -Epoch: 24, Train Err.:0.04498, Validation Err.:0.08851, Test Accuracy:0.97280, Max Test Accuracy:0.97280

SGD     -Epoch: 25, Train Err.:2.29936, Validation Err.:2.29886, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 25, Train Err.:0.26706, Validation Err.:0.22039, Test Accuracy:0.91950, Max Test Accuracy:0.91950
Nesterov-Epoch: 25, Train Err.:0.25377, Validation Err.:0.20999, Test Accuracy:0.92590, Max Test Accuracy:0.92590
AdaGrad -Epoch: 25, Train Err.:0.07900, Validation Err.:0.10687, Test Accuracy:0.96270, Max Test Accuracy:0.96270
RMSprop -Epoch: 25, Train Err.:0.03537, Validation Err.:0.15894, Test Accuracy:0.95850, Max Test Accuracy:0.96210
Adam    -Epoch: 25, Train Err.:0.04244, Validation Err.:0.08818, Test Accuracy:0.97300, Max Test Accuracy:0.97300

SGD     -Epoch: 26, Train Err.:2.29908, Validation Err.:2.29857, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 26, Train Err.:0.25591, Validation Err.:0.21261, Test Accuracy:0.92300, Max Test Accuracy:0.92300
Nesterov-Epoch: 26, Train Err.:0.24368, Validation Err.:0.20279, Test Accuracy:0.92920, Max Test Accuracy:0.92920
AdaGrad -Epoch: 26, Train Err.:0.07697, Validation Err.:0.10584, Test Accuracy:0.96300, Max Test Accuracy:0.96300
RMSprop -Epoch: 26, Train Err.:0.03275, Validation Err.:0.14621, Test Accuracy:0.96050, Max Test Accuracy:0.96210
Adam    -Epoch: 26, Train Err.:0.03977, Validation Err.:0.08783, Test Accuracy:0.97350, Max Test Accuracy:0.97350

SGD     -Epoch: 27, Train Err.:2.29876, Validation Err.:2.29824, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 27, Train Err.:0.24504, Validation Err.:0.20522, Test Accuracy:0.92530, Max Test Accuracy:0.92530
Nesterov-Epoch: 27, Train Err.:0.23380, Validation Err.:0.19589, Test Accuracy:0.93160, Max Test Accuracy:0.93160
AdaGrad -Epoch: 27, Train Err.:0.07503, Validation Err.:0.10485, Test Accuracy:0.96320, Max Test Accuracy:0.96320
RMSprop -Epoch: 27, Train Err.:0.03175, Validation Err.:0.16259, Test Accuracy:0.95770, Max Test Accuracy:0.96210
Adam    -Epoch: 27, Train Err.:0.03691, Validation Err.:0.08701, Test Accuracy:0.97380, Max Test Accuracy:0.97380

SGD     -Epoch: 28, Train Err.:2.29840, Validation Err.:2.29786, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 28, Train Err.:0.23472, Validation Err.:0.19819, Test Accuracy:0.92750, Max Test Accuracy:0.92750
Nesterov-Epoch: 28, Train Err.:0.22421, Validation Err.:0.18927, Test Accuracy:0.93390, Max Test Accuracy:0.93390
AdaGrad -Epoch: 28, Train Err.:0.07324, Validation Err.:0.10394, Test Accuracy:0.96330, Max Test Accuracy:0.96330
RMSprop -Epoch: 28, Train Err.:0.17835, Validation Err.:0.24013, Test Accuracy:0.93630, Max Test Accuracy:0.96210
Adam    -Epoch: 28, Train Err.:0.03338, Validation Err.:0.08594, Test Accuracy:0.97440, Max Test Accuracy:0.97440

SGD     -Epoch: 29, Train Err.:2.29800, Validation Err.:2.29744, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 29, Train Err.:0.22499, Validation Err.:0.19161, Test Accuracy:0.93020, Max Test Accuracy:0.93020
Nesterov-Epoch: 29, Train Err.:0.21478, Validation Err.:0.18299, Test Accuracy:0.93610, Max Test Accuracy:0.93610
AdaGrad -Epoch: 29, Train Err.:0.07145, Validation Err.:0.10316, Test Accuracy:0.96350, Max Test Accuracy:0.96350
RMSprop -Epoch: 29, Train Err.:0.02845, Validation Err.:0.16085, Test Accuracy:0.95980, Max Test Accuracy:0.96210
Adam    -Epoch: 29, Train Err.:0.03040, Validation Err.:0.08523, Test Accuracy:0.97440, Max Test Accuracy:0.97440

SGD     -Epoch: 30, Train Err.:2.29754, Validation Err.:2.29697, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 30, Train Err.:0.21594, Validation Err.:0.18552, Test Accuracy:0.93310, Max Test Accuracy:0.93310
Nesterov-Epoch: 30, Train Err.:0.20573, Validation Err.:0.17704, Test Accuracy:0.93720, Max Test Accuracy:0.93720
AdaGrad -Epoch: 30, Train Err.:0.06974, Validation Err.:0.10239, Test Accuracy:0.96380, Max Test Accuracy:0.96380
RMSprop -Epoch: 30, Train Err.:0.02792, Validation Err.:0.15828, Test Accuracy:0.96130, Max Test Accuracy:0.96210
Adam    -Epoch: 30, Train Err.:0.02746, Validation Err.:0.08450, Test Accuracy:0.97490, Max Test Accuracy:0.97490

SGD     -Epoch: 31, Train Err.:2.29701, Validation Err.:2.29642, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 31, Train Err.:0.20734, Validation Err.:0.17987, Test Accuracy:0.93520, Max Test Accuracy:0.93520
Nesterov-Epoch: 31, Train Err.:0.19709, Validation Err.:0.17147, Test Accuracy:0.93880, Max Test Accuracy:0.93880
AdaGrad -Epoch: 31, Train Err.:0.06815, Validation Err.:0.10168, Test Accuracy:0.96440, Max Test Accuracy:0.96440
RMSprop -Epoch: 31, Train Err.:0.16350, Validation Err.:0.25763, Test Accuracy:0.93890, Max Test Accuracy:0.96210
Adam    -Epoch: 31, Train Err.:0.02490, Validation Err.:0.08450, Test Accuracy:0.97530, Max Test Accuracy:0.97530

SGD     -Epoch: 32, Train Err.:2.29640, Validation Err.:2.29580, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 32, Train Err.:0.19935, Validation Err.:0.17455, Test Accuracy:0.93810, Max Test Accuracy:0.93810
Nesterov-Epoch: 32, Train Err.:0.18893, Validation Err.:0.16627, Test Accuracy:0.94110, Max Test Accuracy:0.94110
AdaGrad -Epoch: 32, Train Err.:0.06662, Validation Err.:0.10104, Test Accuracy:0.96500, Max Test Accuracy:0.96500
RMSprop -Epoch: 32, Train Err.:0.02141, Validation Err.:0.15125, Test Accuracy:0.96140, Max Test Accuracy:0.96210
Adam    -Epoch: 32, Train Err.:0.02262, Validation Err.:0.08507, Test Accuracy:0.97580, Max Test Accuracy:0.97580

SGD     -Epoch: 33, Train Err.:2.29570, Validation Err.:2.29508, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 33, Train Err.:0.19193, Validation Err.:0.16955, Test Accuracy:0.93960, Max Test Accuracy:0.93960
Nesterov-Epoch: 33, Train Err.:0.18109, Validation Err.:0.16139, Test Accuracy:0.94270, Max Test Accuracy:0.94270
AdaGrad -Epoch: 33, Train Err.:0.06514, Validation Err.:0.10040, Test Accuracy:0.96530, Max Test Accuracy:0.96530
RMSprop -Epoch: 33, Train Err.:0.02137, Validation Err.:0.15856, Test Accuracy:0.96120, Max Test Accuracy:0.96210
Adam    -Epoch: 33, Train Err.:0.02097, Validation Err.:0.08539, Test Accuracy:0.97520, Max Test Accuracy:0.97580

SGD     -Epoch: 34, Train Err.:2.29489, Validation Err.:2.29424, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 34, Train Err.:0.18498, Validation Err.:0.16488, Test Accuracy:0.94080, Max Test Accuracy:0.94080
Nesterov-Epoch: 34, Train Err.:0.17368, Validation Err.:0.15684, Test Accuracy:0.94490, Max Test Accuracy:0.94490
AdaGrad -Epoch: 34, Train Err.:0.06373, Validation Err.:0.09985, Test Accuracy:0.96570, Max Test Accuracy:0.96570
RMSprop -Epoch: 34, Train Err.:0.15996, Validation Err.:0.31347, Test Accuracy:0.93310, Max Test Accuracy:0.96210
Adam    -Epoch: 34, Train Err.:0.01955, Validation Err.:0.08661, Test Accuracy:0.97520, Max Test Accuracy:0.97580

SGD     -Epoch: 35, Train Err.:2.29395, Validation Err.:2.29327, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 35, Train Err.:0.17851, Validation Err.:0.16048, Test Accuracy:0.94180, Max Test Accuracy:0.94180
Nesterov-Epoch: 35, Train Err.:0.16684, Validation Err.:0.15265, Test Accuracy:0.94660, Max Test Accuracy:0.94660
AdaGrad -Epoch: 35, Train Err.:0.06240, Validation Err.:0.09927, Test Accuracy:0.96600, Max Test Accuracy:0.96600
RMSprop -Epoch: 35, Train Err.:0.04172, Validation Err.:0.16044, Test Accuracy:0.95570, Max Test Accuracy:0.96210
Adam    -Epoch: 35, Train Err.:0.01830, Validation Err.:0.08720, Test Accuracy:0.97500, Max Test Accuracy:0.97580

SGD     -Epoch: 36, Train Err.:2.29283, Validation Err.:2.29212, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 36, Train Err.:0.17243, Validation Err.:0.15633, Test Accuracy:0.94400, Max Test Accuracy:0.94400
Nesterov-Epoch: 36, Train Err.:0.16029, Validation Err.:0.14876, Test Accuracy:0.94790, Max Test Accuracy:0.94790
AdaGrad -Epoch: 36, Train Err.:0.06108, Validation Err.:0.09881, Test Accuracy:0.96650, Max Test Accuracy:0.96650
RMSprop -Epoch: 36, Train Err.:0.02288, Validation Err.:0.15130, Test Accuracy:0.96100, Max Test Accuracy:0.96210
Adam    -Epoch: 36, Train Err.:0.01720, Validation Err.:0.08735, Test Accuracy:0.97500, Max Test Accuracy:0.97580

SGD     -Epoch: 37, Train Err.:2.29150, Validation Err.:2.29077, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 37, Train Err.:0.16678, Validation Err.:0.15252, Test Accuracy:0.94540, Max Test Accuracy:0.94540
Nesterov-Epoch: 37, Train Err.:0.15404, Validation Err.:0.14512, Test Accuracy:0.94920, Max Test Accuracy:0.94920
AdaGrad -Epoch: 37, Train Err.:0.05982, Validation Err.:0.09829, Test Accuracy:0.96660, Max Test Accuracy:0.96660
RMSprop -Epoch: 37, Train Err.:0.01641, Validation Err.:0.15179, Test Accuracy:0.96230, Max Test Accuracy:0.96230
Adam    -Epoch: 37, Train Err.:0.01607, Validation Err.:0.08739, Test Accuracy:0.97530, Max Test Accuracy:0.97580

SGD     -Epoch: 38, Train Err.:2.28992, Validation Err.:2.28914, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Momentum-Epoch: 38, Train Err.:0.16153, Validation Err.:0.14891, Test Accuracy:0.94610, Max Test Accuracy:0.94610
Nesterov-Epoch: 38, Train Err.:0.14825, Validation Err.:0.14177, Test Accuracy:0.95140, Max Test Accuracy:0.95140
AdaGrad -Epoch: 38, Train Err.:0.05856, Validation Err.:0.09782, Test Accuracy:0.96690, Max Test Accuracy:0.96690
RMSprop -Epoch: 38, Train Err.:0.01397, Validation Err.:0.16313, Test Accuracy:0.96130, Max Test Accuracy:0.96230
Adam    -Epoch: 38, Train Err.:0.01479, Validation Err.:0.08712, Test Accuracy:0.97560, Max Test Accuracy:0.97580

SGD     -Epoch: 39, Train Err.:2.28801, Validation Err.:2.28719, Test Accuracy:0.11360, Max Test Accuracy:0.11360
Momentum-Epoch: 39, Train Err.:0.15648, Validation Err.:0.14556, Test Accuracy:0.94710, Max Test Accuracy:0.94710
Nesterov-Epoch: 39, Train Err.:0.14277, Validation Err.:0.13866, Test Accuracy:0.95230, Max Test Accuracy:0.95230
AdaGrad -Epoch: 39, Train Err.:0.05739, Validation Err.:0.09737, Test Accuracy:0.96690, Max Test Accuracy:0.96690
RMSprop -Epoch: 39, Train Err.:0.02124, Validation Err.:0.15782, Test Accuracy:0.96080, Max Test Accuracy:0.96230
Adam    -Epoch: 39, Train Err.:0.01373, Validation Err.:0.08674, Test Accuracy:0.97550, Max Test Accuracy:0.97580

SGD     -Epoch: 40, Train Err.:2.28569, Validation Err.:2.28480, Test Accuracy:0.11690, Max Test Accuracy:0.11690
Momentum-Epoch: 40, Train Err.:0.15166, Validation Err.:0.14237, Test Accuracy:0.94890, Max Test Accuracy:0.94890
Nesterov-Epoch: 40, Train Err.:0.13763, Validation Err.:0.13579, Test Accuracy:0.95290, Max Test Accuracy:0.95290
AdaGrad -Epoch: 40, Train Err.:0.05628, Validation Err.:0.09692, Test Accuracy:0.96720, Max Test Accuracy:0.96720
RMSprop -Epoch: 40, Train Err.:0.01887, Validation Err.:0.17953, Test Accuracy:0.96080, Max Test Accuracy:0.96230
Adam    -Epoch: 40, Train Err.:0.01423, Validation Err.:0.08902, Test Accuracy:0.97550, Max Test Accuracy:0.97580

SGD     -Epoch: 41, Train Err.:2.28282, Validation Err.:2.28187, Test Accuracy:0.13060, Max Test Accuracy:0.13060
Momentum-Epoch: 41, Train Err.:0.14708, Validation Err.:0.13941, Test Accuracy:0.95020, Max Test Accuracy:0.95020
Nesterov-Epoch: 41, Train Err.:0.13284, Validation Err.:0.13308, Test Accuracy:0.95420, Max Test Accuracy:0.95420
AdaGrad -Epoch: 41, Train Err.:0.05518, Validation Err.:0.09652, Test Accuracy:0.96750, Max Test Accuracy:0.96750
RMSprop -Epoch: 41, Train Err.:0.01823, Validation Err.:0.16061, Test Accuracy:0.95930, Max Test Accuracy:0.96230
Adam    -Epoch: 41, Train Err.:0.01692, Validation Err.:0.09520, Test Accuracy:0.97390, Max Test Accuracy:0.97580

SGD     -Epoch: 42, Train Err.:2.27925, Validation Err.:2.27820, Test Accuracy:0.15500, Max Test Accuracy:0.15500
Momentum-Epoch: 42, Train Err.:0.14273, Validation Err.:0.13656, Test Accuracy:0.95170, Max Test Accuracy:0.95170
Nesterov-Epoch: 42, Train Err.:0.12842, Validation Err.:0.13059, Test Accuracy:0.95520, Max Test Accuracy:0.95520
AdaGrad -Epoch: 42, Train Err.:0.05416, Validation Err.:0.09617, Test Accuracy:0.96780, Max Test Accuracy:0.96780
RMSprop -Epoch: 42, Train Err.:0.01753, Validation Err.:0.16497, Test Accuracy:0.96070, Max Test Accuracy:0.96230
Adam    -Epoch: 42, Train Err.:0.01751, Validation Err.:0.10192, Test Accuracy:0.97230, Max Test Accuracy:0.97580

SGD     -Epoch: 43, Train Err.:2.27473, Validation Err.:2.27357, Test Accuracy:0.19250, Max Test Accuracy:0.19250
Momentum-Epoch: 43, Train Err.:0.13867, Validation Err.:0.13391, Test Accuracy:0.95190, Max Test Accuracy:0.95190
Nesterov-Epoch: 43, Train Err.:0.12420, Validation Err.:0.12831, Test Accuracy:0.95550, Max Test Accuracy:0.95550
AdaGrad -Epoch: 43, Train Err.:0.05315, Validation Err.:0.09582, Test Accuracy:0.96800, Max Test Accuracy:0.96800
RMSprop -Epoch: 43, Train Err.:0.01176, Validation Err.:0.17764, Test Accuracy:0.96170, Max Test Accuracy:0.96230
Adam    -Epoch: 43, Train Err.:0.01311, Validation Err.:0.09795, Test Accuracy:0.97300, Max Test Accuracy:0.97580

SGD     -Epoch: 44, Train Err.:2.26893, Validation Err.:2.26763, Test Accuracy:0.23160, Max Test Accuracy:0.23160
Momentum-Epoch: 44, Train Err.:0.13481, Validation Err.:0.13142, Test Accuracy:0.95260, Max Test Accuracy:0.95260
Nesterov-Epoch: 44, Train Err.:0.12018, Validation Err.:0.12616, Test Accuracy:0.95590, Max Test Accuracy:0.95590
AdaGrad -Epoch: 44, Train Err.:0.05221, Validation Err.:0.09550, Test Accuracy:0.96820, Max Test Accuracy:0.96820
RMSprop -Epoch: 44, Train Err.:0.02055, Validation Err.:0.21524, Test Accuracy:0.96050, Max Test Accuracy:0.96230
Adam    -Epoch: 44, Train Err.:0.01011, Validation Err.:0.09299, Test Accuracy:0.97580, Max Test Accuracy:0.97580

SGD     -Epoch: 45, Train Err.:2.26138, Validation Err.:2.25988, Test Accuracy:0.26570, Max Test Accuracy:0.26570
Momentum-Epoch: 45, Train Err.:0.13131, Validation Err.:0.12913, Test Accuracy:0.95330, Max Test Accuracy:0.95330
Nesterov-Epoch: 45, Train Err.:0.11642, Validation Err.:0.12419, Test Accuracy:0.95610, Max Test Accuracy:0.95610
AdaGrad -Epoch: 45, Train Err.:0.05132, Validation Err.:0.09516, Test Accuracy:0.96850, Max Test Accuracy:0.96850
RMSprop -Epoch: 45, Train Err.:0.15238, Validation Err.:0.32637, Test Accuracy:0.93070, Max Test Accuracy:0.96230
Adam    -Epoch: 45, Train Err.:0.00816, Validation Err.:0.09157, Test Accuracy:0.97590, Max Test Accuracy:0.97590

SGD     -Epoch: 46, Train Err.:2.25138, Validation Err.:2.24962, Test Accuracy:0.29880, Max Test Accuracy:0.29880
Momentum-Epoch: 46, Train Err.:0.12793, Validation Err.:0.12690, Test Accuracy:0.95470, Max Test Accuracy:0.95470
Nesterov-Epoch: 46, Train Err.:0.11282, Validation Err.:0.12229, Test Accuracy:0.95720, Max Test Accuracy:0.95720
AdaGrad -Epoch: 46, Train Err.:0.05040, Validation Err.:0.09490, Test Accuracy:0.96880, Max Test Accuracy:0.96880
RMSprop -Epoch: 46, Train Err.:0.02356, Validation Err.:0.17880, Test Accuracy:0.96000, Max Test Accuracy:0.96230
Adam    -Epoch: 46, Train Err.:0.00721, Validation Err.:0.09468, Test Accuracy:0.97480, Max Test Accuracy:0.97590

SGD     -Epoch: 47, Train Err.:2.23791, Validation Err.:2.23581, Test Accuracy:0.32260, Max Test Accuracy:0.32260
Momentum-Epoch: 47, Train Err.:0.12461, Validation Err.:0.12479, Test Accuracy:0.95550, Max Test Accuracy:0.95550
Nesterov-Epoch: 47, Train Err.:0.10946, Validation Err.:0.12048, Test Accuracy:0.95730, Max Test Accuracy:0.95730
AdaGrad -Epoch: 47, Train Err.:0.04944, Validation Err.:0.09460, Test Accuracy:0.96890, Max Test Accuracy:0.96890
RMSprop -Epoch: 47, Train Err.:0.01843, Validation Err.:0.17483, Test Accuracy:0.95740, Max Test Accuracy:0.96230
Adam    -Epoch: 47, Train Err.:0.00699, Validation Err.:0.09929, Test Accuracy:0.97400, Max Test Accuracy:0.97590

SGD     -Epoch: 48, Train Err.:2.21949, Validation Err.:2.21693, Test Accuracy:0.33610, Max Test Accuracy:0.33610
Momentum-Epoch: 48, Train Err.:0.12145, Validation Err.:0.12283, Test Accuracy:0.95590, Max Test Accuracy:0.95590
Nesterov-Epoch: 48, Train Err.:0.10632, Validation Err.:0.11882, Test Accuracy:0.95770, Max Test Accuracy:0.95770
AdaGrad -Epoch: 48, Train Err.:0.04856, Validation Err.:0.09435, Test Accuracy:0.96910, Max Test Accuracy:0.96910
RMSprop -Epoch: 48, Train Err.:0.01105, Validation Err.:0.18191, Test Accuracy:0.96160, Max Test Accuracy:0.96230
Adam    -Epoch: 48, Train Err.:0.00694, Validation Err.:0.10309, Test Accuracy:0.97330, Max Test Accuracy:0.97590

SGD     -Epoch: 49, Train Err.:2.19402, Validation Err.:2.19086, Test Accuracy:0.33860, Max Test Accuracy:0.33860
Momentum-Epoch: 49, Train Err.:0.11848, Validation Err.:0.12097, Test Accuracy:0.95670, Max Test Accuracy:0.95670
Nesterov-Epoch: 49, Train Err.:0.10323, Validation Err.:0.11721, Test Accuracy:0.95820, Max Test Accuracy:0.95820
AdaGrad -Epoch: 49, Train Err.:0.04767, Validation Err.:0.09413, Test Accuracy:0.96920, Max Test Accuracy:0.96920
RMSprop -Epoch: 49, Train Err.:0.02792, Validation Err.:0.19250, Test Accuracy:0.95830, Max Test Accuracy:0.96230
Adam    -Epoch: 49, Train Err.:0.00680, Validation Err.:0.10461, Test Accuracy:0.97380, Max Test Accuracy:0.97590

In [6]:
markers = {"SGD": "o", "Momentum": "x", "Nesterov": "s", "AdaGrad": "o", "RMSprop": "x", "Adam": "s"}

f, axarr = plt.subplots(2, 2, figsize=(15,10))
for key in optimizers.keys():
    axarr[0, 0].plot(epoch_list[1:], train_errors[key][1:], marker=markers[key], markevery=2, label=key)
axarr[0, 0].set_ylabel('Train - Total Error')
axarr[0, 0].set_xlabel('Epochs')
axarr[0, 0].grid(True)
axarr[0, 0].set_title('Train Error')
axarr[0, 0].legend(loc='upper left')

for key in optimizers.keys():
    axarr[0, 1].plot(epoch_list[1:], validation_errors[key][1:], marker=markers[key], markevery=2, label=key)
axarr[0, 1].set_ylabel('Validation - Total Error')
axarr[0, 1].set_xlabel('Epochs')
axarr[0, 1].grid(True)
axarr[0, 1].set_title('Validation Error')
axarr[0, 1].legend(loc='upper left')

for key in optimizers.keys():
    axarr[1, 0].plot(epoch_list[1:], train_errors[key][1:], marker=markers[key], markevery=2, label=key)
axarr[1, 0].set_ylabel('Train - Total Error')
axarr[1, 0].set_xlabel('Epochs')
axarr[1, 0].grid(True)
axarr[1, 0].set_ylim(0, 0.3)
axarr[1, 0].set_title('Train Error (0.00 ~ 0.30)')
axarr[1, 0].legend(loc='upper left')

for key in optimizers.keys():
    axarr[1, 1].plot(epoch_list[1:], validation_errors[key][1:], marker=markers[key], markevery=2, label=key)
axarr[1, 1].set_ylabel('Validation - Total Error')
axarr[1, 1].set_xlabel('Epochs')
axarr[1, 1].grid(True)
axarr[1, 1].set_ylim(0, 0.3)
axarr[1, 1].set_title('Validation Error (0.00 ~ 0.30)')
axarr[1, 1].legend(loc='upper left')

f.subplots_adjust(hspace=0.3)

plt.show()
In [7]:
f, axarr = plt.subplots(2, 1, figsize=(15,10))
for key in optimizers.keys():
    axarr[0].plot(epoch_list[1:], test_accuracy_values[key][1:], marker=markers[key], markevery=1, label=key)
axarr[0].set_ylabel('Test Accuracy')
axarr[0].set_xlabel('Epochs')
axarr[0].grid(True)
axarr[0].set_title('Test Accuracy')
axarr[0].legend(loc='upper left')

for key in optimizers.keys():
    axarr[1].plot(epoch_list[1:], test_accuracy_values[key][1:], marker=markers[key], markevery=1, label=key)
axarr[1].set_ylabel('Test Accuracy')
axarr[1].set_xlabel('Epochs')
axarr[1].grid(True)
axarr[1].set_ylim(0.9, 1.0)
axarr[1].set_title('Test Accuracy (0.9 ~ 1.0)')
axarr[1].legend(loc='upper left')

f.subplots_adjust(hspace=0.3)
plt.show()
In [9]:
for key in optimizers.keys():
    print("{0:26s} - Epoch:{1:3d}, Max Test Accuracy: {2:7.5f}".format(key, max_test_accuracy_epoch[key], max_test_accuracy_value[key]))
SGD                        - Epoch: 49, Max Test Accuracy: 0.33860
Momentum                   - Epoch: 49, Max Test Accuracy: 0.95670
Nesterov                   - Epoch: 49, Max Test Accuracy: 0.95820
AdaGrad                    - Epoch: 49, Max Test Accuracy: 0.96920
RMSprop                    - Epoch: 37, Max Test Accuracy: 0.96230
Adam                       - Epoch: 45, Max Test Accuracy: 0.97590