MNIST-Neural Network-Two Hidden Layers with Variable Initializers

In [1]:
# coding: utf-8
import sys, os
import numpy as np
import matplotlib.pyplot as plt
import math

sys.path.append(os.pardir)
from deeplink.mnist import *
from deeplink.functions import *
from deeplink.layers import *
from deeplink.util import *
from deeplink.optimizers import *

Initializers

In [2]:
class Zero_Initializer:
    def __init__(self, input_size, hidden_layer1_size, hidden_layer2_size, output_size):
        self.input_size = input_size
        self.hidden_layer1_size = hidden_layer1_size
        self.hidden_layer2_size = hidden_layer2_size
        self.output_size = output_size
        self.params = {}        
        self.params['W1'] = np.zeros((self.input_size, self.hidden_layer1_size))
        self.params['b1'] = np.zeros(self.hidden_layer1_size)
        self.params['W2'] = np.zeros((self.hidden_layer1_size, self.hidden_layer2_size))
        self.params['b2'] = np.zeros(self.hidden_layer2_size)
        self.params['W3'] = np.zeros((self.hidden_layer2_size, self.output_size))
        self.params['b3'] = np.zeros(self.output_size)
        
    def get_params(self):
        return self.params
    
class N1_Initializer:
    def __init__(self, input_size, hidden_layer1_size, hidden_layer2_size, output_size):
        self.input_size = input_size
        self.hidden_layer1_size = hidden_layer1_size
        self.hidden_layer2_size = hidden_layer2_size
        self.output_size = output_size
        self.params = {}
        self.params['W1'] = np.random.randn(self.input_size, self.hidden_layer1_size)
        self.params['b1'] = np.random.randn(self.hidden_layer1_size)
        self.params['W2'] = np.random.randn(self.hidden_layer1_size, self.hidden_layer2_size)
        self.params['b2'] = np.random.randn(self.hidden_layer2_size)
        self.params['W3'] = np.random.randn(self.hidden_layer2_size, self.output_size)
        self.params['b3'] = np.random.randn(self.output_size)
        
    def get_params(self):
        return self.params
    
class N2_Initializer:
    def __init__(self, input_size, hidden_layer1_size, hidden_layer2_size, output_size):
        self.input_size = input_size
        self.hidden_layer1_size = hidden_layer1_size
        self.hidden_layer2_size = hidden_layer2_size
        self.output_size = output_size
        self.params = {}
        self.params['W1'] = np.random.randn(self.input_size, self.hidden_layer1_size) * 0.01
        self.params['b1'] = np.random.randn(self.hidden_layer1_size) * 0.01
        self.params['W2'] = np.random.randn(self.hidden_layer1_size, self.hidden_layer2_size) * 0.01
        self.params['b2'] = np.random.randn(self.hidden_layer2_size) * 0.01
        self.params['W3'] = np.random.randn(self.hidden_layer2_size, self.output_size) * 0.01
        self.params['b3'] = np.random.randn(self.output_size) * 0.01
    
    def get_params(self):
        return self.params
    
class Xavier_Initializer:
    def __init__(self, input_size, hidden_layer1_size, hidden_layer2_size, output_size):
        self.input_size = input_size
        self.hidden_layer1_size = hidden_layer1_size
        self.hidden_layer2_size = hidden_layer2_size
        self.output_size = output_size
        self.params = {}
        self.params['W1'] = np.random.randn(self.input_size, self.hidden_layer1_size) / np.sqrt(self.input_size)
        self.params['b1'] = np.random.randn(self.hidden_layer1_size) / np.sqrt(self.input_size)
        self.params['W2'] = np.random.randn(self.hidden_layer1_size, self.hidden_layer2_size) / np.sqrt(self.hidden_layer1_size)
        self.params['b2'] = np.random.randn(self.hidden_layer2_size) / np.sqrt(self.hidden_layer1_size)
        self.params['W3'] = np.random.randn(self.hidden_layer2_size, self.output_size) / np.sqrt(self.hidden_layer2_size)
        self.params['b3'] = np.random.randn(self.output_size) / np.sqrt(self.hidden_layer2_size)
    
    def get_params(self):
        return self.params

class He_Initializer:
    def __init__(self, input_size, hidden_layer1_size, hidden_layer2_size, output_size):
        self.input_size = input_size
        self.hidden_layer1_size = hidden_layer1_size
        self.hidden_layer2_size = hidden_layer2_size
        self.output_size = output_size
        self.params = {}
        self.params['W1'] = np.random.randn(self.input_size, self.hidden_layer1_size) * np.sqrt(2) / np.sqrt(self.input_size)
        self.params['b1'] = np.random.randn(self.hidden_layer1_size) * np.sqrt(2) / np.sqrt(self.input_size)
        self.params['W2'] = np.random.randn(self.hidden_layer1_size, self.hidden_layer2_size) * np.sqrt(2) / np.sqrt(self.hidden_layer1_size)
        self.params['b2'] = np.random.randn(self.hidden_layer2_size) * np.sqrt(2) / np.sqrt(self.hidden_layer1_size)
        self.params['W3'] = np.random.randn(self.hidden_layer2_size, self.output_size) * np.sqrt(2) / np.sqrt(self.hidden_layer2_size)
        self.params['b3'] = np.random.randn(self.output_size) * np.sqrt(2) / np.sqrt(self.hidden_layer2_size)
    
    def get_params(self):
        return self.params  

Multilayer Neural Network Model

In [5]:
import sys, os
from collections import OrderedDict
from scipy import stats
from pandas import DataFrame

markers = {"Zero": "o", "N1": "x", "N2": "s", "Xavier": "o", "He": "x"}

class TwoLayerNet2:
    def __init__(self, initializer):
        self.params = initializer.get_params()

        self.layers = OrderedDict()
        self.layers['Affine1'] = Affine(self.params['W1'], self.params['b1'])
        self.layers['Relu1'] = ReLU()
        self.layers['Affine2'] = Affine(self.params['W2'], self.params['b2'])
        self.layers['Relu2'] = ReLU()
        self.layers['Affine3'] = Affine(self.params['W3'], self.params['b3'])
        
        self.activations = OrderedDict()
        self.activations[0] = None
        

        self.lastLayer = SoftmaxWithCrossEntropyLoss()
            
    def predict(self, x):
        for layer in self.layers.values():
            x = layer.forward(x)
        
        return x
        
    def loss(self, x, t):
        y = self.predict(x)
        return self.lastLayer.forward(y, t)
    
    def accuracy(self, x, t):
        y = self.predict(x)
        y = np.argmax(y, axis=1)
        if t.ndim != 1 : t = np.argmax(t, axis=1)
        
        accuracy = np.sum(y == t) / float(x.shape[0])
        return accuracy
        
    def backpropagation_gradient(self, x, t):
        # forward
        self.loss(x, t)

        # backward
        din = 1
        din = self.lastLayer.backward(din)
        
        layers = list(self.layers.values())
        layers.reverse()
        for layer in layers:
            din = layer.backward(din)

        grads = {}
        grads['W1'], grads['b1'] = self.layers['Affine1'].dW, self.layers['Affine1'].db
        grads['W2'], grads['b2'] = self.layers['Affine2'].dW, self.layers['Affine2'].db
        grads['W3'], grads['b3'] = self.layers['Affine3'].dW, self.layers['Affine3'].db

        return grads
    
    def learning(self, learning_rate, x_batch, t_batch, optimizer):
        grads = self.backpropagation_gradient(x_batch, t_batch)
        optimizer.update(self.params, grads)

Learning and Validation

In [10]:
data = mnist_data("/Users/yhhan/git/aiclass/0.Professor/data/MNIST_data/.")
(img_train, label_train), (img_validation, label_validation), (img_test, label_test) = data.load_mnist(flatten=True, normalize=True, one_hot_label=True)

input_size=784
hidden_layer1_size=128
hidden_layer2_size=128
output_size=10

initializers = {}
initializers['Zero'] = Zero_Initializer(input_size, hidden_layer1_size, hidden_layer2_size, output_size)
initializers['N1'] = N1_Initializer(input_size, hidden_layer1_size, hidden_layer2_size, output_size)
initializers['N2'] = N2_Initializer(input_size, hidden_layer1_size, hidden_layer2_size, output_size)
initializers['Xavier'] = Xavier_Initializer(input_size, hidden_layer1_size, hidden_layer2_size, output_size)
initializers['He'] = He_Initializer(input_size, hidden_layer1_size, hidden_layer2_size, output_size)

num_epochs = 50
train_size = img_train.shape[0]
batch_size = 1000
learning_rate = 0.1

networks = {}
train_errors = {}
validation_errors = {}
test_accuracy_values = {}
max_test_accuracy_epoch = {}
max_test_accuracy_value = {}

optimizer = AdaGrad()

for key in initializers.keys():
    networks[key] = TwoLayerNet2(initializer=initializers[key])
    train_errors[key] = [] 
    validation_errors[key] = []
    test_accuracy_values[key] = []
    max_test_accuracy_epoch[key] = 0
    max_test_accuracy_value[key] = 0.0

num_batch = math.ceil(train_size / batch_size)
epoch_list = []

for i in range(num_epochs):
    epoch_list.append(i)
    for key in initializers.keys():
        for k in range(num_batch):
            x_batch = img_train[k * batch_size : k * batch_size + batch_size]
            t_batch = label_train[k * batch_size : k * batch_size + batch_size]
            networks[key].learning(learning_rate, x_batch, t_batch, optimizer)

        train_loss = networks[key].loss(x_batch, t_batch)
        train_errors[key].append(train_loss)

        validation_loss = networks[key].loss(img_validation, label_validation)
        validation_errors[key].append(validation_loss)    

        test_accuracy = networks[key].accuracy(img_test, label_test)
        test_accuracy_values[key].append(test_accuracy)
        if test_accuracy > max_test_accuracy_value[key]:
            max_test_accuracy_epoch[key] = i            
            max_test_accuracy_value[key] = test_accuracy
        print("{0:6s}-Epoch:{1:3d}, Train Err.:{2:7.5f}, Validation Err.:{3:7.5f}, Test Accuracy:{4:7.5f}, Max Test Accuracy:{5:7.5f}".format(
            key,
            i,
            train_loss,
            validation_loss,
            test_accuracy,
            max_test_accuracy_value[key]
        ))
    print()
Zero  -Epoch:  0, Train Err.:2.30211, Validation Err.:2.30169, Test Accuracy:0.11350, Max Test Accuracy:0.11350
../common/functions.py:56: RuntimeWarning: divide by zero encountered in log
  return -np.sum(np.log(y[np.arange(batch_size), t])) / batch_size
N1    -Epoch:  0, Train Err.:    inf, Validation Err.:    inf, Test Accuracy:0.63990, Max Test Accuracy:0.63990
N2    -Epoch:  0, Train Err.:2.30212, Validation Err.:2.30208, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  0, Train Err.:1.99279, Validation Err.:1.98679, Test Accuracy:0.45400, Max Test Accuracy:0.45400
He    -Epoch:  0, Train Err.:1.80990, Validation Err.:1.80165, Test Accuracy:0.51980, Max Test Accuracy:0.51980

Zero  -Epoch:  1, Train Err.:2.30212, Validation Err.:2.30169, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch:  1, Train Err.:    inf, Validation Err.:    inf, Test Accuracy:0.72610, Max Test Accuracy:0.72610
N2    -Epoch:  1, Train Err.:2.30186, Validation Err.:2.30176, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  1, Train Err.:1.55536, Validation Err.:1.52620, Test Accuracy:0.62590, Max Test Accuracy:0.62590
He    -Epoch:  1, Train Err.:1.27103, Validation Err.:1.24178, Test Accuracy:0.73610, Max Test Accuracy:0.73610

Zero  -Epoch:  2, Train Err.:2.30213, Validation Err.:2.30169, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch:  2, Train Err.:    inf, Validation Err.:    inf, Test Accuracy:0.76480, Max Test Accuracy:0.76480
N2    -Epoch:  2, Train Err.:2.30168, Validation Err.:2.30154, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  2, Train Err.:1.16768, Validation Err.:1.11996, Test Accuracy:0.73410, Max Test Accuracy:0.73410
He    -Epoch:  2, Train Err.:0.91962, Validation Err.:0.86791, Test Accuracy:0.79910, Max Test Accuracy:0.79910

Zero  -Epoch:  3, Train Err.:2.30214, Validation Err.:2.30169, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch:  3, Train Err.:    inf, Validation Err.:    inf, Test Accuracy:0.78750, Max Test Accuracy:0.78750
N2    -Epoch:  3, Train Err.:2.30154, Validation Err.:2.30137, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  3, Train Err.:0.89864, Validation Err.:0.84130, Test Accuracy:0.78700, Max Test Accuracy:0.78700
He    -Epoch:  3, Train Err.:0.73808, Validation Err.:0.67435, Test Accuracy:0.82790, Max Test Accuracy:0.82790

Zero  -Epoch:  4, Train Err.:2.30215, Validation Err.:2.30170, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch:  4, Train Err.:    inf, Validation Err.:    inf, Test Accuracy:0.80310, Max Test Accuracy:0.80310
N2    -Epoch:  4, Train Err.:2.30142, Validation Err.:2.30121, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  4, Train Err.:0.73909, Validation Err.:0.67867, Test Accuracy:0.81790, Max Test Accuracy:0.81790
He    -Epoch:  4, Train Err.:0.63672, Validation Err.:0.56600, Test Accuracy:0.84640, Max Test Accuracy:0.84640

Zero  -Epoch:  5, Train Err.:2.30216, Validation Err.:2.30171, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch:  5, Train Err.:    inf, Validation Err.:    inf, Test Accuracy:0.81480, Max Test Accuracy:0.81480
N2    -Epoch:  5, Train Err.:2.30130, Validation Err.:2.30106, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  5, Train Err.:0.64144, Validation Err.:0.57992, Test Accuracy:0.83800, Max Test Accuracy:0.83800
He    -Epoch:  5, Train Err.:0.57145, Validation Err.:0.49721, Test Accuracy:0.85920, Max Test Accuracy:0.85920

Zero  -Epoch:  6, Train Err.:2.30217, Validation Err.:2.30171, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch:  6, Train Err.:    inf, Validation Err.:    inf, Test Accuracy:0.82320, Max Test Accuracy:0.82320
N2    -Epoch:  6, Train Err.:2.30117, Validation Err.:2.30092, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  6, Train Err.:0.57679, Validation Err.:0.51468, Test Accuracy:0.85160, Max Test Accuracy:0.85160
He    -Epoch:  6, Train Err.:0.52531, Validation Err.:0.44912, Test Accuracy:0.86880, Max Test Accuracy:0.86880

Zero  -Epoch:  7, Train Err.:2.30218, Validation Err.:2.30172, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch:  7, Train Err.:25.82635, Validation Err.:    inf, Test Accuracy:0.83060, Max Test Accuracy:0.83060
N2    -Epoch:  7, Train Err.:2.30104, Validation Err.:2.30076, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  7, Train Err.:0.53110, Validation Err.:0.46841, Test Accuracy:0.85990, Max Test Accuracy:0.85990
He    -Epoch:  7, Train Err.:0.49032, Validation Err.:0.41337, Test Accuracy:0.87820, Max Test Accuracy:0.87820

Zero  -Epoch:  8, Train Err.:2.30219, Validation Err.:2.30173, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch:  8, Train Err.:24.29886, Validation Err.:    inf, Test Accuracy:0.83710, Max Test Accuracy:0.83710
N2    -Epoch:  8, Train Err.:2.30090, Validation Err.:2.30059, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  8, Train Err.:0.49731, Validation Err.:0.43389, Test Accuracy:0.86890, Max Test Accuracy:0.86890
He    -Epoch:  8, Train Err.:0.46288, Validation Err.:0.38573, Test Accuracy:0.88560, Max Test Accuracy:0.88560

Zero  -Epoch:  9, Train Err.:2.30220, Validation Err.:2.30173, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch:  9, Train Err.:22.92013, Validation Err.:    inf, Test Accuracy:0.84160, Max Test Accuracy:0.84160
N2    -Epoch:  9, Train Err.:2.30073, Validation Err.:2.30041, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch:  9, Train Err.:0.47136, Validation Err.:0.40724, Test Accuracy:0.87580, Max Test Accuracy:0.87580
He    -Epoch:  9, Train Err.:0.44062, Validation Err.:0.36367, Test Accuracy:0.88950, Max Test Accuracy:0.88950

Zero  -Epoch: 10, Train Err.:2.30221, Validation Err.:2.30174, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 10, Train Err.:21.74523, Validation Err.:    inf, Test Accuracy:0.84640, Max Test Accuracy:0.84640
N2    -Epoch: 10, Train Err.:2.30054, Validation Err.:2.30019, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch: 10, Train Err.:0.45090, Validation Err.:0.38611, Test Accuracy:0.88000, Max Test Accuracy:0.88000
He    -Epoch: 10, Train Err.:0.42223, Validation Err.:0.34566, Test Accuracy:0.89290, Max Test Accuracy:0.89290

Zero  -Epoch: 11, Train Err.:2.30222, Validation Err.:2.30174, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 11, Train Err.:20.72326, Validation Err.:    inf, Test Accuracy:0.84980, Max Test Accuracy:0.84980
N2    -Epoch: 11, Train Err.:2.30031, Validation Err.:2.29995, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch: 11, Train Err.:0.43443, Validation Err.:0.36900, Test Accuracy:0.88320, Max Test Accuracy:0.88320
He    -Epoch: 11, Train Err.:0.40670, Validation Err.:0.33065, Test Accuracy:0.89730, Max Test Accuracy:0.89730

Zero  -Epoch: 12, Train Err.:2.30223, Validation Err.:2.30175, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 12, Train Err.:19.85780, Validation Err.:    inf, Test Accuracy:0.85310, Max Test Accuracy:0.85310
N2    -Epoch: 12, Train Err.:2.30004, Validation Err.:2.29965, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch: 12, Train Err.:0.42087, Validation Err.:0.35487, Test Accuracy:0.88700, Max Test Accuracy:0.88700
He    -Epoch: 12, Train Err.:0.39340, Validation Err.:0.31791, Test Accuracy:0.90090, Max Test Accuracy:0.90090

Zero  -Epoch: 13, Train Err.:2.30224, Validation Err.:2.30175, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 13, Train Err.:19.08249, Validation Err.:    inf, Test Accuracy:0.85600, Max Test Accuracy:0.85600
N2    -Epoch: 13, Train Err.:2.29971, Validation Err.:2.29930, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch: 13, Train Err.:0.40948, Validation Err.:0.34302, Test Accuracy:0.88890, Max Test Accuracy:0.88890
He    -Epoch: 13, Train Err.:0.38190, Validation Err.:0.30701, Test Accuracy:0.90310, Max Test Accuracy:0.90310

Zero  -Epoch: 14, Train Err.:2.30224, Validation Err.:2.30176, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 14, Train Err.:18.36657, Validation Err.:    inf, Test Accuracy:0.85860, Max Test Accuracy:0.85860
N2    -Epoch: 14, Train Err.:2.29929, Validation Err.:2.29886, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch: 14, Train Err.:0.39973, Validation Err.:0.33291, Test Accuracy:0.89100, Max Test Accuracy:0.89100
He    -Epoch: 14, Train Err.:0.37180, Validation Err.:0.29755, Test Accuracy:0.90530, Max Test Accuracy:0.90530

Zero  -Epoch: 15, Train Err.:2.30225, Validation Err.:2.30176, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 15, Train Err.:17.69516, Validation Err.:    inf, Test Accuracy:0.86080, Max Test Accuracy:0.86080
N2    -Epoch: 15, Train Err.:2.29876, Validation Err.:2.29831, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch: 15, Train Err.:0.39128, Validation Err.:0.32419, Test Accuracy:0.89400, Max Test Accuracy:0.89400
He    -Epoch: 15, Train Err.:0.36285, Validation Err.:0.28926, Test Accuracy:0.90760, Max Test Accuracy:0.90760

Zero  -Epoch: 16, Train Err.:2.30226, Validation Err.:2.30177, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 16, Train Err.:17.06487, Validation Err.:    inf, Test Accuracy:0.86250, Max Test Accuracy:0.86250
N2    -Epoch: 16, Train Err.:2.29809, Validation Err.:2.29761, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch: 16, Train Err.:0.38383, Validation Err.:0.31657, Test Accuracy:0.89540, Max Test Accuracy:0.89540
He    -Epoch: 16, Train Err.:0.35486, Validation Err.:0.28192, Test Accuracy:0.90990, Max Test Accuracy:0.90990

Zero  -Epoch: 17, Train Err.:2.30227, Validation Err.:2.30177, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 17, Train Err.:16.50245, Validation Err.:    inf, Test Accuracy:0.86420, Max Test Accuracy:0.86420
N2    -Epoch: 17, Train Err.:2.29720, Validation Err.:2.29670, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch: 17, Train Err.:0.37723, Validation Err.:0.30982, Test Accuracy:0.89730, Max Test Accuracy:0.89730
He    -Epoch: 17, Train Err.:0.34763, Validation Err.:0.27539, Test Accuracy:0.91080, Max Test Accuracy:0.91080

Zero  -Epoch: 18, Train Err.:2.30227, Validation Err.:2.30177, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 18, Train Err.:15.96660, Validation Err.:    inf, Test Accuracy:0.86560, Max Test Accuracy:0.86560
N2    -Epoch: 18, Train Err.:2.29600, Validation Err.:2.29546, Test Accuracy:0.11350, Max Test Accuracy:0.11350
Xavier-Epoch: 18, Train Err.:0.37127, Validation Err.:0.30382, Test Accuracy:0.89820, Max Test Accuracy:0.89820
He    -Epoch: 18, Train Err.:0.34103, Validation Err.:0.26950, Test Accuracy:0.91220, Max Test Accuracy:0.91220

Zero  -Epoch: 19, Train Err.:2.30228, Validation Err.:2.30178, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 19, Train Err.:15.47522, Validation Err.:    inf, Test Accuracy:0.86730, Max Test Accuracy:0.86730
N2    -Epoch: 19, Train Err.:2.29431, Validation Err.:2.29374, Test Accuracy:0.11400, Max Test Accuracy:0.11400
Xavier-Epoch: 19, Train Err.:0.36583, Validation Err.:0.29841, Test Accuracy:0.89890, Max Test Accuracy:0.89890
He    -Epoch: 19, Train Err.:0.33502, Validation Err.:0.26416, Test Accuracy:0.91280, Max Test Accuracy:0.91280

Zero  -Epoch: 20, Train Err.:2.30228, Validation Err.:2.30178, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 20, Train Err.:15.02592, Validation Err.:    inf, Test Accuracy:0.86850, Max Test Accuracy:0.86850
N2    -Epoch: 20, Train Err.:2.29184, Validation Err.:2.29123, Test Accuracy:0.13490, Max Test Accuracy:0.13490
Xavier-Epoch: 20, Train Err.:0.36084, Validation Err.:0.29351, Test Accuracy:0.89980, Max Test Accuracy:0.89980
He    -Epoch: 20, Train Err.:0.32943, Validation Err.:0.25932, Test Accuracy:0.91400, Max Test Accuracy:0.91400

Zero  -Epoch: 21, Train Err.:2.30229, Validation Err.:2.30178, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 21, Train Err.:14.60892, Validation Err.:    inf, Test Accuracy:0.87060, Max Test Accuracy:0.87060
N2    -Epoch: 21, Train Err.:2.28804, Validation Err.:2.28739, Test Accuracy:0.18360, Max Test Accuracy:0.18360
Xavier-Epoch: 21, Train Err.:0.35622, Validation Err.:0.28905, Test Accuracy:0.90150, Max Test Accuracy:0.90150
He    -Epoch: 21, Train Err.:0.32420, Validation Err.:0.25490, Test Accuracy:0.91470, Max Test Accuracy:0.91470

Zero  -Epoch: 22, Train Err.:2.30229, Validation Err.:2.30179, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 22, Train Err.:14.21866, Validation Err.:    inf, Test Accuracy:0.87180, Max Test Accuracy:0.87180
N2    -Epoch: 22, Train Err.:2.28187, Validation Err.:2.28116, Test Accuracy:0.22900, Max Test Accuracy:0.22900
Xavier-Epoch: 22, Train Err.:0.35193, Validation Err.:0.28494, Test Accuracy:0.90300, Max Test Accuracy:0.90300
He    -Epoch: 22, Train Err.:0.31933, Validation Err.:0.25083, Test Accuracy:0.91540, Max Test Accuracy:0.91540

Zero  -Epoch: 23, Train Err.:2.30230, Validation Err.:2.30179, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 23, Train Err.:13.85946, Validation Err.:    inf, Test Accuracy:0.87340, Max Test Accuracy:0.87340
N2    -Epoch: 23, Train Err.:2.27135, Validation Err.:2.27056, Test Accuracy:0.24610, Max Test Accuracy:0.24610
Xavier-Epoch: 23, Train Err.:0.34791, Validation Err.:0.28115, Test Accuracy:0.90390, Max Test Accuracy:0.90390
He    -Epoch: 23, Train Err.:0.31482, Validation Err.:0.24708, Test Accuracy:0.91590, Max Test Accuracy:0.91590

Zero  -Epoch: 24, Train Err.:2.30230, Validation Err.:2.30179, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 24, Train Err.:13.51581, Validation Err.:    inf, Test Accuracy:0.87490, Max Test Accuracy:0.87490
N2    -Epoch: 24, Train Err.:2.25307, Validation Err.:2.25220, Test Accuracy:0.24780, Max Test Accuracy:0.24780
Xavier-Epoch: 24, Train Err.:0.34411, Validation Err.:0.27763, Test Accuracy:0.90440, Max Test Accuracy:0.90440
He    -Epoch: 24, Train Err.:0.31060, Validation Err.:0.24359, Test Accuracy:0.91730, Max Test Accuracy:0.91730

Zero  -Epoch: 25, Train Err.:2.30230, Validation Err.:2.30179, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 25, Train Err.:13.18274, Validation Err.:    inf, Test Accuracy:0.87580, Max Test Accuracy:0.87580
N2    -Epoch: 25, Train Err.:2.22317, Validation Err.:2.22237, Test Accuracy:0.24820, Max Test Accuracy:0.24820
Xavier-Epoch: 25, Train Err.:0.34053, Validation Err.:0.27436, Test Accuracy:0.90600, Max Test Accuracy:0.90600
He    -Epoch: 25, Train Err.:0.30662, Validation Err.:0.24035, Test Accuracy:0.91800, Max Test Accuracy:0.91800

Zero  -Epoch: 26, Train Err.:2.30231, Validation Err.:2.30180, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 26, Train Err.:12.87416, Validation Err.:    inf, Test Accuracy:0.87680, Max Test Accuracy:0.87680
N2    -Epoch: 26, Train Err.:2.18291, Validation Err.:2.18254, Test Accuracy:0.25720, Max Test Accuracy:0.25720
Xavier-Epoch: 26, Train Err.:0.33713, Validation Err.:0.27129, Test Accuracy:0.90670, Max Test Accuracy:0.90670
He    -Epoch: 26, Train Err.:0.30286, Validation Err.:0.23731, Test Accuracy:0.91890, Max Test Accuracy:0.91890

Zero  -Epoch: 27, Train Err.:2.30231, Validation Err.:2.30180, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 27, Train Err.:12.60325, Validation Err.:    inf, Test Accuracy:0.87760, Max Test Accuracy:0.87760
N2    -Epoch: 27, Train Err.:2.14090, Validation Err.:2.14130, Test Accuracy:0.27190, Max Test Accuracy:0.27190
Xavier-Epoch: 27, Train Err.:0.33391, Validation Err.:0.26840, Test Accuracy:0.90760, Max Test Accuracy:0.90760
He    -Epoch: 27, Train Err.:0.29934, Validation Err.:0.23448, Test Accuracy:0.92000, Max Test Accuracy:0.92000

Zero  -Epoch: 28, Train Err.:2.30231, Validation Err.:2.30180, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 28, Train Err.:12.34224, Validation Err.:    inf, Test Accuracy:0.87900, Max Test Accuracy:0.87900
N2    -Epoch: 28, Train Err.:2.10159, Validation Err.:2.10259, Test Accuracy:0.28700, Max Test Accuracy:0.28700
Xavier-Epoch: 28, Train Err.:0.33086, Validation Err.:0.26567, Test Accuracy:0.90840, Max Test Accuracy:0.90840
He    -Epoch: 28, Train Err.:0.29601, Validation Err.:0.23183, Test Accuracy:0.92040, Max Test Accuracy:0.92040

Zero  -Epoch: 29, Train Err.:2.30232, Validation Err.:2.30180, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 29, Train Err.:12.10488, Validation Err.:    inf, Test Accuracy:0.88010, Max Test Accuracy:0.88010
N2    -Epoch: 29, Train Err.:2.06191, Validation Err.:2.06333, Test Accuracy:0.29930, Max Test Accuracy:0.29930
Xavier-Epoch: 29, Train Err.:0.32795, Validation Err.:0.26309, Test Accuracy:0.90900, Max Test Accuracy:0.90900
He    -Epoch: 29, Train Err.:0.29285, Validation Err.:0.22933, Test Accuracy:0.92100, Max Test Accuracy:0.92100

Zero  -Epoch: 30, Train Err.:2.30232, Validation Err.:2.30180, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 30, Train Err.:11.88343, Validation Err.:    inf, Test Accuracy:0.88050, Max Test Accuracy:0.88050
N2    -Epoch: 30, Train Err.:2.01704, Validation Err.:2.01853, Test Accuracy:0.30670, Max Test Accuracy:0.30670
Xavier-Epoch: 30, Train Err.:0.32514, Validation Err.:0.26063, Test Accuracy:0.90980, Max Test Accuracy:0.90980
He    -Epoch: 30, Train Err.:0.28985, Validation Err.:0.22697, Test Accuracy:0.92140, Max Test Accuracy:0.92140

Zero  -Epoch: 31, Train Err.:2.30232, Validation Err.:2.30180, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 31, Train Err.:11.68085, Validation Err.:    inf, Test Accuracy:0.88180, Max Test Accuracy:0.88180
N2    -Epoch: 31, Train Err.:1.96356, Validation Err.:1.96416, Test Accuracy:0.30510, Max Test Accuracy:0.30670
Xavier-Epoch: 31, Train Err.:0.32242, Validation Err.:0.25830, Test Accuracy:0.91010, Max Test Accuracy:0.91010
He    -Epoch: 31, Train Err.:0.28697, Validation Err.:0.22476, Test Accuracy:0.92190, Max Test Accuracy:0.92190

Zero  -Epoch: 32, Train Err.:2.30232, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 32, Train Err.:11.50463, Validation Err.:    inf, Test Accuracy:0.88240, Max Test Accuracy:0.88240
N2    -Epoch: 32, Train Err.:1.90156, Validation Err.:1.90006, Test Accuracy:0.30510, Max Test Accuracy:0.30670
Xavier-Epoch: 32, Train Err.:0.31978, Validation Err.:0.25606, Test Accuracy:0.91060, Max Test Accuracy:0.91060
He    -Epoch: 32, Train Err.:0.28422, Validation Err.:0.22265, Test Accuracy:0.92320, Max Test Accuracy:0.92320

Zero  -Epoch: 33, Train Err.:2.30232, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 33, Train Err.:11.32855, Validation Err.:    inf, Test Accuracy:0.88290, Max Test Accuracy:0.88290
N2    -Epoch: 33, Train Err.:1.83510, Validation Err.:1.83060, Test Accuracy:0.31820, Max Test Accuracy:0.31820
Xavier-Epoch: 33, Train Err.:0.31722, Validation Err.:0.25393, Test Accuracy:0.91090, Max Test Accuracy:0.91090
He    -Epoch: 33, Train Err.:0.28158, Validation Err.:0.22065, Test Accuracy:0.92390, Max Test Accuracy:0.92390

Zero  -Epoch: 34, Train Err.:2.30232, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 34, Train Err.:11.15499, Validation Err.:    inf, Test Accuracy:0.88320, Max Test Accuracy:0.88320
N2    -Epoch: 34, Train Err.:1.76730, Validation Err.:1.75975, Test Accuracy:0.34470, Max Test Accuracy:0.34470
Xavier-Epoch: 34, Train Err.:0.31477, Validation Err.:0.25190, Test Accuracy:0.91150, Max Test Accuracy:0.91150
He    -Epoch: 34, Train Err.:0.27906, Validation Err.:0.21874, Test Accuracy:0.92430, Max Test Accuracy:0.92430

Zero  -Epoch: 35, Train Err.:2.30233, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 35, Train Err.:10.99644, Validation Err.:    inf, Test Accuracy:0.88330, Max Test Accuracy:0.88330
N2    -Epoch: 35, Train Err.:1.69725, Validation Err.:1.68666, Test Accuracy:0.38870, Max Test Accuracy:0.38870
Xavier-Epoch: 35, Train Err.:0.31241, Validation Err.:0.24995, Test Accuracy:0.91190, Max Test Accuracy:0.91190
He    -Epoch: 35, Train Err.:0.27665, Validation Err.:0.21692, Test Accuracy:0.92510, Max Test Accuracy:0.92510

Zero  -Epoch: 36, Train Err.:2.30233, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 36, Train Err.:10.83669, Validation Err.:    inf, Test Accuracy:0.88400, Max Test Accuracy:0.88400
N2    -Epoch: 36, Train Err.:1.62150, Validation Err.:1.60803, Test Accuracy:0.42470, Max Test Accuracy:0.42470
Xavier-Epoch: 36, Train Err.:0.31014, Validation Err.:0.24809, Test Accuracy:0.91240, Max Test Accuracy:0.91240
He    -Epoch: 36, Train Err.:0.27434, Validation Err.:0.21520, Test Accuracy:0.92560, Max Test Accuracy:0.92560

Zero  -Epoch: 37, Train Err.:2.30233, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 37, Train Err.:10.67719, Validation Err.:    inf, Test Accuracy:0.88430, Max Test Accuracy:0.88430
N2    -Epoch: 37, Train Err.:1.53975, Validation Err.:1.52371, Test Accuracy:0.44970, Max Test Accuracy:0.44970
Xavier-Epoch: 37, Train Err.:0.30793, Validation Err.:0.24630, Test Accuracy:0.91320, Max Test Accuracy:0.91320
He    -Epoch: 37, Train Err.:0.27211, Validation Err.:0.21356, Test Accuracy:0.92590, Max Test Accuracy:0.92590

Zero  -Epoch: 38, Train Err.:2.30233, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 38, Train Err.:10.52266, Validation Err.:    inf, Test Accuracy:0.88480, Max Test Accuracy:0.88480
N2    -Epoch: 38, Train Err.:1.45775, Validation Err.:1.43971, Test Accuracy:0.47500, Max Test Accuracy:0.47500
Xavier-Epoch: 38, Train Err.:0.30582, Validation Err.:0.24457, Test Accuracy:0.91390, Max Test Accuracy:0.91390
He    -Epoch: 38, Train Err.:0.26993, Validation Err.:0.21198, Test Accuracy:0.92630, Max Test Accuracy:0.92630

Zero  -Epoch: 39, Train Err.:2.30233, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 39, Train Err.:10.37247, Validation Err.:    inf, Test Accuracy:0.88510, Max Test Accuracy:0.88510
N2    -Epoch: 39, Train Err.:1.38173, Validation Err.:1.36205, Test Accuracy:0.51080, Max Test Accuracy:0.51080
Xavier-Epoch: 39, Train Err.:0.30377, Validation Err.:0.24290, Test Accuracy:0.91460, Max Test Accuracy:0.91460
He    -Epoch: 39, Train Err.:0.26784, Validation Err.:0.21048, Test Accuracy:0.92700, Max Test Accuracy:0.92700

Zero  -Epoch: 40, Train Err.:2.30233, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 40, Train Err.:10.22131, Validation Err.:    inf, Test Accuracy:0.88560, Max Test Accuracy:0.88560
N2    -Epoch: 40, Train Err.:1.31435, Validation Err.:1.29278, Test Accuracy:0.54690, Max Test Accuracy:0.54690
Xavier-Epoch: 40, Train Err.:0.30177, Validation Err.:0.24129, Test Accuracy:0.91490, Max Test Accuracy:0.91490
He    -Epoch: 40, Train Err.:0.26581, Validation Err.:0.20903, Test Accuracy:0.92760, Max Test Accuracy:0.92760

Zero  -Epoch: 41, Train Err.:2.30233, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 41, Train Err.:10.07976, Validation Err.:    inf, Test Accuracy:0.88650, Max Test Accuracy:0.88650
N2    -Epoch: 41, Train Err.:1.25556, Validation Err.:1.23176, Test Accuracy:0.57420, Max Test Accuracy:0.57420
Xavier-Epoch: 41, Train Err.:0.29981, Validation Err.:0.23973, Test Accuracy:0.91530, Max Test Accuracy:0.91530
He    -Epoch: 41, Train Err.:0.26385, Validation Err.:0.20765, Test Accuracy:0.92820, Max Test Accuracy:0.92820

Zero  -Epoch: 42, Train Err.:2.30233, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 42, Train Err.:9.94493, Validation Err.:    inf, Test Accuracy:0.88690, Max Test Accuracy:0.88690
N2    -Epoch: 42, Train Err.:1.20456, Validation Err.:1.17817, Test Accuracy:0.59440, Max Test Accuracy:0.59440
Xavier-Epoch: 42, Train Err.:0.29792, Validation Err.:0.23822, Test Accuracy:0.91550, Max Test Accuracy:0.91550
He    -Epoch: 42, Train Err.:0.26195, Validation Err.:0.20631, Test Accuracy:0.92880, Max Test Accuracy:0.92880

Zero  -Epoch: 43, Train Err.:2.30234, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 43, Train Err.:9.81044, Validation Err.:    inf, Test Accuracy:0.88780, Max Test Accuracy:0.88780
N2    -Epoch: 43, Train Err.:1.16002, Validation Err.:1.13097, Test Accuracy:0.60890, Max Test Accuracy:0.60890
Xavier-Epoch: 43, Train Err.:0.29603, Validation Err.:0.23676, Test Accuracy:0.91570, Max Test Accuracy:0.91570
He    -Epoch: 43, Train Err.:0.26010, Validation Err.:0.20503, Test Accuracy:0.92910, Max Test Accuracy:0.92910

Zero  -Epoch: 44, Train Err.:2.30234, Validation Err.:2.30181, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 44, Train Err.:9.67931, Validation Err.:    inf, Test Accuracy:0.88810, Max Test Accuracy:0.88810
N2    -Epoch: 44, Train Err.:1.12095, Validation Err.:1.08926, Test Accuracy:0.62150, Max Test Accuracy:0.62150
Xavier-Epoch: 44, Train Err.:0.29418, Validation Err.:0.23536, Test Accuracy:0.91660, Max Test Accuracy:0.91660
He    -Epoch: 44, Train Err.:0.25831, Validation Err.:0.20379, Test Accuracy:0.92980, Max Test Accuracy:0.92980

Zero  -Epoch: 45, Train Err.:2.30234, Validation Err.:2.30182, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 45, Train Err.:9.55656, Validation Err.:    inf, Test Accuracy:0.88890, Max Test Accuracy:0.88890
N2    -Epoch: 45, Train Err.:1.08674, Validation Err.:1.05227, Test Accuracy:0.63450, Max Test Accuracy:0.63450
Xavier-Epoch: 45, Train Err.:0.29238, Validation Err.:0.23400, Test Accuracy:0.91700, Max Test Accuracy:0.91700
He    -Epoch: 45, Train Err.:0.25655, Validation Err.:0.20259, Test Accuracy:0.93040, Max Test Accuracy:0.93040

Zero  -Epoch: 46, Train Err.:2.30234, Validation Err.:2.30182, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 46, Train Err.:9.43401, Validation Err.:    inf, Test Accuracy:0.88900, Max Test Accuracy:0.88900
N2    -Epoch: 46, Train Err.:1.05650, Validation Err.:1.01938, Test Accuracy:0.64380, Max Test Accuracy:0.64380
Xavier-Epoch: 46, Train Err.:0.29062, Validation Err.:0.23268, Test Accuracy:0.91760, Max Test Accuracy:0.91760
He    -Epoch: 46, Train Err.:0.25483, Validation Err.:0.20144, Test Accuracy:0.93060, Max Test Accuracy:0.93060

Zero  -Epoch: 47, Train Err.:2.30234, Validation Err.:2.30182, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 47, Train Err.:9.31201, Validation Err.:    inf, Test Accuracy:0.88940, Max Test Accuracy:0.88940
N2    -Epoch: 47, Train Err.:1.02976, Validation Err.:0.98996, Test Accuracy:0.65430, Max Test Accuracy:0.65430
Xavier-Epoch: 47, Train Err.:0.28890, Validation Err.:0.23140, Test Accuracy:0.91790, Max Test Accuracy:0.91790
He    -Epoch: 47, Train Err.:0.25316, Validation Err.:0.20032, Test Accuracy:0.93080, Max Test Accuracy:0.93080

Zero  -Epoch: 48, Train Err.:2.30234, Validation Err.:2.30182, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 48, Train Err.:9.19183, Validation Err.:    inf, Test Accuracy:0.89010, Max Test Accuracy:0.89010
N2    -Epoch: 48, Train Err.:1.00585, Validation Err.:0.96336, Test Accuracy:0.66150, Max Test Accuracy:0.66150
Xavier-Epoch: 48, Train Err.:0.28722, Validation Err.:0.23017, Test Accuracy:0.91810, Max Test Accuracy:0.91810
He    -Epoch: 48, Train Err.:0.25154, Validation Err.:0.19924, Test Accuracy:0.93090, Max Test Accuracy:0.93090

Zero  -Epoch: 49, Train Err.:2.30234, Validation Err.:2.30182, Test Accuracy:0.11350, Max Test Accuracy:0.11350
N1    -Epoch: 49, Train Err.:9.07459, Validation Err.:    inf, Test Accuracy:0.89060, Max Test Accuracy:0.89060
N2    -Epoch: 49, Train Err.:0.98422, Validation Err.:0.93909, Test Accuracy:0.66920, Max Test Accuracy:0.66920
Xavier-Epoch: 49, Train Err.:0.28558, Validation Err.:0.22897, Test Accuracy:0.91830, Max Test Accuracy:0.91830
He    -Epoch: 49, Train Err.:0.24996, Validation Err.:0.19820, Test Accuracy:0.93090, Max Test Accuracy:0.93090

In [16]:
f, axarr = plt.subplots(2, 2, figsize=(15,10))
for key in initializers.keys():
    axarr[0, 0].plot(epoch_list[1:], train_errors[key][1:], marker=markers[key], markevery=2, label=key)
axarr[0, 0].set_ylabel('Train - Total Error')
axarr[0, 0].set_xlabel('Epochs')
axarr[0, 0].grid(True)
axarr[0, 0].set_title('Train Error')
axarr[0, 0].legend(loc='upper left')

for key in initializers.keys():
    axarr[0, 1].plot(epoch_list[1:], validation_errors[key][1:], marker=markers[key], markevery=2, label=key)
axarr[0, 1].set_ylabel('Validation - Total Error')
axarr[0, 1].set_xlabel('Epochs')
axarr[0, 1].grid(True)
axarr[0, 1].set_title('Validation Error')
axarr[0, 1].legend(loc='upper left')

for key in initializers.keys():
    axarr[1, 0].plot(epoch_list[1:], train_errors[key][1:], marker=markers[key], markevery=2, label=key)
axarr[1, 0].set_ylabel('Train - Total Error')
axarr[1, 0].set_xlabel('Epochs')
axarr[1, 0].grid(True)
axarr[1, 0].set_ylim(0, 3.0)
axarr[1, 0].set_title('Train Error (0.00 ~ 3.00)')
axarr[1, 0].legend(loc='upper left')

for key in initializers.keys():
    axarr[1, 1].plot(epoch_list[1:], validation_errors[key][1:], marker=markers[key], markevery=2, label=key)
axarr[1, 1].set_ylabel('Validation - Total Error')
axarr[1, 1].set_xlabel('Epochs')
axarr[1, 1].grid(True)
axarr[1, 1].set_ylim(0, 1.0)
axarr[1, 1].set_title('Validation Error (0.00 ~ 1.00)')
axarr[1, 1].legend(loc='upper right')

f.subplots_adjust(hspace=0.3)

plt.show()
In [17]:
f, axarr = plt.subplots(2, 1, figsize=(15,10))

for key in initializers.keys():
    axarr[0].plot(epoch_list[1:], test_accuracy_values[key][1:], marker=markers[key], markevery=1, label=key)
axarr[0].set_ylabel('Test Accuracy')
axarr[0].set_xlabel('Epochs')
axarr[0].grid(True)
axarr[0].set_title('Test Accuracy')
axarr[0].legend(loc='upper left')

for key in initializers.keys():
    axarr[1].plot(epoch_list[1:], test_accuracy_values[key][1:], marker=markers[key], markevery=1, label=key)
axarr[1].set_ylabel('Test Accuracy')
axarr[1].set_xlabel('Epochs')
axarr[1].grid(True)
axarr[1].set_ylim(0.7, 1.0)
axarr[1].set_title('Test Accuracy (0.7 ~ 1.0)')
axarr[1].legend(loc='upper left')

f.subplots_adjust(hspace=0.3)
plt.show()
In [18]:
for key in initializers.keys():
    print("{0:26s} - Epoch:{1:3d}, Max Test Accuracy: {2:7.5f}".format(key, max_test_accuracy_epoch[key], max_test_accuracy_value[key]))
Zero                       - Epoch:  0, Max Test Accuracy: 0.11350
N1                         - Epoch: 49, Max Test Accuracy: 0.89060
N2                         - Epoch: 49, Max Test Accuracy: 0.66920
Xavier                     - Epoch: 49, Max Test Accuracy: 0.91830
He                         - Epoch: 48, Max Test Accuracy: 0.93090