MNIST-Neural Network-Two Hidden Layers with Modules (and Backward Propagation)

In [1]:
# coding: utf-8
import sys, os
import numpy as np
import matplotlib.pyplot as plt
import math

sys.path.append(os.pardir)
from deeplink.mnist import *
from deeplink.networks import *

Multilayer Neural Network Model

In [2]:
data = mnist_data("/Users/yhhan/git/aiclass/0.Professor/data/MNIST_data/.")
(img_train, label_train), (img_validation, label_validation), (img_test, label_test) = data.load_mnist(flatten=True, normalize=True, one_hot_label=True)

input_size=784
hidden_layer1_size=128
hidden_layer2_size=128
output_size=10

network = MultiLayerNet(input_size, 
                        [hidden_layer1_size, hidden_layer2_size], 
                        output_size, 
                        activation='ReLU', 
                        initializer='N2',
                        optimizer='SGD',
                        learning_rate=0.1)

Learning and Validation

In [3]:
import math

num_epochs = 50
train_size = img_train.shape[0]
batch_size = 1000
learning_rate = 0.1

train_error_list = []
validation_error_list = []

test_accuracy_list = []
epoch_list = []

num_batch = math.ceil(train_size / batch_size)

for i in range(num_epochs):
#     batch_mask = np.random.choice(train_size, batch_size)
#     x_batch = img_train[batch_mask]
#     t_batch = label_train[batch_mask]
#     network.learning(learning_rate, x_batch, t_batch)
    for j in range(num_batch):
        x_batch = img_train[j * batch_size : j * batch_size + batch_size]
        t_batch = label_train[j * batch_size : j * batch_size + batch_size]
        network.learning(x_batch, t_batch)

    epoch_list.append(i)
    
    train_loss = network.loss(x_batch, t_batch)
    train_error_list.append(train_loss)
    
    validation_loss = network.loss(img_validation, label_validation)
    validation_error_list.append(validation_loss)    
    
    test_accuracy = network.accuracy(img_test, label_test)
    test_accuracy_list.append(test_accuracy)       
    
    print("Epoch: {0:5d}, Train Error: {1:7.5f}, Validation Error: {2:7.5f} - Test Accuracy: {3:7.5f}".format(
        i,
        train_loss,
        validation_loss,
        test_accuracy
    ))
Epoch:     0, Train Error: 2.30132, Validation Error: 2.30100 - Test Accuracy: 0.11350
Epoch:     1, Train Error: 2.29954, Validation Error: 2.29909 - Test Accuracy: 0.11350
Epoch:     2, Train Error: 2.29462, Validation Error: 2.29401 - Test Accuracy: 0.11350
Epoch:     3, Train Error: 2.26639, Validation Error: 2.26552 - Test Accuracy: 0.22400
Epoch:     4, Train Error: 1.94810, Validation Error: 1.94571 - Test Accuracy: 0.34400
Epoch:     5, Train Error: 1.13778, Validation Error: 1.11439 - Test Accuracy: 0.64440
Epoch:     6, Train Error: 0.74804, Validation Error: 0.69582 - Test Accuracy: 0.76010
Epoch:     7, Train Error: 0.63823, Validation Error: 0.57519 - Test Accuracy: 0.79880
Epoch:     8, Train Error: 0.57787, Validation Error: 0.51073 - Test Accuracy: 0.82620
Epoch:     9, Train Error: 0.53000, Validation Error: 0.46092 - Test Accuracy: 0.84390
Epoch:    10, Train Error: 0.48749, Validation Error: 0.41557 - Test Accuracy: 0.85720
Epoch:    11, Train Error: 0.45482, Validation Error: 0.37893 - Test Accuracy: 0.87010
Epoch:    12, Train Error: 0.42858, Validation Error: 0.35104 - Test Accuracy: 0.87720
Epoch:    13, Train Error: 0.40488, Validation Error: 0.32823 - Test Accuracy: 0.88440
Epoch:    14, Train Error: 0.38301, Validation Error: 0.30900 - Test Accuracy: 0.89130
Epoch:    15, Train Error: 0.36259, Validation Error: 0.29245 - Test Accuracy: 0.89790
Epoch:    16, Train Error: 0.34413, Validation Error: 0.27802 - Test Accuracy: 0.90310
Epoch:    17, Train Error: 0.32763, Validation Error: 0.26542 - Test Accuracy: 0.90850
Epoch:    18, Train Error: 0.31285, Validation Error: 0.25442 - Test Accuracy: 0.91170
Epoch:    19, Train Error: 0.29934, Validation Error: 0.24462 - Test Accuracy: 0.91410
Epoch:    20, Train Error: 0.28681, Validation Error: 0.23572 - Test Accuracy: 0.91730
Epoch:    21, Train Error: 0.27489, Validation Error: 0.22745 - Test Accuracy: 0.91950
Epoch:    22, Train Error: 0.26363, Validation Error: 0.21964 - Test Accuracy: 0.92210
Epoch:    23, Train Error: 0.25300, Validation Error: 0.21222 - Test Accuracy: 0.92470
Epoch:    24, Train Error: 0.24271, Validation Error: 0.20531 - Test Accuracy: 0.92760
Epoch:    25, Train Error: 0.23295, Validation Error: 0.19870 - Test Accuracy: 0.93040
Epoch:    26, Train Error: 0.22366, Validation Error: 0.19246 - Test Accuracy: 0.93340
Epoch:    27, Train Error: 0.21491, Validation Error: 0.18647 - Test Accuracy: 0.93600
Epoch:    28, Train Error: 0.20668, Validation Error: 0.18083 - Test Accuracy: 0.93720
Epoch:    29, Train Error: 0.19904, Validation Error: 0.17549 - Test Accuracy: 0.93920
Epoch:    30, Train Error: 0.19195, Validation Error: 0.17050 - Test Accuracy: 0.94090
Epoch:    31, Train Error: 0.18531, Validation Error: 0.16574 - Test Accuracy: 0.94320
Epoch:    32, Train Error: 0.17904, Validation Error: 0.16126 - Test Accuracy: 0.94530
Epoch:    33, Train Error: 0.17316, Validation Error: 0.15701 - Test Accuracy: 0.94680
Epoch:    34, Train Error: 0.16754, Validation Error: 0.15305 - Test Accuracy: 0.94720
Epoch:    35, Train Error: 0.16216, Validation Error: 0.14933 - Test Accuracy: 0.94910
Epoch:    36, Train Error: 0.15692, Validation Error: 0.14588 - Test Accuracy: 0.95050
Epoch:    37, Train Error: 0.15203, Validation Error: 0.14260 - Test Accuracy: 0.95150
Epoch:    38, Train Error: 0.14745, Validation Error: 0.13951 - Test Accuracy: 0.95210
Epoch:    39, Train Error: 0.14304, Validation Error: 0.13667 - Test Accuracy: 0.95320
Epoch:    40, Train Error: 0.13893, Validation Error: 0.13396 - Test Accuracy: 0.95420
Epoch:    41, Train Error: 0.13488, Validation Error: 0.13145 - Test Accuracy: 0.95490
Epoch:    42, Train Error: 0.13115, Validation Error: 0.12904 - Test Accuracy: 0.95580
Epoch:    43, Train Error: 0.12736, Validation Error: 0.12679 - Test Accuracy: 0.95590
Epoch:    44, Train Error: 0.12374, Validation Error: 0.12469 - Test Accuracy: 0.95650
Epoch:    45, Train Error: 0.12021, Validation Error: 0.12268 - Test Accuracy: 0.95660
Epoch:    46, Train Error: 0.11689, Validation Error: 0.12074 - Test Accuracy: 0.95740
Epoch:    47, Train Error: 0.11372, Validation Error: 0.11896 - Test Accuracy: 0.95850
Epoch:    48, Train Error: 0.11067, Validation Error: 0.11721 - Test Accuracy: 0.95900
Epoch:    49, Train Error: 0.10777, Validation Error: 0.11559 - Test Accuracy: 0.96020

Analysis with Graph

In [4]:
import matplotlib.pyplot as plt
%matplotlib inline
# Draw Graph about Error Values & Accuracy Values
def draw_error_values_and_accuracy(epoch_list, train_error_list, validation_error_list, test_accuracy_list):
    # Draw Error Values and Accuracy
    fig = plt.figure(figsize=(20, 5))
    plt.subplot(121)
    plt.plot(epoch_list[1:], train_error_list[1:], 'r', label='Train')
    plt.plot(epoch_list[1:], validation_error_list[1:], 'g', label='Validation')
    plt.ylabel('Total Error')
    plt.xlabel('Epochs')
    plt.grid(True)
    plt.legend(loc='upper right')

    plt.subplot(122)
    plt.plot(epoch_list[1:], test_accuracy_list[1:], 'b', label='Test')
    plt.ylabel('Accuracy')
    plt.xlabel('Epochs')
    plt.yticks(np.arange(0.0, 1.0, 0.05))
    plt.grid(True)
    plt.legend(loc='lower right')
    plt.show()

draw_error_values_and_accuracy(epoch_list, train_error_list, validation_error_list, test_accuracy_list)
    
def draw_false_prediction(diff_index_list):
    fig = plt.figure(figsize=(20, 5))
    for i in range(5):
        j = diff_index_list[i]
        print("False Prediction Index: %s, Prediction: %s, Ground Truth: %s" % (j, prediction[j], ground_truth[j]))
        img = np.array(img_test[j])
        img.shape = (28, 28)
        plt.subplot(150 + (i+1))
        plt.imshow(img, cmap='gray')
        
prediction = np.argmax(network.predict(img_test), axis=1)
ground_truth = np.argmax(label_test, axis=1)
        
print(prediction)
print(ground_truth)

diff_index_list = []
for i in range(len(img_test)):
    if (prediction[i] != ground_truth[i]):
        diff_index_list.append(i)

print("Total Test Image: {0}, Number of False Prediction: {1}".format(len(img_test), len(diff_index_list)))
print("Test Accuracy:", float(len(img_test) - len(diff_index_list)) / float(len(img_test)))
draw_false_prediction(diff_index_list)
[7 2 1 ..., 4 5 6]
[7 2 1 ..., 4 5 6]
Total Test Image: 10000, Number of False Prediction: 398
Test Accuracy: 0.9602
False Prediction Index: 8, Prediction: 6, Ground Truth: 5
False Prediction Index: 33, Prediction: 6, Ground Truth: 4
False Prediction Index: 124, Prediction: 4, Ground Truth: 7
False Prediction Index: 151, Prediction: 8, Ground Truth: 9
False Prediction Index: 233, Prediction: 7, Ground Truth: 8
In [ ]: