Convolutional Neural Networks

In [1]:
import tensorflow as tf
import math
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
import matplotlib.pyplot as plt
%matplotlib inline
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
Extracting MNIST_data/train-images-idx3-ubyte.gz
Extracting MNIST_data/train-labels-idx1-ubyte.gz
Extracting MNIST_data/t10k-images-idx3-ubyte.gz
Extracting MNIST_data/t10k-labels-idx1-ubyte.gz
In [2]:
x = tf.placeholder("float", [None, 784])
y = tf.placeholder("float", [None, 10])
  • Image reshape to 4D [-1, 28, 28, 1]
    • The MNIST images have just one gray color value (the number of color channels is one), so that the last value is one.
In [3]:
x_image = tf.reshape(x, [-1, 28, 28, 1])
print x_image.get_shape()
(?, 28, 28, 1)
In [4]:
keep_prob = tf.placeholder(tf.float32)
  • stride
    • we must specify the stride with which we slide the filter.
    • stride = 1: we slide the filters one pixel at a time.
    • stride = 2: we slide the filters two pixel at a time.
    • This will produce smaller output volumes spatially.
  • padding
    • we need pad the input volume with zeros around the border.
    • zero padding
  • The CONV layer’s parameters consist of a set of learnable filters.
  • The convolution will compute 32 kernels (features) for each 5x5 patch.
  • Its weight tensor will have a shape of [5, 5, 1, 32].
    • The first two dimensions are the kernel size
    • The third is the number of input channels (here, it is one channel)
    • The last one is the number of kernels (features).
  • A bias vector has a component per kernel.
  • If padding == "SAME": output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides[i])
In [5]:
W_conv1 = tf.Variable(tf.truncated_normal([5, 5, 1, 32], stddev=0.1))
b_conv1 = tf.Variable(tf.constant(0.1, shape=[32]))
h_conv1 = tf.nn.relu(tf.nn.conv2d(x_image, W_conv1, strides=[1, 1, 1, 1], padding='SAME') + b_conv1)
h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
d_h_pool1 = tf.nn.dropout(h_pool1, keep_prob=keep_prob)
print h_conv1
print h_pool1
print d_h_pool1
Tensor("Relu:0", shape=(?, 28, 28, 32), dtype=float32)
Tensor("MaxPool:0", shape=(?, 14, 14, 32), dtype=float32)
Tensor("dropout/mul:0", shape=(?, 14, 14, 32), dtype=float32)
In [6]:
W_conv2 = tf.Variable(tf.truncated_normal([5, 5, 32, 64], stddev=0.1))
b_conv2 = tf.Variable(tf.constant(0.1, shape=[64]))
h_conv2 = tf.nn.relu(tf.nn.conv2d(d_h_pool1, W_conv2, strides=[1, 1, 1, 1], padding='SAME') + b_conv2)
h_pool2 = tf.nn.max_pool(h_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
d_h_pool2 = tf.nn.dropout(h_pool2, keep_prob=keep_prob)
print h_conv2
print h_pool2
print d_h_pool2
Tensor("Relu_1:0", shape=(?, 14, 14, 64), dtype=float32)
Tensor("MaxPool_1:0", shape=(?, 7, 7, 64), dtype=float32)
Tensor("dropout_1/mul:0", shape=(?, 7, 7, 64), dtype=float32)
In [7]:
h_pool2_flat = tf.reshape(d_h_pool2, [-1, 7 * 7 * 64])
In [8]:
W_fc1 = tf.Variable(tf.truncated_normal([7 * 7 * 64, 1024], stddev=0.1))
b_fc1 = tf.Variable(tf.constant(0.1, shape=[1024]))
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
d_h_fc1 = tf.nn.dropout(h_fc1, keep_prob=keep_prob)
In [9]:
W_fc2 = tf.Variable(tf.truncated_normal([1024, 10], stddev=0.1))
b_fc2 = tf.Variable(tf.constant(0.1, shape=[10]))
pred = tf.matmul(d_h_fc1, W_fc2) + b_fc2
In [10]:
error = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
optimizer = tf.train.AdamOptimizer(1e-4).minimize(error)
In [14]:
# Initializing the variables
init = tf.global_variables_initializer()

# Parameters
training_epochs = 50
learning_rate = 0.001
batch_size = 100

# Calculate accuracy with a Test model 
prediction_ground_truth = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(prediction_ground_truth, tf.float32))
In [15]:
def printLossAccuracyForTestData(epoch, sess, test_error_value_list, test_accuracy_list):
    accuracy_value, error_value = sess.run((accuracy, error), 
                                           feed_dict={x: mnist.test.images, y: mnist.test.labels, keep_prob: 0.5})
    test_error_value_list.append(error_value)
    test_accuracy_list.append(accuracy_value)
    print "epoch: %d, test_error_value: %f, test_accuracy: %f" % ( epoch, error_value, accuracy_value )

def drawErrorValues(epoch_list, train_error_value_list, validation_error_value_list, test_error_value_list, test_accuracy_list):
    fig = plt.figure(figsize=(20, 5))
    plt.subplot(121)
    plt.plot(epoch_list, train_error_value_list, 'r', label='Train')
    plt.plot(epoch_list, validation_error_value_list, 'g', label='Validation')
    plt.plot(epoch_list, test_error_value_list, 'b', label='Test')
    plt.ylabel('Total Error')
    plt.xlabel('Epochs')
    plt.grid(True)
    plt.legend(loc='upper right')

    plt.subplot(122)
    plt.plot(epoch_list, test_accuracy_list, 'b', label='Test')
    plt.ylabel('Accuracy')
    plt.xlabel('Epochs')
    plt.yticks(np.arange(min(test_accuracy_list), max(test_accuracy_list), 0.05))
    plt.grid(True)
    plt.legend(loc='lower right')            
    plt.show()

def drawFalsePrediction(sess, numPrintImages):
    ground_truth = sess.run(tf.argmax(y, 1), feed_dict={y: mnist.test.labels})
    prediction = sess.run(tf.argmax(pred, 1), feed_dict={x: mnist.test.images, keep_prob: 0.5})

    fig = plt.figure(figsize=(20, 5))
    j = 1
    for i in range(mnist.test.num_examples):
        if (j > numPrintImages):
            break;
        if (prediction[i] != ground_truth[i]):
            print "Error Index: %s, Prediction: %s, Ground Truth: %s" % (i, prediction[i], ground_truth[i])
            img = np.array(mnist.test.images[i])
            img.shape = (28, 28)
            plt.subplot(1, numPrintImages, j)
            plt.imshow(img)
            j += 1
In [16]:
# Launch the tensorflow graph
with tf.Session() as sess:
    sess.run(init)
    total_batch = int(math.ceil(mnist.train.num_examples/float(batch_size)))
    
    print "total batch: %d" % total_batch

    epoch_list                  = []
    train_error_value_list      = []
    validation_error_value_list = []
    test_error_value_list       = []
    test_accuracy_list          = []
            
    # Training cycle
    for epoch in range(training_epochs):

        # Loop over all batches
        for i in range(total_batch):
            batch_images, batch_labels = mnist.train.next_batch(batch_size)
            _, error_value = sess.run((optimizer, error), feed_dict={x: batch_images, y: batch_labels, keep_prob: 0.5})
        
        epoch_list.append(epoch)

        train_error_value_list.append(error_value)

        v_error_value = sess.run(error, 
                                 feed_dict={x: mnist.validation.images, y: mnist.validation.labels, keep_prob: 0.5})
        validation_error_value_list.append(v_error_value)

        printLossAccuracyForTestData(epoch, sess, test_error_value_list, test_accuracy_list)

    drawErrorValues(epoch_list, train_error_value_list, validation_error_value_list, test_error_value_list, test_accuracy_list)

    drawFalsePrediction(sess, 10)
            
    print("Optimization Finished!")
total batch: 550
epoch: 0, test_error_value: 1.786316, test_accuracy: 0.453600
epoch: 1, test_error_value: 0.958993, test_accuracy: 0.688100
epoch: 2, test_error_value: 0.627309, test_accuracy: 0.795600
epoch: 3, test_error_value: 0.466425, test_accuracy: 0.847700
epoch: 4, test_error_value: 0.367266, test_accuracy: 0.883100
epoch: 5, test_error_value: 0.315511, test_accuracy: 0.899900
epoch: 6, test_error_value: 0.271055, test_accuracy: 0.915000
epoch: 7, test_error_value: 0.220461, test_accuracy: 0.928400
epoch: 8, test_error_value: 0.194227, test_accuracy: 0.938500
epoch: 9, test_error_value: 0.178022, test_accuracy: 0.945100
epoch: 10, test_error_value: 0.164812, test_accuracy: 0.949900
epoch: 11, test_error_value: 0.151278, test_accuracy: 0.950700
epoch: 12, test_error_value: 0.135054, test_accuracy: 0.957600
epoch: 13, test_error_value: 0.123334, test_accuracy: 0.960900
epoch: 14, test_error_value: 0.124456, test_accuracy: 0.959500
epoch: 15, test_error_value: 0.104929, test_accuracy: 0.964500
epoch: 16, test_error_value: 0.103304, test_accuracy: 0.966100
epoch: 17, test_error_value: 0.101153, test_accuracy: 0.967000
epoch: 18, test_error_value: 0.096878, test_accuracy: 0.969700
epoch: 19, test_error_value: 0.087424, test_accuracy: 0.972700
epoch: 20, test_error_value: 0.085922, test_accuracy: 0.971200
epoch: 21, test_error_value: 0.082867, test_accuracy: 0.972800
epoch: 22, test_error_value: 0.070529, test_accuracy: 0.977500
epoch: 23, test_error_value: 0.077665, test_accuracy: 0.976300
epoch: 24, test_error_value: 0.068397, test_accuracy: 0.975400
epoch: 25, test_error_value: 0.065125, test_accuracy: 0.978400
epoch: 26, test_error_value: 0.064642, test_accuracy: 0.979700
epoch: 27, test_error_value: 0.063334, test_accuracy: 0.978600
epoch: 28, test_error_value: 0.059693, test_accuracy: 0.980900
epoch: 29, test_error_value: 0.061158, test_accuracy: 0.979300
epoch: 30, test_error_value: 0.058337, test_accuracy: 0.980200
epoch: 31, test_error_value: 0.060031, test_accuracy: 0.980800
epoch: 32, test_error_value: 0.055434, test_accuracy: 0.981100
epoch: 33, test_error_value: 0.056090, test_accuracy: 0.981600
epoch: 34, test_error_value: 0.057316, test_accuracy: 0.981600
epoch: 35, test_error_value: 0.052082, test_accuracy: 0.983000
epoch: 36, test_error_value: 0.057104, test_accuracy: 0.981600
epoch: 37, test_error_value: 0.053510, test_accuracy: 0.982100
epoch: 38, test_error_value: 0.053915, test_accuracy: 0.982600
epoch: 39, test_error_value: 0.057275, test_accuracy: 0.980700
epoch: 40, test_error_value: 0.047013, test_accuracy: 0.984500
epoch: 41, test_error_value: 0.046121, test_accuracy: 0.984400
epoch: 42, test_error_value: 0.043070, test_accuracy: 0.985000
epoch: 43, test_error_value: 0.050886, test_accuracy: 0.982800
epoch: 44, test_error_value: 0.048252, test_accuracy: 0.984400
epoch: 45, test_error_value: 0.046291, test_accuracy: 0.984200
epoch: 46, test_error_value: 0.047329, test_accuracy: 0.984100
epoch: 47, test_error_value: 0.044261, test_accuracy: 0.985500
epoch: 48, test_error_value: 0.042320, test_accuracy: 0.985600
epoch: 49, test_error_value: 0.042592, test_accuracy: 0.986200
Error Index: 18, Prediction: 8, Ground Truth: 3
Error Index: 62, Prediction: 5, Ground Truth: 9
Error Index: 92, Prediction: 4, Ground Truth: 9
Error Index: 115, Prediction: 9, Ground Truth: 4
Error Index: 247, Prediction: 2, Ground Truth: 4
Error Index: 321, Prediction: 7, Ground Truth: 2
Error Index: 340, Prediction: 3, Ground Truth: 5
Error Index: 358, Prediction: 4, Ground Truth: 7
Error Index: 368, Prediction: 5, Ground Truth: 6
Error Index: 444, Prediction: 3, Ground Truth: 2
Optimization Finished!