## Convolutional Neural Networks¶

In [1]:
import tensorflow as tf
import math
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
import matplotlib.pyplot as plt
%matplotlib inline

Extracting MNIST_data/train-images-idx3-ubyte.gz
Extracting MNIST_data/train-labels-idx1-ubyte.gz
Extracting MNIST_data/t10k-images-idx3-ubyte.gz
Extracting MNIST_data/t10k-labels-idx1-ubyte.gz

In [2]:
x = tf.placeholder("float", [None, 784])
y = tf.placeholder("float", [None, 10])

• Image reshape to 4D [-1, 28, 28, 1]
• The MNIST images have just one gray color value (the number of color channels is one), so that the last value is one.
In [3]:
x_image = tf.reshape(x, [-1, 28, 28, 1])
print x_image.get_shape()

(?, 28, 28, 1)

In [4]:
keep_prob = tf.placeholder(tf.float32)

• stride
• we must specify the stride with which we slide the filter.
• stride = 1: we slide the filters one pixel at a time.
• stride = 2: we slide the filters two pixel at a time.
• This will produce smaller output volumes spatially.
• we need pad the input volume with zeros around the border.
• The CONV layerâ€™s parameters consist of a set of learnable filters.
• The convolution will compute 32 kernels (features) for each 5x5 patch.
• Its weight tensor will have a shape of [5, 5, 1, 32].
• The first two dimensions are the kernel size
• The third is the number of input channels (here, it is one channel)
• The last one is the number of kernels (features).
• A bias vector has a component per kernel.
• If padding == "SAME": output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides[i])
In [5]:
W_conv1 = tf.Variable(tf.truncated_normal([5, 5, 1, 32], stddev=0.1))
b_conv1 = tf.Variable(tf.constant(0.1, shape=[32]))
h_conv1 = tf.nn.relu(tf.nn.conv2d(x_image, W_conv1, strides=[1, 1, 1, 1], padding='SAME') + b_conv1)
h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
d_h_pool1 = tf.nn.dropout(h_pool1, keep_prob=keep_prob)
print h_conv1
print h_pool1
print d_h_pool1

Tensor("Relu:0", shape=(?, 28, 28, 32), dtype=float32)
Tensor("MaxPool:0", shape=(?, 14, 14, 32), dtype=float32)
Tensor("dropout/mul:0", shape=(?, 14, 14, 32), dtype=float32)

In [6]:
W_conv2 = tf.Variable(tf.truncated_normal([5, 5, 32, 64], stddev=0.1))
b_conv2 = tf.Variable(tf.constant(0.1, shape=[64]))
h_conv2 = tf.nn.relu(tf.nn.conv2d(d_h_pool1, W_conv2, strides=[1, 1, 1, 1], padding='SAME') + b_conv2)
h_pool2 = tf.nn.max_pool(h_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
d_h_pool2 = tf.nn.dropout(h_pool2, keep_prob=keep_prob)
print h_conv2
print h_pool2
print d_h_pool2

Tensor("Relu_1:0", shape=(?, 14, 14, 64), dtype=float32)
Tensor("MaxPool_1:0", shape=(?, 7, 7, 64), dtype=float32)
Tensor("dropout_1/mul:0", shape=(?, 7, 7, 64), dtype=float32)

In [7]:
h_pool2_flat = tf.reshape(d_h_pool2, [-1, 7 * 7 * 64])

In [8]:
W_fc1 = tf.Variable(tf.truncated_normal([7 * 7 * 64, 1024], stddev=0.1))
b_fc1 = tf.Variable(tf.constant(0.1, shape=[1024]))
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
d_h_fc1 = tf.nn.dropout(h_fc1, keep_prob=keep_prob)

In [9]:
W_fc2 = tf.Variable(tf.truncated_normal([1024, 10], stddev=0.1))
b_fc2 = tf.Variable(tf.constant(0.1, shape=[10]))
pred = tf.matmul(d_h_fc1, W_fc2) + b_fc2

In [10]:
error = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))

In [14]:
# Initializing the variables
init = tf.global_variables_initializer()

# Parameters
training_epochs = 50
learning_rate = 0.001
batch_size = 100

# Calculate accuracy with a Test model
prediction_ground_truth = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(prediction_ground_truth, tf.float32))

In [15]:
def printLossAccuracyForTestData(epoch, sess, test_error_value_list, test_accuracy_list):
accuracy_value, error_value = sess.run((accuracy, error),
feed_dict={x: mnist.test.images, y: mnist.test.labels, keep_prob: 0.5})
test_error_value_list.append(error_value)
test_accuracy_list.append(accuracy_value)
print "epoch: %d, test_error_value: %f, test_accuracy: %f" % ( epoch, error_value, accuracy_value )

def drawErrorValues(epoch_list, train_error_value_list, validation_error_value_list, test_error_value_list, test_accuracy_list):
fig = plt.figure(figsize=(20, 5))
plt.subplot(121)
plt.plot(epoch_list, train_error_value_list, 'r', label='Train')
plt.plot(epoch_list, validation_error_value_list, 'g', label='Validation')
plt.plot(epoch_list, test_error_value_list, 'b', label='Test')
plt.ylabel('Total Error')
plt.xlabel('Epochs')
plt.grid(True)
plt.legend(loc='upper right')

plt.subplot(122)
plt.plot(epoch_list, test_accuracy_list, 'b', label='Test')
plt.ylabel('Accuracy')
plt.xlabel('Epochs')
plt.yticks(np.arange(min(test_accuracy_list), max(test_accuracy_list), 0.05))
plt.grid(True)
plt.legend(loc='lower right')
plt.show()

def drawFalsePrediction(sess, numPrintImages):
ground_truth = sess.run(tf.argmax(y, 1), feed_dict={y: mnist.test.labels})
prediction = sess.run(tf.argmax(pred, 1), feed_dict={x: mnist.test.images, keep_prob: 0.5})

fig = plt.figure(figsize=(20, 5))
j = 1
for i in range(mnist.test.num_examples):
if (j > numPrintImages):
break;
if (prediction[i] != ground_truth[i]):
print "Error Index: %s, Prediction: %s, Ground Truth: %s" % (i, prediction[i], ground_truth[i])
img = np.array(mnist.test.images[i])
img.shape = (28, 28)
plt.subplot(1, numPrintImages, j)
plt.imshow(img)
j += 1

In [16]:
# Launch the tensorflow graph
with tf.Session() as sess:
sess.run(init)
total_batch = int(math.ceil(mnist.train.num_examples/float(batch_size)))

print "total batch: %d" % total_batch

epoch_list                  = []
train_error_value_list      = []
validation_error_value_list = []
test_error_value_list       = []
test_accuracy_list          = []

# Training cycle
for epoch in range(training_epochs):

# Loop over all batches
for i in range(total_batch):
batch_images, batch_labels = mnist.train.next_batch(batch_size)
_, error_value = sess.run((optimizer, error), feed_dict={x: batch_images, y: batch_labels, keep_prob: 0.5})

epoch_list.append(epoch)

train_error_value_list.append(error_value)

v_error_value = sess.run(error,
feed_dict={x: mnist.validation.images, y: mnist.validation.labels, keep_prob: 0.5})
validation_error_value_list.append(v_error_value)

printLossAccuracyForTestData(epoch, sess, test_error_value_list, test_accuracy_list)

drawErrorValues(epoch_list, train_error_value_list, validation_error_value_list, test_error_value_list, test_accuracy_list)

drawFalsePrediction(sess, 10)

print("Optimization Finished!")

total batch: 550
epoch: 0, test_error_value: 1.786316, test_accuracy: 0.453600
epoch: 1, test_error_value: 0.958993, test_accuracy: 0.688100
epoch: 2, test_error_value: 0.627309, test_accuracy: 0.795600
epoch: 3, test_error_value: 0.466425, test_accuracy: 0.847700
epoch: 4, test_error_value: 0.367266, test_accuracy: 0.883100
epoch: 5, test_error_value: 0.315511, test_accuracy: 0.899900
epoch: 6, test_error_value: 0.271055, test_accuracy: 0.915000
epoch: 7, test_error_value: 0.220461, test_accuracy: 0.928400
epoch: 8, test_error_value: 0.194227, test_accuracy: 0.938500
epoch: 9, test_error_value: 0.178022, test_accuracy: 0.945100
epoch: 10, test_error_value: 0.164812, test_accuracy: 0.949900
epoch: 11, test_error_value: 0.151278, test_accuracy: 0.950700
epoch: 12, test_error_value: 0.135054, test_accuracy: 0.957600
epoch: 13, test_error_value: 0.123334, test_accuracy: 0.960900
epoch: 14, test_error_value: 0.124456, test_accuracy: 0.959500
epoch: 15, test_error_value: 0.104929, test_accuracy: 0.964500
epoch: 16, test_error_value: 0.103304, test_accuracy: 0.966100
epoch: 17, test_error_value: 0.101153, test_accuracy: 0.967000
epoch: 18, test_error_value: 0.096878, test_accuracy: 0.969700
epoch: 19, test_error_value: 0.087424, test_accuracy: 0.972700
epoch: 20, test_error_value: 0.085922, test_accuracy: 0.971200
epoch: 21, test_error_value: 0.082867, test_accuracy: 0.972800
epoch: 22, test_error_value: 0.070529, test_accuracy: 0.977500
epoch: 23, test_error_value: 0.077665, test_accuracy: 0.976300
epoch: 24, test_error_value: 0.068397, test_accuracy: 0.975400
epoch: 25, test_error_value: 0.065125, test_accuracy: 0.978400
epoch: 26, test_error_value: 0.064642, test_accuracy: 0.979700
epoch: 27, test_error_value: 0.063334, test_accuracy: 0.978600
epoch: 28, test_error_value: 0.059693, test_accuracy: 0.980900
epoch: 29, test_error_value: 0.061158, test_accuracy: 0.979300
epoch: 30, test_error_value: 0.058337, test_accuracy: 0.980200
epoch: 31, test_error_value: 0.060031, test_accuracy: 0.980800
epoch: 32, test_error_value: 0.055434, test_accuracy: 0.981100
epoch: 33, test_error_value: 0.056090, test_accuracy: 0.981600
epoch: 34, test_error_value: 0.057316, test_accuracy: 0.981600
epoch: 35, test_error_value: 0.052082, test_accuracy: 0.983000
epoch: 36, test_error_value: 0.057104, test_accuracy: 0.981600
epoch: 37, test_error_value: 0.053510, test_accuracy: 0.982100
epoch: 38, test_error_value: 0.053915, test_accuracy: 0.982600
epoch: 39, test_error_value: 0.057275, test_accuracy: 0.980700
epoch: 40, test_error_value: 0.047013, test_accuracy: 0.984500
epoch: 41, test_error_value: 0.046121, test_accuracy: 0.984400
epoch: 42, test_error_value: 0.043070, test_accuracy: 0.985000
epoch: 43, test_error_value: 0.050886, test_accuracy: 0.982800
epoch: 44, test_error_value: 0.048252, test_accuracy: 0.984400
epoch: 45, test_error_value: 0.046291, test_accuracy: 0.984200
epoch: 46, test_error_value: 0.047329, test_accuracy: 0.984100
epoch: 47, test_error_value: 0.044261, test_accuracy: 0.985500
epoch: 48, test_error_value: 0.042320, test_accuracy: 0.985600
epoch: 49, test_error_value: 0.042592, test_accuracy: 0.986200

Error Index: 18, Prediction: 8, Ground Truth: 3
Error Index: 62, Prediction: 5, Ground Truth: 9
Error Index: 92, Prediction: 4, Ground Truth: 9
Error Index: 115, Prediction: 9, Ground Truth: 4
Error Index: 247, Prediction: 2, Ground Truth: 4
Error Index: 321, Prediction: 7, Ground Truth: 2
Error Index: 340, Prediction: 3, Ground Truth: 5
Error Index: 358, Prediction: 4, Ground Truth: 7
Error Index: 368, Prediction: 5, Ground Truth: 6
Error Index: 444, Prediction: 3, Ground Truth: 2
Optimization Finished!