#!/usr/bin/env python
# coding: utf-8

# ## Python imports

# In[1]:


from tensorflow.examples.tutorials.mnist import input_data
from keras.utils import to_categorical
from keras import models
from keras import layers
from keras import optimizers
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import classification_report, confusion_matrix


# ## Read MNIST data (download if necessary)

# In[2]:


mnist = input_data.read_data_sets('../data/', one_hot=True)


# ## Verify data shape
# 
# * Training set
# * Validation set
# * Test set

# In[3]:


mnist.train.images.shape, mnist.train.labels.shape


# In[4]:


mnist.validation.images.shape, mnist.validation.labels.shape


# In[5]:


mnist.test.images.shape, mnist.test.labels.shape


# ## Configure neural network
# 
# * 784 input units
# * 15 hidden units with sigmoid activation functions
# * 10 softmax output units

# In[6]:


model = models.Sequential()
model.add(layers.Dense(15, activation = 'sigmoid', input_shape = (28 * 28,)))
model.add(layers.Dense(10, activation = 'softmax'))


# ## Specify loss function and optimizer
# 
# * Categorical cross-entropy loss function
# * RMSprop optimizer (learning rate is adapted for each parameter)

# In[7]:


model.compile(optimizer = optimizers.RMSprop(lr = 0.001),
              loss = 'categorical_crossentropy',
              metrics = ['accuracy'])


# ## Model summary

# In[8]:


model.model.summary()


# ## Train the neural network

# In[9]:


history = model.fit(mnist.train.images,
                    mnist.train.labels,
                    epochs = 20,
                    verbose = 2,
                    batch_size = 512,
                    validation_data = (mnist.validation.images, mnist.validation.labels))


# ## Evaluate model accuracy on test data

# In[10]:


test_loss, test_acc = model.evaluate(mnist.test.images, mnist.test.labels, verbose = 0)
print()
print("Test accuracy:")
test_acc


# ## Compute predicted values and predicted probabilities in test data

# In[11]:


y_pred = model.predict_classes(mnist.test.images, verbose = 0)
y_actual = mnist.test.labels.argmax(1)

digits = [str(i) for i in range(10)]

y_prob = model.predict(mnist.test.images)


# ## Diagnostic measures

# In[12]:


print()
print("Classification report")
print(classification_report(y_actual, y_pred, target_names = digits))


# ## Confusion matrix

# In[13]:


print()
print("Confusion matrix")
print(confusion_matrix(y_actual, y_pred))


# ## Identify test examples where the model is wrong

# In[14]:


errors = np.where(np.not_equal(y_actual, y_pred))
print()
print("Errors")
print(len(errors[0]))
# print(errors)


# ## Utility functions

# In[15]:


def show(image):
    """
    Render a given numpy.uint8 2D array of pixel data.
    """
    from matplotlib import pyplot
    import matplotlib as mpl
    fig = pyplot.figure()
    ax = fig.add_subplot(1,1,1)
    imgplot = ax.imshow(image, cmap=mpl.cm.Greys)
    imgplot.set_interpolation('nearest')
    ax.xaxis.set_ticks_position('top')
    ax.yaxis.set_ticks_position('left')
    pyplot.show()


# In[16]:


def review(idx):
    img = mnist.test.images[idx]
    img = img.reshape((28, 28))
    show(img)
    print("Index: {}".format(idx))
    print("Predicted: {0}, Actual: {1}".format(y_pred[idx], y_actual[idx]))
    print("y_prob: {0}".format(np.array_str(y_prob[idx], precision = 2, suppress_small = True)))


# ## Look at some randomly selected errors

# In[17]:


from random import sample
for i in sample(list(errors[0]), 5):
    review(i)


# In[ ]: