import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from keras.datasets import mnist
from keras.utils import np_utils

(X_train, y_train), (X_test, y_test) = mnist.load_data()
data = X_train
labels = y_train

fig = plt.figure(figsize=(8, 6))

for i in range(15):
    ax = fig.add_subplot(3, 5, i + 1, xticks=[], yticks=[])
    ax.imshow(X_train[i])

num_pixels = X_train.shape[1] * X_train.shape[2]
data = data.reshape(X_train.shape[0], num_pixels)
X_train = X_train.reshape(X_train.shape[0], num_pixels).astype('float32')
X_test = X_test.reshape(X_test.shape[0], num_pixels).astype('float32')

data = pd.DataFrame(data=data,    # values
                    index=np.arange(data.shape[0]),    # 1st column as index
                    columns=np.arange(data.shape[1]))  # 1st row as the column names
data.head()

X_train = X_train/255
X_test = X_test/255

y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]

sns.countplot(labels)

data.isnull().any().describe()

#corr_matrix = data.corr()

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.preprocessing.image import ImageDataGenerator

def baseline_model(optimizer='adam', param=num_pixels):
	
	model = Sequential()
	model.add(Dense(param, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
	model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
	
	model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
	return model

optimizers = ['adam', 'SGD', 'nadam']
accuracy = list()
history = dict()

for optimizer in optimizers:
    model = baseline_model(optimizer)
    history[optimizer] = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=200, verbose=1)
    scores = model.evaluate(X_test, y_test, verbose=0)
    accuracy.append(scores[1])
    print("Baseline Error: %.2f%%" % (100-scores[1]*100))

for optimizer in optimizers:
    # summarize history for accuracy
    t_train_acc = np.array(history[optimizer].history['acc'])
    std_train_acc = t_train_acc.std()
    t_test_acc = np.array(history[optimizer].history['val_acc'])
    std_test_acc = t_test_acc.std()
    plt.plot(t_train_acc)
    plt.fill_between(np.arange(t_train_acc.shape[0]), t_train_acc+std_train_acc,
                     t_train_acc-std_train_acc, facecolor='blue', alpha=0.5)
    plt.plot(t_test_acc)
    plt.fill_between(np.arange(t_test_acc.shape[0]), t_test_acc+std_test_acc,
                     t_test_acc-std_test_acc, facecolor='green', alpha=0.5)
    plt.title('{} accuracy'.format(optimizer))
    plt.ylabel('accuracy')
    plt.xlabel('epoch')
    plt.legend(['train', 'test'], loc='upper left')
    plt.show()
    
    t_train_loss = np.array(history[optimizer].history['loss'])
    std_train_loss = t_train_loss.std()
    t_test_loss = np.array(history[optimizer].history['val_loss'])
    std_test_loss = t_test_loss.std()
    plt.plot(t_train_loss)
    plt.fill_between(np.arange(t_train_loss.shape[0]), t_train_loss+std_train_loss,
                     t_train_loss-std_train_loss, facecolor='blue', alpha=0.5)
    plt.plot(t_test_loss)
    plt.fill_between(np.arange(t_test_loss.shape[0]), t_test_loss+std_test_loss,
                     t_test_loss-std_test_loss, facecolor='green', alpha=0.5)
    plt.title('{} loss'.format(optimizer))
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train', 'test'], loc='upper left')
    plt.show()

optimizers = ['adam', 'SGD', 'nadam']
params = np.linspace(10, 784, num=5, dtype='Int32')
batch_size_list = np.linspace(10, 300, num=5, dtype='Int32')

accuracy = {optimizer: list() for optimizer in optimizers}

for optimizer in optimizers:
    for param in params:
        for batch_size in batch_size_list:
            model = baseline_model(optimizer, param)
            model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=4, batch_size=batch_size, verbose=1)
            scores = model.evaluate(X_test, y_test, verbose=0)
            accuracy[optimizer].append(scores[1])
            print("Baseline Error: %.2f%%" % (100-scores[1]*100))

from mpl_toolkits.mplot3d import Axes3D

for optimizer in optimizers:
    acc = np.array(accuracy[optimizer]).reshape((5, 5))
    fig = plt.figure()
    ax = Axes3D(fig)
    X = param
    Y = batch_size_list
    X, Y = np.meshgrid(X, Y)
    Z = acc
    
    ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap='CMRmap')

for optimizer in optimizers:
    x = np.array(params)
    acc = np.array(accuracy[optimizer])
    std_acc = acc.std()
    
    plt.plot(x, acc)
    plt.fill_between(x, acc+std_acc,
                     acc-std_acc, facecolor='blue', alpha=0.3)
    plt.title('{} accuracy'.format(optimizer))
    plt.ylabel('accuracy')
    plt.xlabel('param')
    plt.show()

params = np.linspace(1, 50, num=10, dtype='Int32')
optimizer = 'adam'
accuracy = list()

for param in params:
        model = baseline_model(optimizer, param)
        model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=8, batch_size=1000, verbose=1)
        scores = model.evaluate(X_test, y_test, verbose=0)
        accuracy.append(scores[1])
        print("Baseline Error: %.2f%%" % (100-scores[1]*100))

x = np.array(params)
acc = np.array(accuracy)
std_acc = acc.std()

plt.plot(x, acc)
plt.fill_between(x, acc+std_acc,
                 acc-std_acc, facecolor='blue', alpha=0.3)
plt.title('{} accuracy'.format(optimizer))
plt.ylabel('accuracy')
plt.xlabel('param')
plt.show()