import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import tensorflow as tf
import random as rn
import os
os.environ['PYTHONHASHSEED'] = '0'
seed = 123
rn.seed(seed)
np.random.seed(seed)
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
from keras import backend as K
tf.set_random_seed(seed)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
train.shape
(42000, 785)
test.shape
(28000, 784)
label = train.label
label.shape
test_index = test.index
train = train.drop(['label'], axis=1)
train.shape
(42000, 784)
train = train.values.reshape(-1,28,28,1) / 255.0
test = test.values.reshape(-1,28,28,1) / 255.0
plt.figure(figsize=(20,2))
n = 10
for i in range(n):
plt.subplot(1,n,i+1)
plt.imshow(train[i,:,:,0], cmap='gray')
plt.axis('off')
from keras.models import Sequential, Input, Model
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import keras
initializer = keras.initializers.glorot_uniform(seed=seed)
model = Sequential()
model.add(Conv2D(32, kernel_size=(5,5),padding='Same', activation='relu', input_shape=(28,28,1), kernel_initializer=initializer))
model.add(Conv2D(32, kernel_size=(5,5),padding='Same', activation='relu',kernel_initializer=initializer))
model.add(MaxPool2D(pool_size=(2,2)))
model.add(BatchNormalization())
model.add(Dropout(0.25, seed=seed))
model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu', kernel_initializer=initializer))
model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu', kernel_initializer=initializer))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
model.add(BatchNormalization())
model.add(Dropout(0.25, seed=seed))
model.add(Flatten())
model.add(Dense(128, activation = "relu", kernel_initializer=initializer))
model.add(BatchNormalization())
model.add(Dropout(0.5, seed=seed))
model.add(Dense(64, activation = "relu", kernel_initializer=initializer))
model.add(BatchNormalization())
model.add(Dropout(0.5, seed=seed))
model.add(Dense(10, activation = "softmax", kernel_initializer=initializer))
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['acc'])
epochs = 100
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, min_lr=0.000001,verbose=1)
early_stopping = EarlyStopping(patience=10, verbose=1)
checkpointer = ModelCheckpoint(filepath='cnn_final_checkpoint.hdf5', verbose=1, save_best_only=True)
hist = model.fit(train, label,
batch_size=32,
epochs=epochs,
verbose=1,
validation_split=0.2,
callbacks=[reduce_lr, early_stopping, checkpointer])
Train on 33600 samples, validate on 8400 samples Epoch 1/100 33600/33600 [==============================] - 14s 405us/step - loss: 0.4613 - acc: 0.8589 - val_loss: 0.0701 - val_acc: 0.9779 Epoch 00001: val_loss improved from inf to 0.07012, saving model to cnn_final_checkpoint.hdf5 Epoch 2/100 33600/33600 [==============================] - 11s 337us/step - loss: 0.1473 - acc: 0.9584 - val_loss: 0.0528 - val_acc: 0.9836 Epoch 00002: val_loss improved from 0.07012 to 0.05284, saving model to cnn_final_checkpoint.hdf5 Epoch 3/100 33600/33600 [==============================] - 11s 338us/step - loss: 0.1070 - acc: 0.9683 - val_loss: 0.0420 - val_acc: 0.9864 Epoch 00003: val_loss improved from 0.05284 to 0.04198, saving model to cnn_final_checkpoint.hdf5 Epoch 4/100 33600/33600 [==============================] - 11s 339us/step - loss: 0.0920 - acc: 0.9738 - val_loss: 0.0491 - val_acc: 0.9849 Epoch 00004: val_loss did not improve from 0.04198 Epoch 5/100 33600/33600 [==============================] - 11s 341us/step - loss: 0.0834 - acc: 0.9760 - val_loss: 0.0522 - val_acc: 0.9850 Epoch 00005: val_loss did not improve from 0.04198 Epoch 6/100 33600/33600 [==============================] - 11s 340us/step - loss: 0.0744 - acc: 0.9793 - val_loss: 0.0396 - val_acc: 0.9898 Epoch 00006: val_loss improved from 0.04198 to 0.03960, saving model to cnn_final_checkpoint.hdf5 Epoch 7/100 33600/33600 [==============================] - 11s 340us/step - loss: 0.0689 - acc: 0.9815 - val_loss: 0.0306 - val_acc: 0.9915 Epoch 00007: val_loss improved from 0.03960 to 0.03064, saving model to cnn_final_checkpoint.hdf5 Epoch 8/100 33600/33600 [==============================] - 11s 342us/step - loss: 0.0601 - acc: 0.9835 - val_loss: 0.0332 - val_acc: 0.9912 Epoch 00008: val_loss did not improve from 0.03064 Epoch 9/100 33600/33600 [==============================] - 11s 341us/step - loss: 0.0566 - acc: 0.9848 - val_loss: 0.0558 - val_acc: 0.9842 Epoch 00009: val_loss did not improve from 0.03064 Epoch 10/100 33600/33600 [==============================] - 11s 341us/step - loss: 0.0542 - acc: 0.9851 - val_loss: 0.0411 - val_acc: 0.9894 Epoch 00010: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026. Epoch 00010: val_loss did not improve from 0.03064 Epoch 11/100 33600/33600 [==============================] - 11s 339us/step - loss: 0.0384 - acc: 0.9902 - val_loss: 0.0242 - val_acc: 0.9939 Epoch 00011: val_loss improved from 0.03064 to 0.02415, saving model to cnn_final_checkpoint.hdf5 Epoch 12/100 33600/33600 [==============================] - 11s 340us/step - loss: 0.0294 - acc: 0.9921 - val_loss: 0.0242 - val_acc: 0.9936 Epoch 00012: val_loss did not improve from 0.02415 Epoch 13/100 33600/33600 [==============================] - 12s 352us/step - loss: 0.0279 - acc: 0.9921 - val_loss: 0.0246 - val_acc: 0.9938 Epoch 00013: val_loss did not improve from 0.02415 Epoch 14/100 33600/33600 [==============================] - 12s 346us/step - loss: 0.0257 - acc: 0.9926 - val_loss: 0.0249 - val_acc: 0.9932 Epoch 00014: ReduceLROnPlateau reducing learning rate to 4.0000001899898055e-05. Epoch 00014: val_loss did not improve from 0.02415 Epoch 15/100 33600/33600 [==============================] - 12s 346us/step - loss: 0.0233 - acc: 0.9936 - val_loss: 0.0224 - val_acc: 0.9942 Epoch 00015: val_loss improved from 0.02415 to 0.02237, saving model to cnn_final_checkpoint.hdf5 Epoch 16/100 33600/33600 [==============================] - 12s 345us/step - loss: 0.0229 - acc: 0.9934 - val_loss: 0.0225 - val_acc: 0.9944 Epoch 00016: val_loss did not improve from 0.02237 Epoch 17/100 33600/33600 [==============================] - 12s 346us/step - loss: 0.0206 - acc: 0.9950 - val_loss: 0.0228 - val_acc: 0.9940 Epoch 00017: val_loss did not improve from 0.02237 Epoch 18/100 33600/33600 [==============================] - 12s 345us/step - loss: 0.0209 - acc: 0.9940 - val_loss: 0.0225 - val_acc: 0.9944 Epoch 00018: ReduceLROnPlateau reducing learning rate to 8.000000525498762e-06. Epoch 00018: val_loss did not improve from 0.02237 Epoch 19/100 33600/33600 [==============================] - 12s 346us/step - loss: 0.0200 - acc: 0.9944 - val_loss: 0.0224 - val_acc: 0.9944 Epoch 00019: val_loss did not improve from 0.02237 Epoch 20/100 33600/33600 [==============================] - 12s 344us/step - loss: 0.0196 - acc: 0.9946 - val_loss: 0.0225 - val_acc: 0.9943 Epoch 00020: val_loss did not improve from 0.02237 Epoch 21/100 33600/33600 [==============================] - 12s 345us/step - loss: 0.0208 - acc: 0.9944 - val_loss: 0.0226 - val_acc: 0.9944 Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.6000001778593287e-06. Epoch 00021: val_loss did not improve from 0.02237 Epoch 22/100 33600/33600 [==============================] - 12s 345us/step - loss: 0.0204 - acc: 0.9947 - val_loss: 0.0226 - val_acc: 0.9943 Epoch 00022: val_loss did not improve from 0.02237 Epoch 23/100 33600/33600 [==============================] - 12s 345us/step - loss: 0.0192 - acc: 0.9946 - val_loss: 0.0225 - val_acc: 0.9944 Epoch 00023: val_loss did not improve from 0.02237 Epoch 24/100 33600/33600 [==============================] - 12s 347us/step - loss: 0.0191 - acc: 0.9954 - val_loss: 0.0227 - val_acc: 0.9942 Epoch 00024: ReduceLROnPlateau reducing learning rate to 1e-06. Epoch 00024: val_loss did not improve from 0.02237 Epoch 25/100 33600/33600 [==============================] - 12s 343us/step - loss: 0.0200 - acc: 0.9943 - val_loss: 0.0225 - val_acc: 0.9944 Epoch 00025: val_loss did not improve from 0.02237 Epoch 00025: early stopping
# Epoch 00024: ReduceLROnPlateau reducing learning rate to 1e-06.
# Epoch 00024: val_loss did not improve from 0.02237
# Epoch 25/100
# 33600/33600 [==============================] - 12s 343us/step - loss: 0.0200 - acc: 0.9943 - val_loss: 0.0225 - val_acc: 0.9944
# Epoch 00025: val_loss did not improve from 0.02237
# Epoch 00025: early stopping
# 0.02674
plt.plot(hist.history['acc'], 'b-', label='acc' )
plt.plot(hist.history['val_acc'], 'g-', label='val_acc' )
plt.legend()
plt.show()
plt.plot(hist.history['loss'], 'r-', label='loss')
plt.plot(hist.history['val_loss'], 'm-', label='val_loss')
plt.legend()
plt.show()
model.load_weights('cnn_final_checkpoint.hdf5')
pred = model.predict(test)
final = pred.argmax(axis=1)
submission = pd.DataFrame({'ImageId': test_index+1, 'Label': final})
submission.to_csv('cnn_submission.csv', index=False)