Here is implementation of Neural Network from scratch without using any libraries of ML Only numpy is used for NN and matplotlib for plotting the results.
Objective: Objective of this exercise is to understand what difference layers learn, how different activation functions affect the learning rate and importantly what different neurons learn with different activation functions.
Implementation includes following
Optimization: Gradient Decent, Momentum, RMSprop, Adam (RMS+ Momentum)
Regularization: L2 Penalization, Dropouts
Activation Function: Sigmoid, Tanh, Relu, LeakyRelu, Softmax
import numpy as np
import matplotlib.pyplot as plt
%matplotlib notebook
# DL library (code included)
from DeepNet import deepNet
# Toy Datasets (simulated)
import DataSet as ds
# Other datasets
from sklearn import datasets
Data
dtype = ['MOONS','GAUSSIANS','LINEAR','SINUSOIDAL','SPIRAL']
# Moons data
#Training: N=100 examples and no noise
Xr, yr,_ = ds.create_dataset(100, dtype[0],noise=0.0,varargin = 'PRESET');
#Testing: N=100 examples and 10% noise
Xs, ys,_ = ds.create_dataset(100, dtype[0],noise=0.1,varargin = 'PRESET');
print(Xr.shape, yr.shape,Xs.shape, ys.shape)
print('#Features: ',Xr.shape[0])
print('#Examples: ',Xr.shape[1])
Neural Network :: Hidden Layers : [3,4]
NN = deepNet(X=Xr,y=yr,Xts=Xs, yts=ys, Net = [3,4],NetAf =['tanh'], alpha=0.01,
miniBatchSize = 0.3,printCostAt =20,AdamOpt=True,lambd=0,keepProb =[1.0])
Training and plotting
%matplotlib notebook
fig1=plt.figure(1,figsize=(8,4))
fig2=plt.figure(2,figsize=(8,5))
for i in range(20): ## 20 times
NN.fit(itr=10) ## itr=10 iteretion each time
NN.PlotLCurve(pause=0)
fig1.canvas.draw()
NN.PlotBoundries(Layers=True,pause=0)
fig2.canvas.draw()
NN.PlotLCurve()
NN.PlotBoundries(Layers=True)
print(NN)
yri,yrp = NN.predict(Xr)
ysi,ysp = NN.predict(Xs)
print('Training Accuracy ::',100*np.sum(yri==yr)/yri.shape[1])
print('Testing Accuracy ::',100*np.sum(ysi==ys)/ysi.shape[1])
plt.close(fig1)
plt.close(fig2)
Data
dtype = ['MOONS','GAUSSIANS','LINEAR','SINUSOIDAL','SPIRAL']
#Training: N=200 examples and no noise
Xr, yr,_ = ds.create_dataset(200, dtype[3],0.0,varargin = 'PRESET');
#Testing: N=200 examples and 10% noise
Xs, ys,_ = ds.create_dataset(200, dtype[3],0.1,varargin = 'PRESET');
print(Xr.shape, yr.shape,Xs.shape, ys.shape)
print('#Features: ',Xr.shape[0])
print('#Examples: ',Xr.shape[1])
Neural Network :: Hidden Layers : [8,8,5]
NN = deepNet(X=Xr,y=yr,Xts=Xs, yts=ys, Net = [8,8,5],NetAf =['tanh'], alpha=0.01,
miniBatchSize = 0.3, printCostAt =100, AdamOpt=True,lambd=0,keepProb =[1.0])
Training and plotting
%matplotlib notebook
plt.close(fig1)
plt.close(fig2)
fig1=plt.figure(1,figsize=(8,4))
fig2=plt.figure(2,figsize=(8,5))
for i in range(20): ## 20 times
NN.fit(itr=10) ## itr=10 iteretion each time
NN.PlotLCurve(pause=0)
fig1.canvas.draw()
NN.PlotBoundries(Layers=True,pause=0)
fig2.canvas.draw()
NN.PlotLCurve()
NN.PlotBoundries(Layers=True)
print(NN)
yri,yrp = NN.predict(Xr)
ysi,ysp = NN.predict(Xs)
print('Training Accuracy ::',100*np.sum(yri==yr)/yri.shape[1])
print('Testing Accuracy ::',100*np.sum(ysi==ys)/ysi.shape[1])
plt.close(fig1)
plt.close(fig2)
Data (70-30 split)
X, y = ds.mclassGaus(N=500, nClasses = 4,var =0.25,ShowPlot=False)
[n,N] =X.shape
r = np.random.permutation(N)
split =int(0.7*N)
Xr = X[:,r[:split]]
yr = y[:,r[:split]]
Xs = X[:,r[split:]]
ys = y[:,r[split:]]
print(Xr.shape, yr.shape,Xs.shape,ys.shape)
print('#Features: ',Xr.shape[0])
print('#Examples: ',Xr.shape[1])
Neural Network :: Hidden Layers : [8,8,5]
NN = deepNet(X=Xr,y=yr,Xts=Xs, yts=ys, Net = [8,8,5],NetAf =['tanh'], alpha=0.01,
miniBatchSize = 0.3,printCostAt =-1,AdamOpt=True,lambd=0,keepProb =[1.0])
plt.close(fig1)
plt.close(fig2)
fig1=plt.figure(1,figsize=(8,4))
fig2=plt.figure(2,figsize=(8,5))
for i in range(20): ## 20 times
NN.fit(itr=10) ## itr=10 iteretion each time
NN.PlotLCurve(pause=0)
fig1.canvas.draw()
NN.PlotBoundries(Layers=True,pause=0)
fig2.canvas.draw()
NN.PlotLCurve()
NN.PlotBoundries(Layers=True)
print(NN)
yri,yrp = NN.predict(Xr)
ysi,ysp = NN.predict(Xs)
print('Training Accuracy ::',100*np.sum(yri==yr)/yri.shape[1])
print('Testing Accuracy ::',100*np.sum(ysi==ys)/ysi.shape[1])
plt.close(fig1)
plt.close(fig2)
print(Xr.shape, yr.shape,Xs.shape,ys.shape)
print('#Features: ',Xr.shape[0])
print('#Examples: ',Xr.shape[1])
NN = deepNet(X=Xr,y=yr,Xts=Xs, yts=ys, Net = [8,8,5],NetAf =['relu'], alpha=0.01,
miniBatchSize = 0.3,printCostAt =-1,AdamOpt=True,lambd=0,keepProb =[1.0])
plt.close(fig1)
plt.close(fig2)
fig1=plt.figure(1,figsize=(8,4))
fig2=plt.figure(2,figsize=(8,5))
for i in range(20): ## 20 times
NN.fit(itr=10) ## itr=10 iteretion each time
NN.PlotLCurve(pause=0)
fig1.canvas.draw()
NN.PlotBoundries(Layers=True,pause=0)
fig2.canvas.draw()
NN.PlotLCurve()
NN.PlotBoundries(Layers=True)
print(NN)
yri,yrp = NN.predict(Xr)
ysi,ysp = NN.predict(Xs)
print('Training Accuracy ::',100*np.sum(yri==yr)/yri.shape[1])
print('Testing Accuracy ::',100*np.sum(ysi==ys)/ysi.shape[1])
plt.close(fig1)
plt.close(fig2)
Xy= datasets.load_digits()
X = Xy['data']
y = Xy['target']
print(X.shape, y.shape)
fig=plt.figure(1,figsize=(10,1))
for i in range(10):
plt.subplot(1,10,i+1)
plt.imshow(X[i].reshape([8,8]),cmap='gray',aspect='auto')
plt.title('y :' + str(y[i]))
plt.axis('off')
plt.subplots_adjust(top=0.8,wspace=0.12, hspace=0)
plt.show()