import numpy as np
class SGD:
def __init__(self, lr=0.01):
self.lr = lr
def update(self, params, grads):
for key in params.keys():
params[key] -= self.lr * grads[key]
class Momentum:
def __init__(self, lr=0.01, momentum=0.9):
self.lr = lr
self.momentum = momentum
self.v = None
def update(self, params, grads):
if self.v is None:
self.v = {}
for key, val in params.items():
self.v[key] = np.zeros_like(val)
for key in params.keys():
self.v[key] = self.momentum*self.v[key] - self.lr*grads[key]
params[key] += self.v[key]
class AdaGrad:
def __init__(self, lr=0.01):
self.lr = lr
self.h = None
def update(self, params, grads):
if self.h is None:
self.h = {}
for key, val in params.items():
self.h[key] = np.zeros_like(val)
for key in params.keys():
self.h[key] += grads[key] * grads[key]
params[key] -= self.lr * grads[key] / (np.sqrt(self.h[key]) + 1e-7)
from dataset import mnist
(x_train, t_train), (x_test, t_test) = mnist.load_mnist()
x_train.shape
(60000, 784)
t_train.shape
(60000,)
x_test.shape
(10000, 784)
t_test.shape
(10000,)
validation_rate = 0.20
validation_num = int(x_train.shape[0] * validation_rate)
from common.util import shuffle_dataset
x_train, t_train = shuffle_dataset(x_train, t_train)
x_val = x_train[:validation_num]
t_val = t_train[:validation_num]
x_train = x_train[validation_num:]
t_train = t_train[validation_num:]
x_val.shape
(12000, 784)
t_val.shape
(12000,)
x_train.shape
(48000, 784)
t_train.shape
(48000,)