import pickle
from theano import function, shared, config
import numpy as np
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
rng = np.random
theano_rng = RandomStreams()
%pylab inline
Populating the interactive namespace from numpy and matplotlib
Using gpu device 0: Quadro 4000
## load data
train_set, valid_set, test_set = pickle.load(open('data/mnist.pkl'))
train_x, train_y = train_set
valid_x, valid_y = valid_set
test_x, test_y = test_set
print train_x.shape, train_y.shape
(50000, 784) (50000,)
## draw learned weights as images
def W2Img(W, img_shape, tile_shape):
imgs = W.T
nrows, ncols = tile_shape
fig, axes = subplots(nrows = nrows,
ncols = ncols,
figsize = (ncols * 0.5, nrows * 0.5))
fig.subplots_adjust(hspace = 0, wspace = 0)
axes = axes.flatten()
for i, img in enumerate(imgs):
axes[i].imshow(img.reshape(img_shape))
axes[i].get_xaxis().set_visible(False)
axes[i].get_yaxis().set_visible(False)
shared variables
and access it based on the minibatch index
, given a fixed and known batch size.def share_data(data, return_type = None, borrow = True):
"""
Function that loads the dataset into shared variables,
so that theano can copy it into GPU memory.
Since copying data into GPU is slow, copying a minibatch
everytime would lead to a large decrease in performance.
"""
## explicitly convert dtype to float32, save the data into GPU
shared_data = shared(np.asarray(data,
dtype=config.floatX),
borrow = borrow)
if return_type:
shared_data = T.cast(shared_data, dtype = return_type)
## return a reference to GPU copy, with an explicit casting if needed
return shared_data
early-stopping
. In principle, adding a regularization term to the loss will encougrage smooth network mappings in a neural network, by penalizing large values of the parameters, which decreases the amount of nonlinearity that the network models.class SGDOptimizer(object):
def __init__(self,
n_train_batches,
n_valid_batches,
verbose = True,
patience = None,
patience_increase = 2,
improvement_threshold = 0.995):
self.n_train_batches = n_train_batches
self.n_valid_batches = n_valid_batches
self.verbose = verbose
self.patience = patience or 10 * n_train_batches
self.patience_increase = patience_increase
self.improvement_threshold = improvement_threshold
def optimize(self, n_epochs, train_fn, valid_fn, params):
validation_frequency = min(self.n_train_batches, self.patience / 2)
self.best_params = None
self.best_validation_error = np.inf
epoch = 0
out_of_patience = False
patience = self.patience
while (epoch < n_epochs) and (not out_of_patience):
epoch += 1
for minibatch_index in xrange(n_train_batches):
train_cost = train_fn(minibatch_index)
niter = (epoch - 1) * n_train_batches + minibatch_index
if (niter + 1) % validation_frequency == 0:
#validation_error = np.mean([validate_fn(i)
# for i in xrange(self.n_valid_batches)])
validation_error = valid_fn()
if self.verbose:
print 'epoch %i, minibatch %i/%i, validation error %f, patience %i/%i' % (
epoch, minibatch_index + 1, self.n_train_batches,
validation_error, niter, patience
)
if validation_error < self.best_validation_error:
if validation_error < self.best_validation_error * self.improvement_threshold:
patience = max(self.patience, niter * self.patience_increase)
self.best_validation_error = validation_error
self.best_params = [p.get_value(borrow = True) for p in params]
if patience <= niter:
out_of_patience = True
print 'running out of patience ...'
break
if self.verbose:
print 'optimization done with best validation: %f' % self.best_validation_error
return self.best_params
mini-batches stochastic gradient descent (MSGD)
class LogisticRegression(object):
def __init__(self, n_in, n_out,
X = None, y = None,
W = None, b = None):
"""
params here are everything that you need for calculations.
It should define the relationship graph of all the variables.
So that later functions can be defined by using those variables.
"""
self.n_in = n_in
self.n_out = n_out
self.W = W or shared(value = np.zeros((n_in, n_out),
dtype = config.floatX),
name = 'W', borrow = True)
self.b = b or shared(value = np.zeros((n_out, ),
dtype = config.floatX),
name = 'b', borrow = True)
self.X = X or T.matrix('X')
self.y = y or T.ivector('y')
self.p_y_given_x = T.nnet.softmax(T.dot(self.X, self.W) + self.b)
self.y_hat = T.argmax(self.p_y_given_x, axis = 1)
self.nll = -T.mean(T.log(self.p_y_given_x[T.arange(self.y.shape[0]), self.y]))
self.classification_rate = T.mean(T.eq(self.y_hat, self.y))
def get_train_fn(self, data_x, data_y, batch_size,
learning_rate = 0.13):
"""
bind data to the variables of the formula,
return a function with sig: minibatch_cost cost_f(minibatch_index),
where the cost is calculated
"""
shared_x = share_data(data_x)
shared_y = share_data(data_y, return_type = 'int32')
index = T.lscalar('index')
gw, gb = T.grad(self.nll, wrt = (self.W, self.b))
updates = [(self.W, self.W-gw), (self.b, self.b-gb)]
return function(inputs = [index],
outputs = self.nll,
updates = updates,
givens = {self.X: shared_x[index*batch_size:(index+1)*batch_size],
self.y: shared_y[index*batch_size:(index+1)*batch_size]})
def get_validate_fn(self, data_x, data_y):
shared_x = share_data(data_x)
shared_y = share_data(data_y, return_type = 'int32')
return function(inputs = [],
outputs = self.nll,
givens = {self.X: shared_x,
self.y: shared_y})
def get_score_fn(self, X, y):
shared_x = share_data(X)
shared_y = share_data(y, return_type = 'int32')
return function(inputs = [],
outputs = [self.classification_rate, self.nll],
givens = {self.X: shared_x, self.y: shared_y})
def bind_params(self, W_value, b_value):
self.W.set_value(W_value)
self.b.set_value(b_value)
return self
## test sgd with logistic regression
n_train, n_in = train_x.shape
n_valid, n_in = valid_x.shape
batch_size = 600
n_train_batches = n_train / batch_size
n_valid_batches = n_valid / batch_size
n_out = 10
lr = LogisticRegression(n_in, n_out)
train_fn = lr.get_train_fn(train_x, train_y, batch_size)
validate_fn = lr.get_validate_fn(valid_x, valid_y)
sgd = SGDOptimizer(n_train_batches, n_valid_batches)
optimal_w, optimal_b = sgd.optimize(1000, train_fn, validate_fn, params = (lr.W, lr.b))
epoch 1, minibatch 83/83, validation error 0.344630, patience 82/830 epoch 2, minibatch 83/83, validation error 0.315297, patience 165/830 epoch 3, minibatch 83/83, validation error 0.301831, patience 248/830 epoch 4, minibatch 83/83, validation error 0.293705, patience 331/830 epoch 5, minibatch 83/83, validation error 0.288196, patience 414/830 epoch 6, minibatch 83/83, validation error 0.284188, patience 497/830 epoch 7, minibatch 83/83, validation error 0.281126, patience 580/994 epoch 8, minibatch 83/83, validation error 0.278703, patience 663/1160 epoch 9, minibatch 83/83, validation error 0.276733, patience 746/1326 epoch 10, minibatch 83/83, validation error 0.275097, patience 829/1492 epoch 11, minibatch 83/83, validation error 0.273716, patience 912/1658 epoch 12, minibatch 83/83, validation error 0.272534, patience 995/1824 epoch 13, minibatch 83/83, validation error 0.271510, patience 1078/1824 epoch 14, minibatch 83/83, validation error 0.270615, patience 1161/1824 epoch 15, minibatch 83/83, validation error 0.269826, patience 1244/1824 epoch 16, minibatch 83/83, validation error 0.269127, patience 1327/1824 epoch 17, minibatch 83/83, validation error 0.268502, patience 1410/1824 epoch 18, minibatch 83/83, validation error 0.267943, patience 1493/1824 epoch 19, minibatch 83/83, validation error 0.267438, patience 1576/1824 epoch 20, minibatch 83/83, validation error 0.266982, patience 1659/1824 epoch 21, minibatch 83/83, validation error 0.266569, patience 1742/1824 running out of patience ... optimization done with best validation: 0.266569
lr.bind_params(optimal_w, optimal_b)
score_valid = lr.get_score_fn(valid_x, valid_y)
score_test = lr.get_score_fn(test_x, test_y)
print score_valid()
print score_test()
[array(0.9261), array(0.26656901836395264, dtype=float32)] [array(0.9231), array(0.2750992774963379, dtype=float32)]
tanh
and sigmoid
, such as
$$tanh(a) = \frac{e^a-e^{-a}}{e^a+e^{-a}}$$
$$sigmoid(a) = \frac{1}{1+e^{-a}}$$## drawing tanh and sigmoid
tanh = lambda a: (e**a - e**(-a)) / (e**a + e**(-a))
sigmoid = lambda a: 1. / (1 + e**(-a))
xs = np.arange(-10, 10, 0.5)
figure()
plot(xs, [tanh(x) for x in xs], label='$tanh$', color = 'r')
plot(xs, [sigmoid(x) for x in xs], label = '$sigmoid$', color = 'b')
legend(loc='best')
<matplotlib.legend.Legend at 0x8b40110>
class HiddenLayer(object):
def __init__(self, n_in, n_out,
X = None,
W = None, b = None,
activation = T.tanh):
self.n_in = n_in
self.n_out = n_out
self.X = X or T.matrix('X')
if W:
self.W = W
else:
W_values = (rng.uniform(low = -np.sqrt(6. / (n_in + n_out)),
high = np.sqrt(6. / (n_in + n_out)),
size = (n_in, n_out))
.astype(config.floatX))
if activation == T.nnet.sigmoid:
W_values = W_values * 4
self.W = shared(value = W_values, name = 'hidden_W', borrow = True)
if b:
self.b = b
else:
b_values = np.zeros((n_out, ), dtype=config.floatX)
self.b = shared(value = b_values, name = 'hidden_b', borrow = True)
self.lin_output = T.dot(self.X, self.W) + self.b
self.output = (self.lin_output if activation is None
else activation(self.lin_output))
## mlp implementation
class MLP(object):
def __init__(self, n_in, n_out, n_hidden,
X = None, y = None,
activation = T.tanh):
self.n_in = n_in
self.n_out = n_out
self.n_hidden = n_hidden
self.X = X or T.matrix('X')
self.y = y or T.ivector('y')
self.hidden_layer = HiddenLayer(n_in, n_hidden, self.X)
self.output_layer = LogisticRegression(n_hidden, n_out,
X = self.hidden_layer.output, y = self.y)
self.params = (self.hidden_layer.W, self.hidden_layer.b,
self.output_layer.W, self.output_layer.b)
## norm penalty only on W parts, not b parts
self.L1 = T.sum(abs(self.hidden_layer.W)) + T.sum(abs(self.output_layer.W))
self.L2 = T.sum(self.hidden_layer.W**2) + T.sum(self.output_layer.W**2)
self.nll = self.output_layer.nll
self.classification_rate = self.output_layer.classification_rate
def get_train_fn(self, data_x, data_y,
l1_alpha = 0.001, l2_alpha = 0.0001,
batch_size = 20, learning_rate = 0.01):
shared_x = share_data(data_x)
shared_y = share_data(data_y, return_type = 'int32')
minibatch_index = T.lscalar('index')
cost = self.nll + l1_alpha * self.L1 + l2_alpha * self.L2
gparams = T.grad(cost, wrt = self.params)
updates = [(p, p - learning_rate * gp) for p, gp in zip(self.params, gparams)]
return function(inputs = [minibatch_index],
outputs = cost,
updates = updates,
givens = {
self.X: shared_x[minibatch_index*batch_size:(minibatch_index+1)*batch_size],
self.y: shared_y[minibatch_index*batch_size:(minibatch_index+1)*batch_size]
})
def get_validate_fn(self, data_x, data_y):
shared_x = share_data(data_x)
shared_y = share_data(data_y, return_type = 'int32')
cost = 1 - self.classification_rate
return function(inputs = [],
outputs = cost,
givens = {self.X: shared_x,
self.y: shared_y})
def get_score_fn(self, X, y):
shared_x = share_data(X)
shared_y = share_data(y, return_type = 'int32')
return function(inputs = [],
outputs = [self.classification_rate],
givens = {self.X: shared_x, self.y: shared_y})
def bind_params(self, hid_W, hid_b, out_W, out_b):
self.hidden_layer.W.set_value(hid_W)
self.hidden_layer.b.set_value(hid_b)
self.output_layer.W.set_value(out_W)
self.output_layer.b.set_value(out_b)
return self
## test mlp
n_train, n_in = train_x.shape
n_valid, n_in = valid_x.shape
batch_size = 20
n_train_batches = n_train / batch_size
n_valid_batches = n_valid / batch_size
n_hidden = 500
n_out = 10
## L1/L2 alpha is significant here (smaller the better) - perhaps because of early stopping
mlp = MLP(n_in, n_out, n_hidden)
train_fn = mlp.get_train_fn(train_x, train_y,
batch_size=batch_size, l1_alpha = 0.00, l2_alpha = 0.00)
validate_fn = mlp.get_validate_fn(valid_x, valid_y)
sgd = SGDOptimizer(n_train_batches, n_valid_batches, patience=1000)
optimal_param = sgd.optimize(500, train_fn, validate_fn, params = mlp.params)
epoch 1, minibatch 500/2500, validation error 0.151400, patience 499/1000 epoch 1, minibatch 1000/2500, validation error 0.119500, patience 999/1000 epoch 1, minibatch 1500/2500, validation error 0.106300, patience 1499/1998 epoch 1, minibatch 2000/2500, validation error 0.099700, patience 1999/2998 epoch 1, minibatch 2500/2500, validation error 0.094400, patience 2499/3998 epoch 2, minibatch 500/2500, validation error 0.095500, patience 2999/4998 epoch 2, minibatch 1000/2500, validation error 0.089700, patience 3499/4998 epoch 2, minibatch 1500/2500, validation error 0.088700, patience 3999/6998 epoch 2, minibatch 2000/2500, validation error 0.086700, patience 4499/7998 epoch 2, minibatch 2500/2500, validation error 0.085800, patience 4999/8998 epoch 3, minibatch 500/2500, validation error 0.084900, patience 5499/9998 epoch 3, minibatch 1000/2500, validation error 0.082000, patience 5999/10998 epoch 3, minibatch 1500/2500, validation error 0.081400, patience 6499/11998 epoch 3, minibatch 2000/2500, validation error 0.081600, patience 6999/12998 epoch 3, minibatch 2500/2500, validation error 0.080200, patience 7499/12998 epoch 4, minibatch 500/2500, validation error 0.080200, patience 7999/14998 epoch 4, minibatch 1000/2500, validation error 0.078700, patience 8499/14998 epoch 4, minibatch 1500/2500, validation error 0.078000, patience 8999/16998 epoch 4, minibatch 2000/2500, validation error 0.077800, patience 9499/17998 epoch 4, minibatch 2500/2500, validation error 0.076700, patience 9999/17998 epoch 5, minibatch 500/2500, validation error 0.076400, patience 10499/19998 epoch 5, minibatch 1000/2500, validation error 0.075400, patience 10999/19998 epoch 5, minibatch 1500/2500, validation error 0.074100, patience 11499/21998 epoch 5, minibatch 2000/2500, validation error 0.074700, patience 11999/22998 epoch 5, minibatch 2500/2500, validation error 0.074100, patience 12499/22998 epoch 6, minibatch 500/2500, validation error 0.073400, patience 12999/22998 epoch 6, minibatch 1000/2500, validation error 0.070900, patience 13499/25998 epoch 6, minibatch 1500/2500, validation error 0.071300, patience 13999/26998 epoch 6, minibatch 2000/2500, validation error 0.071800, patience 14499/26998 epoch 6, minibatch 2500/2500, validation error 0.069900, patience 14999/26998 epoch 7, minibatch 500/2500, validation error 0.069800, patience 15499/29998 epoch 7, minibatch 1000/2500, validation error 0.068400, patience 15999/29998 epoch 7, minibatch 1500/2500, validation error 0.068500, patience 16499/31998 epoch 7, minibatch 2000/2500, validation error 0.068100, patience 16999/31998 epoch 7, minibatch 2500/2500, validation error 0.066800, patience 17499/31998 epoch 8, minibatch 500/2500, validation error 0.066700, patience 17999/34998 epoch 8, minibatch 1000/2500, validation error 0.065200, patience 18499/34998 epoch 8, minibatch 1500/2500, validation error 0.064800, patience 18999/36998 epoch 8, minibatch 2000/2500, validation error 0.064700, patience 19499/37998 epoch 8, minibatch 2500/2500, validation error 0.063000, patience 19999/37998 epoch 9, minibatch 500/2500, validation error 0.062600, patience 20499/39998 epoch 9, minibatch 1000/2500, validation error 0.061600, patience 20999/40998 epoch 9, minibatch 1500/2500, validation error 0.062400, patience 21499/41998 epoch 9, minibatch 2000/2500, validation error 0.060600, patience 21999/41998 epoch 9, minibatch 2500/2500, validation error 0.058600, patience 22499/43998 epoch 10, minibatch 500/2500, validation error 0.059300, patience 22999/44998 epoch 10, minibatch 1000/2500, validation error 0.058200, patience 23499/44998 epoch 10, minibatch 1500/2500, validation error 0.058600, patience 23999/46998 epoch 10, minibatch 2000/2500, validation error 0.057100, patience 24499/46998 epoch 10, minibatch 2500/2500, validation error 0.055100, patience 24999/48998 epoch 11, minibatch 500/2500, validation error 0.055200, patience 25499/49998 epoch 11, minibatch 1000/2500, validation error 0.056600, patience 25999/49998 epoch 11, minibatch 1500/2500, validation error 0.054200, patience 26499/49998 epoch 11, minibatch 2000/2500, validation error 0.052900, patience 26999/52998 epoch 11, minibatch 2500/2500, validation error 0.051900, patience 27499/53998 epoch 12, minibatch 500/2500, validation error 0.052100, patience 27999/54998 epoch 12, minibatch 1000/2500, validation error 0.052600, patience 28499/54998 epoch 12, minibatch 1500/2500, validation error 0.051200, patience 28999/54998 epoch 12, minibatch 2000/2500, validation error 0.049900, patience 29499/57998 epoch 12, minibatch 2500/2500, validation error 0.049100, patience 29999/58998 epoch 13, minibatch 500/2500, validation error 0.049600, patience 30499/59998 epoch 13, minibatch 1000/2500, validation error 0.049900, patience 30999/59998 epoch 13, minibatch 1500/2500, validation error 0.048600, patience 31499/59998 epoch 13, minibatch 2000/2500, validation error 0.047900, patience 31999/62998 epoch 13, minibatch 2500/2500, validation error 0.046300, patience 32499/63998 epoch 14, minibatch 500/2500, validation error 0.047700, patience 32999/64998 epoch 14, minibatch 1000/2500, validation error 0.048400, patience 33499/64998 epoch 14, minibatch 1500/2500, validation error 0.046700, patience 33999/64998 epoch 14, minibatch 2000/2500, validation error 0.045500, patience 34499/64998 epoch 14, minibatch 2500/2500, validation error 0.043900, patience 34999/68998 epoch 15, minibatch 500/2500, validation error 0.046000, patience 35499/69998 epoch 15, minibatch 1000/2500, validation error 0.046300, patience 35999/69998 epoch 15, minibatch 1500/2500, validation error 0.045000, patience 36499/69998 epoch 15, minibatch 2000/2500, validation error 0.043600, patience 36999/69998 epoch 15, minibatch 2500/2500, validation error 0.042400, patience 37499/73998 epoch 16, minibatch 500/2500, validation error 0.044900, patience 37999/74998 epoch 16, minibatch 1000/2500, validation error 0.044700, patience 38499/74998 epoch 16, minibatch 1500/2500, validation error 0.044300, patience 38999/74998 epoch 16, minibatch 2000/2500, validation error 0.041600, patience 39499/74998 epoch 16, minibatch 2500/2500, validation error 0.040900, patience 39999/78998 epoch 17, minibatch 500/2500, validation error 0.043200, patience 40499/79998 epoch 17, minibatch 1000/2500, validation error 0.042300, patience 40999/79998 epoch 17, minibatch 1500/2500, validation error 0.042600, patience 41499/79998 epoch 17, minibatch 2000/2500, validation error 0.040000, patience 41999/79998 epoch 17, minibatch 2500/2500, validation error 0.039400, patience 42499/83998 epoch 18, minibatch 500/2500, validation error 0.041200, patience 42999/84998 epoch 18, minibatch 1000/2500, validation error 0.040700, patience 43499/84998 epoch 18, minibatch 1500/2500, validation error 0.041000, patience 43999/84998 epoch 18, minibatch 2000/2500, validation error 0.039200, patience 44499/84998 epoch 18, minibatch 2500/2500, validation error 0.038300, patience 44999/88998 epoch 19, minibatch 500/2500, validation error 0.040200, patience 45499/89998 epoch 19, minibatch 1000/2500, validation error 0.039700, patience 45999/89998 epoch 19, minibatch 1500/2500, validation error 0.040200, patience 46499/89998 epoch 19, minibatch 2000/2500, validation error 0.038400, patience 46999/89998 epoch 19, minibatch 2500/2500, validation error 0.036800, patience 47499/89998 epoch 20, minibatch 500/2500, validation error 0.038300, patience 47999/94998 epoch 20, minibatch 1000/2500, validation error 0.038700, patience 48499/94998 epoch 20, minibatch 1500/2500, validation error 0.038800, patience 48999/94998 epoch 20, minibatch 2000/2500, validation error 0.037700, patience 49499/94998 epoch 20, minibatch 2500/2500, validation error 0.035800, patience 49999/94998 epoch 21, minibatch 500/2500, validation error 0.037100, patience 50499/99998 epoch 21, minibatch 1000/2500, validation error 0.037400, patience 50999/99998 epoch 21, minibatch 1500/2500, validation error 0.037600, patience 51499/99998 epoch 21, minibatch 2000/2500, validation error 0.036800, patience 51999/99998 epoch 21, minibatch 2500/2500, validation error 0.035600, patience 52499/99998 epoch 22, minibatch 500/2500, validation error 0.036100, patience 52999/104998 epoch 22, minibatch 1000/2500, validation error 0.036700, patience 53499/104998 epoch 22, minibatch 1500/2500, validation error 0.036300, patience 53999/104998 epoch 22, minibatch 2000/2500, validation error 0.035500, patience 54499/104998 epoch 22, minibatch 2500/2500, validation error 0.034700, patience 54999/104998 epoch 23, minibatch 500/2500, validation error 0.035400, patience 55499/109998 epoch 23, minibatch 1000/2500, validation error 0.035100, patience 55999/109998 epoch 23, minibatch 1500/2500, validation error 0.035000, patience 56499/109998 epoch 23, minibatch 2000/2500, validation error 0.034100, patience 56999/109998 epoch 23, minibatch 2500/2500, validation error 0.034000, patience 57499/113998 epoch 24, minibatch 500/2500, validation error 0.034500, patience 57999/113998 epoch 24, minibatch 1000/2500, validation error 0.034200, patience 58499/113998 epoch 24, minibatch 1500/2500, validation error 0.034400, patience 58999/113998 epoch 24, minibatch 2000/2500, validation error 0.033700, patience 59499/113998 epoch 24, minibatch 2500/2500, validation error 0.033300, patience 59999/118998 epoch 25, minibatch 500/2500, validation error 0.033300, patience 60499/119998 epoch 25, minibatch 1000/2500, validation error 0.033400, patience 60999/119998 epoch 25, minibatch 1500/2500, validation error 0.033500, patience 61499/119998 epoch 25, minibatch 2000/2500, validation error 0.033100, patience 61999/119998 epoch 25, minibatch 2500/2500, validation error 0.033100, patience 62499/123998 epoch 26, minibatch 500/2500, validation error 0.032700, patience 62999/123998 epoch 26, minibatch 1000/2500, validation error 0.033200, patience 63499/125998 epoch 26, minibatch 1500/2500, validation error 0.032800, patience 63999/125998 epoch 26, minibatch 2000/2500, validation error 0.032700, patience 64499/125998 epoch 26, minibatch 2500/2500, validation error 0.032000, patience 64999/125998 epoch 27, minibatch 500/2500, validation error 0.032400, patience 65499/129998 epoch 27, minibatch 1000/2500, validation error 0.032600, patience 65999/129998 epoch 27, minibatch 1500/2500, validation error 0.031700, patience 66499/129998 epoch 27, minibatch 2000/2500, validation error 0.032000, patience 66999/132998 epoch 27, minibatch 2500/2500, validation error 0.031600, patience 67499/132998 epoch 28, minibatch 500/2500, validation error 0.031700, patience 67999/132998 epoch 28, minibatch 1000/2500, validation error 0.031900, patience 68499/132998 epoch 28, minibatch 1500/2500, validation error 0.030700, patience 68999/132998 epoch 28, minibatch 2000/2500, validation error 0.031200, patience 69499/137998 epoch 28, minibatch 2500/2500, validation error 0.030800, patience 69999/137998 epoch 29, minibatch 500/2500, validation error 0.031300, patience 70499/137998 epoch 29, minibatch 1000/2500, validation error 0.031000, patience 70999/137998 epoch 29, minibatch 1500/2500, validation error 0.030400, patience 71499/137998 epoch 29, minibatch 2000/2500, validation error 0.030900, patience 71999/142998 epoch 29, minibatch 2500/2500, validation error 0.030400, patience 72499/142998 epoch 30, minibatch 500/2500, validation error 0.031200, patience 72999/142998 epoch 30, minibatch 1000/2500, validation error 0.030400, patience 73499/142998 epoch 30, minibatch 1500/2500, validation error 0.030000, patience 73999/142998 epoch 30, minibatch 2000/2500, validation error 0.030100, patience 74499/147998 epoch 30, minibatch 2500/2500, validation error 0.029600, patience 74999/147998 epoch 31, minibatch 500/2500, validation error 0.031000, patience 75499/149998 epoch 31, minibatch 1000/2500, validation error 0.029600, patience 75999/149998 epoch 31, minibatch 1500/2500, validation error 0.029700, patience 76499/149998 epoch 31, minibatch 2000/2500, validation error 0.029600, patience 76999/149998 epoch 31, minibatch 2500/2500, validation error 0.029200, patience 77499/149998 epoch 32, minibatch 500/2500, validation error 0.030500, patience 77999/154998 epoch 32, minibatch 1000/2500, validation error 0.029200, patience 78499/154998 epoch 32, minibatch 1500/2500, validation error 0.029200, patience 78999/154998 epoch 32, minibatch 2000/2500, validation error 0.028900, patience 79499/154998 epoch 32, minibatch 2500/2500, validation error 0.029100, patience 79999/158998 epoch 33, minibatch 500/2500, validation error 0.029900, patience 80499/158998 epoch 33, minibatch 1000/2500, validation error 0.028900, patience 80999/158998 epoch 33, minibatch 1500/2500, validation error 0.028600, patience 81499/158998 epoch 33, minibatch 2000/2500, validation error 0.028600, patience 81999/162998 epoch 33, minibatch 2500/2500, validation error 0.029000, patience 82499/162998 epoch 34, minibatch 500/2500, validation error 0.029300, patience 82999/162998 epoch 34, minibatch 1000/2500, validation error 0.028700, patience 83499/162998 epoch 34, minibatch 1500/2500, validation error 0.027900, patience 83999/162998 epoch 34, minibatch 2000/2500, validation error 0.028600, patience 84499/167998 epoch 34, minibatch 2500/2500, validation error 0.029000, patience 84999/167998 epoch 35, minibatch 500/2500, validation error 0.029200, patience 85499/167998 epoch 35, minibatch 1000/2500, validation error 0.028200, patience 85999/167998 epoch 35, minibatch 1500/2500, validation error 0.027700, patience 86499/167998 epoch 35, minibatch 2000/2500, validation error 0.028700, patience 86999/172998 epoch 35, minibatch 2500/2500, validation error 0.028500, patience 87499/172998 epoch 36, minibatch 500/2500, validation error 0.028600, patience 87999/172998 epoch 36, minibatch 1000/2500, validation error 0.027900, patience 88499/172998 epoch 36, minibatch 1500/2500, validation error 0.027100, patience 88999/172998 epoch 36, minibatch 2000/2500, validation error 0.028400, patience 89499/177998 epoch 36, minibatch 2500/2500, validation error 0.028100, patience 89999/177998 epoch 37, minibatch 500/2500, validation error 0.028200, patience 90499/177998 epoch 37, minibatch 1000/2500, validation error 0.027700, patience 90999/177998 epoch 37, minibatch 1500/2500, validation error 0.027000, patience 91499/177998 epoch 37, minibatch 2000/2500, validation error 0.027900, patience 91999/177998 epoch 37, minibatch 2500/2500, validation error 0.027900, patience 92499/177998 epoch 38, minibatch 500/2500, validation error 0.027900, patience 92999/177998 epoch 38, minibatch 1000/2500, validation error 0.027400, patience 93499/177998 epoch 38, minibatch 1500/2500, validation error 0.026700, patience 93999/177998 epoch 38, minibatch 2000/2500, validation error 0.027900, patience 94499/187998 epoch 38, minibatch 2500/2500, validation error 0.027700, patience 94999/187998 epoch 39, minibatch 500/2500, validation error 0.028000, patience 95499/187998 epoch 39, minibatch 1000/2500, validation error 0.026900, patience 95999/187998 epoch 39, minibatch 1500/2500, validation error 0.026200, patience 96499/187998 epoch 39, minibatch 2000/2500, validation error 0.027700, patience 96999/192998 epoch 39, minibatch 2500/2500, validation error 0.027400, patience 97499/192998 epoch 40, minibatch 500/2500, validation error 0.027900, patience 97999/192998 epoch 40, minibatch 1000/2500, validation error 0.026200, patience 98499/192998 epoch 40, minibatch 1500/2500, validation error 0.026100, patience 98999/192998 epoch 40, minibatch 2000/2500, validation error 0.027800, patience 99499/192998 epoch 40, minibatch 2500/2500, validation error 0.027100, patience 99999/192998 epoch 41, minibatch 500/2500, validation error 0.027200, patience 100499/192998 epoch 41, minibatch 1000/2500, validation error 0.026300, patience 100999/192998 epoch 41, minibatch 1500/2500, validation error 0.025700, patience 101499/192998 epoch 41, minibatch 2000/2500, validation error 0.027100, patience 101999/202998 epoch 41, minibatch 2500/2500, validation error 0.026600, patience 102499/202998 epoch 42, minibatch 500/2500, validation error 0.027000, patience 102999/202998 epoch 42, minibatch 1000/2500, validation error 0.025800, patience 103499/202998 epoch 42, minibatch 1500/2500, validation error 0.025700, patience 103999/202998 epoch 42, minibatch 2000/2500, validation error 0.026900, patience 104499/202998 epoch 42, minibatch 2500/2500, validation error 0.026200, patience 104999/202998 epoch 43, minibatch 500/2500, validation error 0.026500, patience 105499/202998 epoch 43, minibatch 1000/2500, validation error 0.025800, patience 105999/202998 epoch 43, minibatch 1500/2500, validation error 0.025700, patience 106499/202998 epoch 43, minibatch 2000/2500, validation error 0.026600, patience 106999/202998 epoch 43, minibatch 2500/2500, validation error 0.026200, patience 107499/202998 epoch 44, minibatch 500/2500, validation error 0.026300, patience 107999/202998 epoch 44, minibatch 1000/2500, validation error 0.025500, patience 108499/202998 epoch 44, minibatch 1500/2500, validation error 0.025400, patience 108999/216998 epoch 44, minibatch 2000/2500, validation error 0.026200, patience 109499/216998 epoch 44, minibatch 2500/2500, validation error 0.026200, patience 109999/216998 epoch 45, minibatch 500/2500, validation error 0.025700, patience 110499/216998 epoch 45, minibatch 1000/2500, validation error 0.025300, patience 110999/216998 epoch 45, minibatch 1500/2500, validation error 0.025200, patience 111499/216998 epoch 45, minibatch 2000/2500, validation error 0.026000, patience 111999/216998 epoch 45, minibatch 2500/2500, validation error 0.026200, patience 112499/216998 epoch 46, minibatch 500/2500, validation error 0.025600, patience 112999/216998 epoch 46, minibatch 1000/2500, validation error 0.025000, patience 113499/216998 epoch 46, minibatch 1500/2500, validation error 0.025200, patience 113999/226998 epoch 46, minibatch 2000/2500, validation error 0.025800, patience 114499/226998 epoch 46, minibatch 2500/2500, validation error 0.025600, patience 114999/226998 epoch 47, minibatch 500/2500, validation error 0.025600, patience 115499/226998 epoch 47, minibatch 1000/2500, validation error 0.024600, patience 115999/226998 epoch 47, minibatch 1500/2500, validation error 0.025000, patience 116499/231998 epoch 47, minibatch 2000/2500, validation error 0.025600, patience 116999/231998 epoch 47, minibatch 2500/2500, validation error 0.025400, patience 117499/231998 epoch 48, minibatch 500/2500, validation error 0.025300, patience 117999/231998 epoch 48, minibatch 1000/2500, validation error 0.024100, patience 118499/231998 epoch 48, minibatch 1500/2500, validation error 0.025000, patience 118999/236998 epoch 48, minibatch 2000/2500, validation error 0.025400, patience 119499/236998 epoch 48, minibatch 2500/2500, validation error 0.025100, patience 119999/236998 epoch 49, minibatch 500/2500, validation error 0.025000, patience 120499/236998 epoch 49, minibatch 1000/2500, validation error 0.024000, patience 120999/236998 epoch 49, minibatch 1500/2500, validation error 0.024900, patience 121499/236998 epoch 49, minibatch 2000/2500, validation error 0.025300, patience 121999/236998 epoch 49, minibatch 2500/2500, validation error 0.025000, patience 122499/236998 epoch 50, minibatch 500/2500, validation error 0.024600, patience 122999/236998 epoch 50, minibatch 1000/2500, validation error 0.023900, patience 123499/236998 epoch 50, minibatch 1500/2500, validation error 0.024800, patience 123999/236998 epoch 50, minibatch 2000/2500, validation error 0.025100, patience 124499/236998 epoch 50, minibatch 2500/2500, validation error 0.024700, patience 124999/236998 epoch 51, minibatch 500/2500, validation error 0.024200, patience 125499/236998 epoch 51, minibatch 1000/2500, validation error 0.023800, patience 125999/236998 epoch 51, minibatch 1500/2500, validation error 0.024800, patience 126499/236998 epoch 51, minibatch 2000/2500, validation error 0.024900, patience 126999/236998 epoch 51, minibatch 2500/2500, validation error 0.024500, patience 127499/236998 epoch 52, minibatch 500/2500, validation error 0.024000, patience 127999/236998 epoch 52, minibatch 1000/2500, validation error 0.023600, patience 128499/236998 epoch 52, minibatch 1500/2500, validation error 0.024900, patience 128999/256998 epoch 52, minibatch 2000/2500, validation error 0.024800, patience 129499/256998 epoch 52, minibatch 2500/2500, validation error 0.024200, patience 129999/256998 epoch 53, minibatch 500/2500, validation error 0.023800, patience 130499/256998 epoch 53, minibatch 1000/2500, validation error 0.023100, patience 130999/256998 epoch 53, minibatch 1500/2500, validation error 0.024600, patience 131499/261998 epoch 53, minibatch 2000/2500, validation error 0.024400, patience 131999/261998 epoch 53, minibatch 2500/2500, validation error 0.024100, patience 132499/261998 epoch 54, minibatch 500/2500, validation error 0.023400, patience 132999/261998 epoch 54, minibatch 1000/2500, validation error 0.022900, patience 133499/261998 epoch 54, minibatch 1500/2500, validation error 0.024700, patience 133999/266998 epoch 54, minibatch 2000/2500, validation error 0.023800, patience 134499/266998 epoch 54, minibatch 2500/2500, validation error 0.024000, patience 134999/266998 epoch 55, minibatch 500/2500, validation error 0.023300, patience 135499/266998 epoch 55, minibatch 1000/2500, validation error 0.023000, patience 135999/266998 epoch 55, minibatch 1500/2500, validation error 0.024400, patience 136499/266998 epoch 55, minibatch 2000/2500, validation error 0.023600, patience 136999/266998 epoch 55, minibatch 2500/2500, validation error 0.024000, patience 137499/266998 epoch 56, minibatch 500/2500, validation error 0.023300, patience 137999/266998 epoch 56, minibatch 1000/2500, validation error 0.022800, patience 138499/266998 epoch 56, minibatch 1500/2500, validation error 0.024200, patience 138999/266998 epoch 56, minibatch 2000/2500, validation error 0.023800, patience 139499/266998 epoch 56, minibatch 2500/2500, validation error 0.023900, patience 139999/266998 epoch 57, minibatch 500/2500, validation error 0.023100, patience 140499/266998 epoch 57, minibatch 1000/2500, validation error 0.022700, patience 140999/266998 epoch 57, minibatch 1500/2500, validation error 0.024000, patience 141499/266998 epoch 57, minibatch 2000/2500, validation error 0.023500, patience 141999/266998 epoch 57, minibatch 2500/2500, validation error 0.023400, patience 142499/266998 epoch 58, minibatch 500/2500, validation error 0.022800, patience 142999/266998 epoch 58, minibatch 1000/2500, validation error 0.022500, patience 143499/266998 epoch 58, minibatch 1500/2500, validation error 0.024100, patience 143999/286998 epoch 58, minibatch 2000/2500, validation error 0.023400, patience 144499/286998 epoch 58, minibatch 2500/2500, validation error 0.023400, patience 144999/286998 epoch 59, minibatch 500/2500, validation error 0.022800, patience 145499/286998 epoch 59, minibatch 1000/2500, validation error 0.022500, patience 145999/286998 epoch 59, minibatch 1500/2500, validation error 0.023900, patience 146499/286998 epoch 59, minibatch 2000/2500, validation error 0.023400, patience 146999/286998 epoch 59, minibatch 2500/2500, validation error 0.023300, patience 147499/286998 epoch 60, minibatch 500/2500, validation error 0.022700, patience 147999/286998 epoch 60, minibatch 1000/2500, validation error 0.022500, patience 148499/286998 epoch 60, minibatch 1500/2500, validation error 0.023900, patience 148999/286998 epoch 60, minibatch 2000/2500, validation error 0.023300, patience 149499/286998 epoch 60, minibatch 2500/2500, validation error 0.023200, patience 149999/286998 epoch 61, minibatch 500/2500, validation error 0.022500, patience 150499/286998 epoch 61, minibatch 1000/2500, validation error 0.022500, patience 150999/286998 epoch 61, minibatch 1500/2500, validation error 0.023900, patience 151499/286998 epoch 61, minibatch 2000/2500, validation error 0.023100, patience 151999/286998 epoch 61, minibatch 2500/2500, validation error 0.023000, patience 152499/286998 epoch 62, minibatch 500/2500, validation error 0.022400, patience 152999/286998 epoch 62, minibatch 1000/2500, validation error 0.022400, patience 153499/286998 epoch 62, minibatch 1500/2500, validation error 0.023800, patience 153999/286998 epoch 62, minibatch 2000/2500, validation error 0.023100, patience 154499/286998 epoch 62, minibatch 2500/2500, validation error 0.023000, patience 154999/286998 epoch 63, minibatch 500/2500, validation error 0.022400, patience 155499/286998 epoch 63, minibatch 1000/2500, validation error 0.022400, patience 155999/286998 epoch 63, minibatch 1500/2500, validation error 0.023700, patience 156499/286998 epoch 63, minibatch 2000/2500, validation error 0.023100, patience 156999/286998 epoch 63, minibatch 2500/2500, validation error 0.022800, patience 157499/286998 epoch 64, minibatch 500/2500, validation error 0.022300, patience 157999/286998 epoch 64, minibatch 1000/2500, validation error 0.022400, patience 158499/286998 epoch 64, minibatch 1500/2500, validation error 0.023600, patience 158999/286998 epoch 64, minibatch 2000/2500, validation error 0.023000, patience 159499/286998 epoch 64, minibatch 2500/2500, validation error 0.022800, patience 159999/286998 epoch 65, minibatch 500/2500, validation error 0.022300, patience 160499/286998 epoch 65, minibatch 1000/2500, validation error 0.022400, patience 160999/286998 epoch 65, minibatch 1500/2500, validation error 0.023100, patience 161499/286998 epoch 65, minibatch 2000/2500, validation error 0.022700, patience 161999/286998 epoch 65, minibatch 2500/2500, validation error 0.022700, patience 162499/286998 epoch 66, minibatch 500/2500, validation error 0.022200, patience 162999/286998 epoch 66, minibatch 1000/2500, validation error 0.022300, patience 163499/286998 epoch 66, minibatch 1500/2500, validation error 0.023000, patience 163999/286998 epoch 66, minibatch 2000/2500, validation error 0.022600, patience 164499/286998 epoch 66, minibatch 2500/2500, validation error 0.022500, patience 164999/286998 epoch 67, minibatch 500/2500, validation error 0.022100, patience 165499/286998 epoch 67, minibatch 1000/2500, validation error 0.022400, patience 165999/286998 epoch 67, minibatch 1500/2500, validation error 0.023000, patience 166499/286998 epoch 67, minibatch 2000/2500, validation error 0.022600, patience 166999/286998 epoch 67, minibatch 2500/2500, validation error 0.022300, patience 167499/286998 epoch 68, minibatch 500/2500, validation error 0.022000, patience 167999/286998 epoch 68, minibatch 1000/2500, validation error 0.022600, patience 168499/286998 epoch 68, minibatch 1500/2500, validation error 0.022900, patience 168999/286998 epoch 68, minibatch 2000/2500, validation error 0.022600, patience 169499/286998 epoch 68, minibatch 2500/2500, validation error 0.022000, patience 169999/286998 epoch 69, minibatch 500/2500, validation error 0.021800, patience 170499/286998 epoch 69, minibatch 1000/2500, validation error 0.022600, patience 170999/340998 epoch 69, minibatch 1500/2500, validation error 0.022700, patience 171499/340998 epoch 69, minibatch 2000/2500, validation error 0.022600, patience 171999/340998 epoch 69, minibatch 2500/2500, validation error 0.021700, patience 172499/340998 epoch 70, minibatch 500/2500, validation error 0.021300, patience 172999/340998 epoch 70, minibatch 1000/2500, validation error 0.022500, patience 173499/345998 epoch 70, minibatch 1500/2500, validation error 0.022700, patience 173999/345998 epoch 70, minibatch 2000/2500, validation error 0.022400, patience 174499/345998 epoch 70, minibatch 2500/2500, validation error 0.021600, patience 174999/345998 epoch 71, minibatch 500/2500, validation error 0.021200, patience 175499/345998 epoch 71, minibatch 1000/2500, validation error 0.022500, patience 175999/345998 epoch 71, minibatch 1500/2500, validation error 0.022800, patience 176499/345998 epoch 71, minibatch 2000/2500, validation error 0.022300, patience 176999/345998 epoch 71, minibatch 2500/2500, validation error 0.021400, patience 177499/345998 epoch 72, minibatch 500/2500, validation error 0.021300, patience 177999/345998 epoch 72, minibatch 1000/2500, validation error 0.022400, patience 178499/345998 epoch 72, minibatch 1500/2500, validation error 0.022600, patience 178999/345998 epoch 72, minibatch 2000/2500, validation error 0.022300, patience 179499/345998 epoch 72, minibatch 2500/2500, validation error 0.021500, patience 179999/345998 epoch 73, minibatch 500/2500, validation error 0.021000, patience 180499/345998 epoch 73, minibatch 1000/2500, validation error 0.022200, patience 180999/360998 epoch 73, minibatch 1500/2500, validation error 0.022500, patience 181499/360998 epoch 73, minibatch 2000/2500, validation error 0.022200, patience 181999/360998 epoch 73, minibatch 2500/2500, validation error 0.021400, patience 182499/360998 epoch 74, minibatch 500/2500, validation error 0.021000, patience 182999/360998 epoch 74, minibatch 1000/2500, validation error 0.022200, patience 183499/360998 epoch 74, minibatch 1500/2500, validation error 0.022500, patience 183999/360998 epoch 74, minibatch 2000/2500, validation error 0.022200, patience 184499/360998 epoch 74, minibatch 2500/2500, validation error 0.021500, patience 184999/360998 epoch 75, minibatch 500/2500, validation error 0.021000, patience 185499/360998 epoch 75, minibatch 1000/2500, validation error 0.022000, patience 185999/360998 epoch 75, minibatch 1500/2500, validation error 0.022500, patience 186499/360998 epoch 75, minibatch 2000/2500, validation error 0.022000, patience 186999/360998 epoch 75, minibatch 2500/2500, validation error 0.021400, patience 187499/360998 epoch 76, minibatch 500/2500, validation error 0.021000, patience 187999/360998 epoch 76, minibatch 1000/2500, validation error 0.021800, patience 188499/360998 epoch 76, minibatch 1500/2500, validation error 0.022300, patience 188999/360998 epoch 76, minibatch 2000/2500, validation error 0.021700, patience 189499/360998 epoch 76, minibatch 2500/2500, validation error 0.021400, patience 189999/360998 epoch 77, minibatch 500/2500, validation error 0.021000, patience 190499/360998 epoch 77, minibatch 1000/2500, validation error 0.021600, patience 190999/360998 epoch 77, minibatch 1500/2500, validation error 0.022100, patience 191499/360998 epoch 77, minibatch 2000/2500, validation error 0.021400, patience 191999/360998 epoch 77, minibatch 2500/2500, validation error 0.021300, patience 192499/360998 epoch 78, minibatch 500/2500, validation error 0.021100, patience 192999/360998 epoch 78, minibatch 1000/2500, validation error 0.021500, patience 193499/360998 epoch 78, minibatch 1500/2500, validation error 0.022100, patience 193999/360998 epoch 78, minibatch 2000/2500, validation error 0.021300, patience 194499/360998 epoch 78, minibatch 2500/2500, validation error 0.021400, patience 194999/360998 epoch 79, minibatch 500/2500, validation error 0.021100, patience 195499/360998 epoch 79, minibatch 1000/2500, validation error 0.021500, patience 195999/360998 epoch 79, minibatch 1500/2500, validation error 0.022000, patience 196499/360998 epoch 79, minibatch 2000/2500, validation error 0.021200, patience 196999/360998 epoch 79, minibatch 2500/2500, validation error 0.021300, patience 197499/360998 epoch 80, minibatch 500/2500, validation error 0.021100, patience 197999/360998 epoch 80, minibatch 1000/2500, validation error 0.021400, patience 198499/360998 epoch 80, minibatch 1500/2500, validation error 0.021700, patience 198999/360998 epoch 80, minibatch 2000/2500, validation error 0.021200, patience 199499/360998 epoch 80, minibatch 2500/2500, validation error 0.021300, patience 199999/360998 epoch 81, minibatch 500/2500, validation error 0.021100, patience 200499/360998 epoch 81, minibatch 1000/2500, validation error 0.021300, patience 200999/360998 epoch 81, minibatch 1500/2500, validation error 0.021700, patience 201499/360998 epoch 81, minibatch 2000/2500, validation error 0.021200, patience 201999/360998 epoch 81, minibatch 2500/2500, validation error 0.021300, patience 202499/360998 epoch 82, minibatch 500/2500, validation error 0.021000, patience 202999/360998 epoch 82, minibatch 1000/2500, validation error 0.021200, patience 203499/360998 epoch 82, minibatch 1500/2500, validation error 0.021800, patience 203999/360998 epoch 82, minibatch 2000/2500, validation error 0.021200, patience 204499/360998 epoch 82, minibatch 2500/2500, validation error 0.021300, patience 204999/360998 epoch 83, minibatch 500/2500, validation error 0.021000, patience 205499/360998 epoch 83, minibatch 1000/2500, validation error 0.021300, patience 205999/360998 epoch 83, minibatch 1500/2500, validation error 0.021600, patience 206499/360998 epoch 83, minibatch 2000/2500, validation error 0.021100, patience 206999/360998 epoch 83, minibatch 2500/2500, validation error 0.021200, patience 207499/360998 epoch 84, minibatch 500/2500, validation error 0.021000, patience 207999/360998 epoch 84, minibatch 1000/2500, validation error 0.021200, patience 208499/360998 epoch 84, minibatch 1500/2500, validation error 0.021300, patience 208999/360998 epoch 84, minibatch 2000/2500, validation error 0.021100, patience 209499/360998 epoch 84, minibatch 2500/2500, validation error 0.021200, patience 209999/360998 epoch 85, minibatch 500/2500, validation error 0.021000, patience 210499/360998 epoch 85, minibatch 1000/2500, validation error 0.021200, patience 210999/360998 epoch 85, minibatch 1500/2500, validation error 0.021300, patience 211499/360998 epoch 85, minibatch 2000/2500, validation error 0.021100, patience 211999/360998 epoch 85, minibatch 2500/2500, validation error 0.021200, patience 212499/360998 epoch 86, minibatch 500/2500, validation error 0.020900, patience 212999/360998 epoch 86, minibatch 1000/2500, validation error 0.021100, patience 213499/360998 epoch 86, minibatch 1500/2500, validation error 0.021000, patience 213999/360998 epoch 86, minibatch 2000/2500, validation error 0.021000, patience 214499/360998 epoch 86, minibatch 2500/2500, validation error 0.020900, patience 214999/360998 epoch 87, minibatch 500/2500, validation error 0.021000, patience 215499/360998 epoch 87, minibatch 1000/2500, validation error 0.021000, patience 215999/360998 epoch 87, minibatch 1500/2500, validation error 0.020800, patience 216499/360998 epoch 87, minibatch 2000/2500, validation error 0.021000, patience 216999/360998 epoch 87, minibatch 2500/2500, validation error 0.020900, patience 217499/360998 epoch 88, minibatch 500/2500, validation error 0.020900, patience 217999/360998 epoch 88, minibatch 1000/2500, validation error 0.020900, patience 218499/360998 epoch 88, minibatch 1500/2500, validation error 0.020800, patience 218999/360998 epoch 88, minibatch 2000/2500, validation error 0.020900, patience 219499/360998 epoch 88, minibatch 2500/2500, validation error 0.020900, patience 219999/360998 epoch 89, minibatch 500/2500, validation error 0.020800, patience 220499/360998 epoch 89, minibatch 1000/2500, validation error 0.020700, patience 220999/360998 epoch 89, minibatch 1500/2500, validation error 0.020600, patience 221499/360998 epoch 89, minibatch 2000/2500, validation error 0.020900, patience 221999/360998 epoch 89, minibatch 2500/2500, validation error 0.020900, patience 222499/360998 epoch 90, minibatch 500/2500, validation error 0.020800, patience 222999/360998 epoch 90, minibatch 1000/2500, validation error 0.020800, patience 223499/360998 epoch 90, minibatch 1500/2500, validation error 0.020500, patience 223999/360998 epoch 90, minibatch 2000/2500, validation error 0.020800, patience 224499/360998 epoch 90, minibatch 2500/2500, validation error 0.020900, patience 224999/360998 epoch 91, minibatch 500/2500, validation error 0.020800, patience 225499/360998 epoch 91, minibatch 1000/2500, validation error 0.020700, patience 225999/360998 epoch 91, minibatch 1500/2500, validation error 0.020500, patience 226499/360998 epoch 91, minibatch 2000/2500, validation error 0.020700, patience 226999/360998 epoch 91, minibatch 2500/2500, validation error 0.020700, patience 227499/360998 epoch 92, minibatch 500/2500, validation error 0.020700, patience 227999/360998 epoch 92, minibatch 1000/2500, validation error 0.020500, patience 228499/360998 epoch 92, minibatch 1500/2500, validation error 0.020500, patience 228999/360998 epoch 92, minibatch 2000/2500, validation error 0.020700, patience 229499/360998 epoch 92, minibatch 2500/2500, validation error 0.020900, patience 229999/360998 epoch 93, minibatch 500/2500, validation error 0.020700, patience 230499/360998 epoch 93, minibatch 1000/2500, validation error 0.020500, patience 230999/360998 epoch 93, minibatch 1500/2500, validation error 0.020500, patience 231499/360998 epoch 93, minibatch 2000/2500, validation error 0.020700, patience 231999/360998 epoch 93, minibatch 2500/2500, validation error 0.021000, patience 232499/360998 epoch 94, minibatch 500/2500, validation error 0.020600, patience 232999/360998 epoch 94, minibatch 1000/2500, validation error 0.020500, patience 233499/360998 epoch 94, minibatch 1500/2500, validation error 0.020500, patience 233999/360998 epoch 94, minibatch 2000/2500, validation error 0.020800, patience 234499/360998 epoch 94, minibatch 2500/2500, validation error 0.020900, patience 234999/360998 epoch 95, minibatch 500/2500, validation error 0.020500, patience 235499/360998 epoch 95, minibatch 1000/2500, validation error 0.020500, patience 235999/360998 epoch 95, minibatch 1500/2500, validation error 0.020600, patience 236499/360998 epoch 95, minibatch 2000/2500, validation error 0.020800, patience 236999/360998 epoch 95, minibatch 2500/2500, validation error 0.020800, patience 237499/360998 epoch 96, minibatch 500/2500, validation error 0.020500, patience 237999/360998 epoch 96, minibatch 1000/2500, validation error 0.020600, patience 238499/360998 epoch 96, minibatch 1500/2500, validation error 0.020500, patience 238999/360998 epoch 96, minibatch 2000/2500, validation error 0.020800, patience 239499/360998 epoch 96, minibatch 2500/2500, validation error 0.020800, patience 239999/360998 epoch 97, minibatch 500/2500, validation error 0.020400, patience 240499/360998 epoch 97, minibatch 1000/2500, validation error 0.020600, patience 240999/360998 epoch 97, minibatch 1500/2500, validation error 0.020400, patience 241499/360998 epoch 97, minibatch 2000/2500, validation error 0.020700, patience 241999/360998 epoch 97, minibatch 2500/2500, validation error 0.020700, patience 242499/360998 epoch 98, minibatch 500/2500, validation error 0.020400, patience 242999/360998 epoch 98, minibatch 1000/2500, validation error 0.020700, patience 243499/360998 epoch 98, minibatch 1500/2500, validation error 0.020400, patience 243999/360998 epoch 98, minibatch 2000/2500, validation error 0.020800, patience 244499/360998 epoch 98, minibatch 2500/2500, validation error 0.020600, patience 244999/360998 epoch 99, minibatch 500/2500, validation error 0.020400, patience 245499/360998 epoch 99, minibatch 1000/2500, validation error 0.020700, patience 245999/360998 epoch 99, minibatch 1500/2500, validation error 0.020400, patience 246499/360998 epoch 99, minibatch 2000/2500, validation error 0.020700, patience 246999/360998 epoch 99, minibatch 2500/2500, validation error 0.020500, patience 247499/360998 epoch 100, minibatch 500/2500, validation error 0.020400, patience 247999/360998 epoch 100, minibatch 1000/2500, validation error 0.020700, patience 248499/360998 epoch 100, minibatch 1500/2500, validation error 0.020300, patience 248999/360998 epoch 100, minibatch 2000/2500, validation error 0.020700, patience 249499/360998 epoch 100, minibatch 2500/2500, validation error 0.020500, patience 249999/360998 epoch 101, minibatch 500/2500, validation error 0.020500, patience 250499/360998 epoch 101, minibatch 1000/2500, validation error 0.020700, patience 250999/360998 epoch 101, minibatch 1500/2500, validation error 0.020300, patience 251499/360998 epoch 101, minibatch 2000/2500, validation error 0.020500, patience 251999/360998 epoch 101, minibatch 2500/2500, validation error 0.020500, patience 252499/360998 epoch 102, minibatch 500/2500, validation error 0.020400, patience 252999/360998 epoch 102, minibatch 1000/2500, validation error 0.020700, patience 253499/360998 epoch 102, minibatch 1500/2500, validation error 0.020200, patience 253999/360998 epoch 102, minibatch 2000/2500, validation error 0.020500, patience 254499/360998 epoch 102, minibatch 2500/2500, validation error 0.020500, patience 254999/360998 epoch 103, minibatch 500/2500, validation error 0.020400, patience 255499/360998 epoch 103, minibatch 1000/2500, validation error 0.020800, patience 255999/360998 epoch 103, minibatch 1500/2500, validation error 0.020100, patience 256499/360998 epoch 103, minibatch 2000/2500, validation error 0.020300, patience 256999/360998 epoch 103, minibatch 2500/2500, validation error 0.020500, patience 257499/360998 epoch 104, minibatch 500/2500, validation error 0.020400, patience 257999/360998 epoch 104, minibatch 1000/2500, validation error 0.020800, patience 258499/360998 epoch 104, minibatch 1500/2500, validation error 0.020100, patience 258999/360998 epoch 104, minibatch 2000/2500, validation error 0.020300, patience 259499/360998 epoch 104, minibatch 2500/2500, validation error 0.020600, patience 259999/360998 epoch 105, minibatch 500/2500, validation error 0.020300, patience 260499/360998 epoch 105, minibatch 1000/2500, validation error 0.020700, patience 260999/360998 epoch 105, minibatch 1500/2500, validation error 0.020200, patience 261499/360998 epoch 105, minibatch 2000/2500, validation error 0.020100, patience 261999/360998 epoch 105, minibatch 2500/2500, validation error 0.020700, patience 262499/360998 epoch 106, minibatch 500/2500, validation error 0.020400, patience 262999/360998 epoch 106, minibatch 1000/2500, validation error 0.020500, patience 263499/360998 epoch 106, minibatch 1500/2500, validation error 0.020300, patience 263999/360998 epoch 106, minibatch 2000/2500, validation error 0.020100, patience 264499/360998 epoch 106, minibatch 2500/2500, validation error 0.020800, patience 264999/360998 epoch 107, minibatch 500/2500, validation error 0.020500, patience 265499/360998 epoch 107, minibatch 1000/2500, validation error 0.020500, patience 265999/360998 epoch 107, minibatch 1500/2500, validation error 0.020300, patience 266499/360998 epoch 107, minibatch 2000/2500, validation error 0.020100, patience 266999/360998 epoch 107, minibatch 2500/2500, validation error 0.020700, patience 267499/360998 epoch 108, minibatch 500/2500, validation error 0.020400, patience 267999/360998 epoch 108, minibatch 1000/2500, validation error 0.020500, patience 268499/360998 epoch 108, minibatch 1500/2500, validation error 0.020300, patience 268999/360998 epoch 108, minibatch 2000/2500, validation error 0.020000, patience 269499/360998 epoch 108, minibatch 2500/2500, validation error 0.020700, patience 269999/360998 epoch 109, minibatch 500/2500, validation error 0.020300, patience 270499/360998 epoch 109, minibatch 1000/2500, validation error 0.020500, patience 270999/360998 epoch 109, minibatch 1500/2500, validation error 0.020300, patience 271499/360998 epoch 109, minibatch 2000/2500, validation error 0.020100, patience 271999/360998 epoch 109, minibatch 2500/2500, validation error 0.020700, patience 272499/360998 epoch 110, minibatch 500/2500, validation error 0.020300, patience 272999/360998 epoch 110, minibatch 1000/2500, validation error 0.020500, patience 273499/360998 epoch 110, minibatch 1500/2500, validation error 0.020300, patience 273999/360998 epoch 110, minibatch 2000/2500, validation error 0.020000, patience 274499/360998 epoch 110, minibatch 2500/2500, validation error 0.020500, patience 274999/360998 epoch 111, minibatch 500/2500, validation error 0.020100, patience 275499/360998 epoch 111, minibatch 1000/2500, validation error 0.020300, patience 275999/360998 epoch 111, minibatch 1500/2500, validation error 0.020200, patience 276499/360998 epoch 111, minibatch 2000/2500, validation error 0.020000, patience 276999/360998 epoch 111, minibatch 2500/2500, validation error 0.020500, patience 277499/360998 epoch 112, minibatch 500/2500, validation error 0.020100, patience 277999/360998 epoch 112, minibatch 1000/2500, validation error 0.020300, patience 278499/360998 epoch 112, minibatch 1500/2500, validation error 0.020200, patience 278999/360998 epoch 112, minibatch 2000/2500, validation error 0.020000, patience 279499/360998 epoch 112, minibatch 2500/2500, validation error 0.020400, patience 279999/360998 epoch 113, minibatch 500/2500, validation error 0.020100, patience 280499/360998 epoch 113, minibatch 1000/2500, validation error 0.020300, patience 280999/360998 epoch 113, minibatch 1500/2500, validation error 0.020200, patience 281499/360998 epoch 113, minibatch 2000/2500, validation error 0.020000, patience 281999/360998 epoch 113, minibatch 2500/2500, validation error 0.020400, patience 282499/360998 epoch 114, minibatch 500/2500, validation error 0.020200, patience 282999/360998 epoch 114, minibatch 1000/2500, validation error 0.020200, patience 283499/360998 epoch 114, minibatch 1500/2500, validation error 0.020200, patience 283999/360998 epoch 114, minibatch 2000/2500, validation error 0.020100, patience 284499/360998 epoch 114, minibatch 2500/2500, validation error 0.020500, patience 284999/360998 epoch 115, minibatch 500/2500, validation error 0.020300, patience 285499/360998 epoch 115, minibatch 1000/2500, validation error 0.020200, patience 285999/360998 epoch 115, minibatch 1500/2500, validation error 0.020200, patience 286499/360998 epoch 115, minibatch 2000/2500, validation error 0.020000, patience 286999/360998 epoch 115, minibatch 2500/2500, validation error 0.020500, patience 287499/360998 epoch 116, minibatch 500/2500, validation error 0.020300, patience 287999/360998 epoch 116, minibatch 1000/2500, validation error 0.020100, patience 288499/360998 epoch 116, minibatch 1500/2500, validation error 0.020100, patience 288999/360998 epoch 116, minibatch 2000/2500, validation error 0.019900, patience 289499/360998 epoch 116, minibatch 2500/2500, validation error 0.020300, patience 289999/360998 epoch 117, minibatch 500/2500, validation error 0.020300, patience 290499/360998 epoch 117, minibatch 1000/2500, validation error 0.020000, patience 290999/360998 epoch 117, minibatch 1500/2500, validation error 0.019900, patience 291499/360998 epoch 117, minibatch 2000/2500, validation error 0.019900, patience 291999/360998 epoch 117, minibatch 2500/2500, validation error 0.020300, patience 292499/360998 epoch 118, minibatch 500/2500, validation error 0.020300, patience 292999/360998 epoch 118, minibatch 1000/2500, validation error 0.019900, patience 293499/360998 epoch 118, minibatch 1500/2500, validation error 0.019900, patience 293999/360998 epoch 118, minibatch 2000/2500, validation error 0.019900, patience 294499/360998 epoch 118, minibatch 2500/2500, validation error 0.020300, patience 294999/360998 epoch 119, minibatch 500/2500, validation error 0.020300, patience 295499/360998 epoch 119, minibatch 1000/2500, validation error 0.019800, patience 295999/360998 epoch 119, minibatch 1500/2500, validation error 0.020000, patience 296499/591998 epoch 119, minibatch 2000/2500, validation error 0.019800, patience 296999/591998 epoch 119, minibatch 2500/2500, validation error 0.020300, patience 297499/591998 epoch 120, minibatch 500/2500, validation error 0.020100, patience 297999/591998 epoch 120, minibatch 1000/2500, validation error 0.019700, patience 298499/591998 epoch 120, minibatch 1500/2500, validation error 0.020000, patience 298999/596998 epoch 120, minibatch 2000/2500, validation error 0.019900, patience 299499/596998 epoch 120, minibatch 2500/2500, validation error 0.020300, patience 299999/596998 epoch 121, minibatch 500/2500, validation error 0.020100, patience 300499/596998 epoch 121, minibatch 1000/2500, validation error 0.019700, patience 300999/596998 epoch 121, minibatch 1500/2500, validation error 0.019900, patience 301499/596998 epoch 121, minibatch 2000/2500, validation error 0.019800, patience 301999/596998 epoch 121, minibatch 2500/2500, validation error 0.020200, patience 302499/596998 epoch 122, minibatch 500/2500, validation error 0.020100, patience 302999/596998 epoch 122, minibatch 1000/2500, validation error 0.019700, patience 303499/596998 epoch 122, minibatch 1500/2500, validation error 0.019900, patience 303999/596998 epoch 122, minibatch 2000/2500, validation error 0.019800, patience 304499/596998 epoch 122, minibatch 2500/2500, validation error 0.020100, patience 304999/596998 epoch 123, minibatch 500/2500, validation error 0.020100, patience 305499/596998 epoch 123, minibatch 1000/2500, validation error 0.019700, patience 305999/596998 epoch 123, minibatch 1500/2500, validation error 0.019900, patience 306499/596998 epoch 123, minibatch 2000/2500, validation error 0.019800, patience 306999/596998 epoch 123, minibatch 2500/2500, validation error 0.020100, patience 307499/596998 epoch 124, minibatch 500/2500, validation error 0.020000, patience 307999/596998 epoch 124, minibatch 1000/2500, validation error 0.019700, patience 308499/596998 epoch 124, minibatch 1500/2500, validation error 0.019900, patience 308999/596998 epoch 124, minibatch 2000/2500, validation error 0.019800, patience 309499/596998 epoch 124, minibatch 2500/2500, validation error 0.020000, patience 309999/596998 epoch 125, minibatch 500/2500, validation error 0.019800, patience 310499/596998 epoch 125, minibatch 1000/2500, validation error 0.019700, patience 310999/596998 epoch 125, minibatch 1500/2500, validation error 0.019700, patience 311499/596998 epoch 125, minibatch 2000/2500, validation error 0.019800, patience 311999/596998 epoch 125, minibatch 2500/2500, validation error 0.020000, patience 312499/596998 epoch 126, minibatch 500/2500, validation error 0.019800, patience 312999/596998 epoch 126, minibatch 1000/2500, validation error 0.019700, patience 313499/596998 epoch 126, minibatch 1500/2500, validation error 0.019700, patience 313999/596998 epoch 126, minibatch 2000/2500, validation error 0.019800, patience 314499/596998 epoch 126, minibatch 2500/2500, validation error 0.020000, patience 314999/596998 epoch 127, minibatch 500/2500, validation error 0.019900, patience 315499/596998 epoch 127, minibatch 1000/2500, validation error 0.019600, patience 315999/596998 epoch 127, minibatch 1500/2500, validation error 0.019700, patience 316499/631998 epoch 127, minibatch 2000/2500, validation error 0.019800, patience 316999/631998 epoch 127, minibatch 2500/2500, validation error 0.020000, patience 317499/631998 epoch 128, minibatch 500/2500, validation error 0.019800, patience 317999/631998 epoch 128, minibatch 1000/2500, validation error 0.019600, patience 318499/631998 epoch 128, minibatch 1500/2500, validation error 0.019700, patience 318999/631998 epoch 128, minibatch 2000/2500, validation error 0.019800, patience 319499/631998 epoch 128, minibatch 2500/2500, validation error 0.019900, patience 319999/631998 epoch 129, minibatch 500/2500, validation error 0.019700, patience 320499/631998 epoch 129, minibatch 1000/2500, validation error 0.019500, patience 320999/631998 epoch 129, minibatch 1500/2500, validation error 0.019700, patience 321499/641998 epoch 129, minibatch 2000/2500, validation error 0.019600, patience 321999/641998 epoch 129, minibatch 2500/2500, validation error 0.019700, patience 322499/641998 epoch 130, minibatch 500/2500, validation error 0.019600, patience 322999/641998 epoch 130, minibatch 1000/2500, validation error 0.019500, patience 323499/641998 epoch 130, minibatch 1500/2500, validation error 0.019700, patience 323999/641998 epoch 130, minibatch 2000/2500, validation error 0.019600, patience 324499/641998 epoch 130, minibatch 2500/2500, validation error 0.019800, patience 324999/641998 epoch 131, minibatch 500/2500, validation error 0.019500, patience 325499/641998 epoch 131, minibatch 1000/2500, validation error 0.019600, patience 325999/641998 epoch 131, minibatch 1500/2500, validation error 0.019700, patience 326499/641998 epoch 131, minibatch 2000/2500, validation error 0.019500, patience 326999/641998 epoch 131, minibatch 2500/2500, validation error 0.019800, patience 327499/641998 epoch 132, minibatch 500/2500, validation error 0.019500, patience 327999/641998 epoch 132, minibatch 1000/2500, validation error 0.019500, patience 328499/641998 epoch 132, minibatch 1500/2500, validation error 0.019700, patience 328999/641998 epoch 132, minibatch 2000/2500, validation error 0.019500, patience 329499/641998 epoch 132, minibatch 2500/2500, validation error 0.019600, patience 329999/641998 epoch 133, minibatch 500/2500, validation error 0.019500, patience 330499/641998 epoch 133, minibatch 1000/2500, validation error 0.019500, patience 330999/641998 epoch 133, minibatch 1500/2500, validation error 0.019700, patience 331499/641998 epoch 133, minibatch 2000/2500, validation error 0.019500, patience 331999/641998 epoch 133, minibatch 2500/2500, validation error 0.019600, patience 332499/641998 epoch 134, minibatch 500/2500, validation error 0.019500, patience 332999/641998 epoch 134, minibatch 1000/2500, validation error 0.019600, patience 333499/641998 epoch 134, minibatch 1500/2500, validation error 0.019700, patience 333999/641998 epoch 134, minibatch 2000/2500, validation error 0.019500, patience 334499/641998 epoch 134, minibatch 2500/2500, validation error 0.019600, patience 334999/641998 epoch 135, minibatch 500/2500, validation error 0.019500, patience 335499/641998 epoch 135, minibatch 1000/2500, validation error 0.019600, patience 335999/641998 epoch 135, minibatch 1500/2500, validation error 0.019700, patience 336499/641998 epoch 135, minibatch 2000/2500, validation error 0.019500, patience 336999/641998 epoch 135, minibatch 2500/2500, validation error 0.019700, patience 337499/641998 epoch 136, minibatch 500/2500, validation error 0.019500, patience 337999/641998 epoch 136, minibatch 1000/2500, validation error 0.019600, patience 338499/641998 epoch 136, minibatch 1500/2500, validation error 0.019600, patience 338999/641998 epoch 136, minibatch 2000/2500, validation error 0.019500, patience 339499/641998 epoch 136, minibatch 2500/2500, validation error 0.019700, patience 339999/641998 epoch 137, minibatch 500/2500, validation error 0.019600, patience 340499/641998 epoch 137, minibatch 1000/2500, validation error 0.019500, patience 340999/641998 epoch 137, minibatch 1500/2500, validation error 0.019600, patience 341499/641998 epoch 137, minibatch 2000/2500, validation error 0.019500, patience 341999/641998 epoch 137, minibatch 2500/2500, validation error 0.019700, patience 342499/641998 epoch 138, minibatch 500/2500, validation error 0.019600, patience 342999/641998 epoch 138, minibatch 1000/2500, validation error 0.019600, patience 343499/641998 epoch 138, minibatch 1500/2500, validation error 0.019500, patience 343999/641998 epoch 138, minibatch 2000/2500, validation error 0.019500, patience 344499/641998 epoch 138, minibatch 2500/2500, validation error 0.019600, patience 344999/641998 epoch 139, minibatch 500/2500, validation error 0.019500, patience 345499/641998 epoch 139, minibatch 1000/2500, validation error 0.019500, patience 345999/641998 epoch 139, minibatch 1500/2500, validation error 0.019400, patience 346499/641998 epoch 139, minibatch 2000/2500, validation error 0.019600, patience 346999/692998 epoch 139, minibatch 2500/2500, validation error 0.019600, patience 347499/692998 epoch 140, minibatch 500/2500, validation error 0.019600, patience 347999/692998 epoch 140, minibatch 1000/2500, validation error 0.019300, patience 348499/692998 epoch 140, minibatch 1500/2500, validation error 0.019400, patience 348999/696998 epoch 140, minibatch 2000/2500, validation error 0.019600, patience 349499/696998 epoch 140, minibatch 2500/2500, validation error 0.019600, patience 349999/696998 epoch 141, minibatch 500/2500, validation error 0.019500, patience 350499/696998 epoch 141, minibatch 1000/2500, validation error 0.019300, patience 350999/696998 epoch 141, minibatch 1500/2500, validation error 0.019300, patience 351499/696998 epoch 141, minibatch 2000/2500, validation error 0.019600, patience 351999/696998 epoch 141, minibatch 2500/2500, validation error 0.019400, patience 352499/696998 epoch 142, minibatch 500/2500, validation error 0.019500, patience 352999/696998 epoch 142, minibatch 1000/2500, validation error 0.019500, patience 353499/696998 epoch 142, minibatch 1500/2500, validation error 0.019300, patience 353999/696998 epoch 142, minibatch 2000/2500, validation error 0.019400, patience 354499/696998 epoch 142, minibatch 2500/2500, validation error 0.019400, patience 354999/696998 epoch 143, minibatch 500/2500, validation error 0.019400, patience 355499/696998 epoch 143, minibatch 1000/2500, validation error 0.019500, patience 355999/696998 epoch 143, minibatch 1500/2500, validation error 0.019300, patience 356499/696998 epoch 143, minibatch 2000/2500, validation error 0.019300, patience 356999/696998 epoch 143, minibatch 2500/2500, validation error 0.019300, patience 357499/696998 epoch 144, minibatch 500/2500, validation error 0.019400, patience 357999/696998 epoch 144, minibatch 1000/2500, validation error 0.019600, patience 358499/696998 epoch 144, minibatch 1500/2500, validation error 0.019300, patience 358999/696998 epoch 144, minibatch 2000/2500, validation error 0.019300, patience 359499/696998 epoch 144, minibatch 2500/2500, validation error 0.019200, patience 359999/696998
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) <ipython-input-62-babbe2a9733d> in <module>() 15 16 sgd = SGDOptimizer(n_train_batches, n_valid_batches, patience=1000) ---> 17 optimal_param = sgd.optimize(500, train_fn, validate_fn, params = mlp.params) <ipython-input-55-4175d7919bf0> in optimize(self, n_epochs, train_fn, valid_fn, params) 24 epoch += 1 25 for minibatch_index in xrange(n_train_batches): ---> 26 train_cost = train_fn(minibatch_index) 27 niter = (epoch - 1) * n_train_batches + minibatch_index 28 if (niter + 1) % validation_frequency == 0: /usr/local/lib/python2.7/dist-packages/Theano-0.6.0rc3-py2.7.egg/theano/compile/function_module.pyc in __call__(self, *args, **kwargs) 578 t0_fn = time.time() 579 try: --> 580 outputs = self.fn() 581 except Exception: 582 if hasattr(self.fn, 'position_of_error'): KeyboardInterrupt:
mlp.bind_params(*optimal_param)
score_valid = mlp.get_score_fn(valid_x, valid_y)
score_test = mlp.get_score_fn(test_x, test_y)
print score_valid()
print score_test()
--------------------------------------------------------------------------- NameError Traceback (most recent call last) <ipython-input-14-17f5cc79291a> in <module>() ----> 1 mlp.bind_params(*optimal_param) 2 score_valid = mlp.get_score_fn(valid_x, valid_y) 3 score_test = mlp.get_score_fn(test_x, test_y) 4 print score_valid() 5 print score_test() NameError: name 'optimal_param' is not defined
Restricted Boltzmann Machines
, as well as in Denoising Auto-encoders
.class AutoEncoder(object):
def __init__(self, n_visible, n_hidden,
X = None, W= None, b_hid = None,
b_vis = None):
self.n_visible = n_visible
self.n_hidden = n_hidden
self.X = X or T.matrix('X')
if W:
self.W = W
else:
## for sigmoid activation
W_value = rng.uniform(low = -4.*np.sqrt(6./(n_visible+n_hidden)),
high = 4.*np.sqrt(6./(n_visible+n_hidden)),
size = (n_visible, n_hidden)).astype(config.floatX)
self.W = shared(W_value, name = 'W')
self.W_prime = self.W.T
self.b_hid = b_hid or shared(np.zeros(self.n_hidden,
dtype = config.floatX),
name = 'b_hid')
self.b_vis = b_vis or shared(np.zeros(self.n_visible,
dtype = config.floatX),
name = 'b_vis')
self.params = (self.W, self.b_hid, self.b_vis)
## hidden values
self.Y = T.nnet.sigmoid(T.dot(self.X,
self.W)
+ self.b_hid)
## reconstructed values from hidden
self.Z = T.nnet.sigmoid(T.dot(self.Y,
self.W_prime)
+ self.b_vis)
self.cost = T.mean(-T.sum(self.X*T.log(self.Z)
+ (1-self.X)*T.log(1-self.Z),
axis = 1))
def get_train_fn(self, data_X,
batch_size = 20, learning_rate = 0.1):
shared_X = share_data(data_X, borrow=True)
minibatch_index = T.lscalar('index')
gparams = T.grad(self.cost, wrt = self.params)
updates = [(p, p - learning_rate * gp)
for p, gp in zip(self.params, gparams)]
return function(inputs = [minibatch_index],
outputs = self.cost,
updates = updates,
givens = {
self.X: shared_X[minibatch_index*batch_size:(minibatch_index+1)*batch_size]
})
def get_validate_fn(self, data_X):
shared_X = share_data(data_X, borrow=True)
return function(inputs = [],
outputs = self.cost,
givens = {self.X: shared_X})
n_train, n_in = train_x.shape
n_valid, n_in = valid_x.shape
batch_size = 20
n_train_batches = n_train / batch_size
n_valid_batches = n_valid / batch_size
n_hidden = 10 * 10
n_out = 10
ae = AutoEncoder(n_visible = n_in, n_hidden = n_hidden)
train_fn = ae.get_train_fn(train_x, batch_size=batch_size)
validate_fn = ae.get_validate_fn(valid_x)
sgd = SGDOptimizer(n_train_batches, n_valid_batches)
optimal_params = sgd.optimize(1000,
train_fn, validate_fn,
ae.params)
epoch 1, minibatch 2500/2500, validation error 83.062576, patience 2499/25000 epoch 2, minibatch 2500/2500, validation error 77.706955, patience 4999/25000 epoch 3, minibatch 2500/2500, validation error 75.848793, patience 7499/25000 epoch 4, minibatch 2500/2500, validation error 74.945427, patience 9999/25000 epoch 5, minibatch 2500/2500, validation error 74.345901, patience 12499/25000 epoch 6, minibatch 2500/2500, validation error 73.893654, patience 14999/25000 epoch 7, minibatch 2500/2500, validation error 73.519775, patience 17499/29998 epoch 8, minibatch 2500/2500, validation error 73.188400, patience 19999/34998 epoch 9, minibatch 2500/2500, validation error 72.894913, patience 22499/34998 epoch 10, minibatch 2500/2500, validation error 72.629341, patience 24999/34998 epoch 11, minibatch 2500/2500, validation error 72.366455, patience 27499/34998 epoch 12, minibatch 2500/2500, validation error 72.120117, patience 29999/34998 epoch 13, minibatch 2500/2500, validation error 71.911362, patience 32499/34998 running out of patience ... optimization done with best validation: 71.911362
%pylab inline
gray()
W, b_hid, b_vis = optimal_params
W2Img(W, img_shape = (28, 28), tile_shape = (10, 10))
Populating the interactive namespace from numpy and matplotlib
WARNING: pylab import has clobbered these variables: ['tanh'] `%pylab --no-import-all` prevents importing * from pylab and numpy
<matplotlib.figure.Figure at 0xdaea590>
$\tilde{x}$ ~ $q_D(\tilde{x}|x)$
$y = s(W\tilde{x} + b)$
$z = s(W'y + b')$
$L(x, z) = -\sum_{k=1}^d[x_k\log{z_k}+(1-x_k)\log{1-z_k}]$
## denoising autoencoder
class dA(object):
def __init__(self, n_vis, n_hid,
corruption_level = 0.2,
X = None, W = None,
b_hid = None, b_vis = None):
self.n_vis = n_vis
self.n_hid = n_hid
self.corruption_level = corruption_level
self.X = X or T.matrix('X')
if W:
self.W = W
else:
W_value = (rng.uniform(low = -4.*np.sqrt(6./(n_vis+n_hid)),
high = 4.*np.sqrt(6./(n_vis+n_hid)),
size = (n_vis, n_hid))
.astype(config.floatX))
self.W = shared(W_value, name = 'W')
self.W_prime = self.W.T
self.b_vis = b_vis or shared(np.zeros(n_vis,
dtype=config.floatX),
name = 'b_vis')
self.b_hid = b_hid or shared(np.zeros(n_hid,
dtype = config.floatX),
name = 'b_hid')
self.params = (self.W, self.b_hid, self.b_vis)
## corrupted input
self.X_tile = (theano_rng.binomial(size = self.X.shape,
n = 1,
p = 1-self.corruption_level)
* self.X)
## hidden value
self.Y = T.nnet.sigmoid(T.dot(self.X_tile, self.W)
+ self.b_hid)
## reconstructed value
self.Z = T.nnet.sigmoid(T.dot(self.Y, self.W_prime)
+ self.b_vis)
self.cost = T.mean(-T.sum(self.X*T.log(self.Z)
+ (1-self.X)*T.log(1-self.Z),
axis = 1))
def get_train_fn(self, data_X,
batch_size = 20, learning_rate = 0.1):
shared_X = share_data(data_X, borrow = True)
gparams = T.grad(self.cost, wrt = self.params)
updates = [(p, p - learning_rate*gp)
for p, gp in zip(self.params, gparams)]
minibatch_index = T.lscalar('index')
return function(inputs = [minibatch_index],
outputs = self.cost,
updates = updates,
givens = {self.X:
shared_X[minibatch_index*batch_size:(minibatch_index+1)*batch_size]})
def get_validate_fn(self, data_X):
shared_X = share_data(data_X, borrow = True)
return function(inputs = [],
outputs = self.cost,
givens = {self.X: shared_X})
n_train, n_in = train_x.shape
n_valid, n_in = valid_x.shape
batch_size = 20
n_train_batches = n_train / batch_size
n_valid_batches = n_valid / batch_size
n_hidden = 20 * 20
n_out = 10
da = dA(n_in, n_hidden, corruption_level = 0.3)
train_fn = da.get_train_fn(train_x, batch_size=batch_size)
validate_fn = da.get_validate_fn(valid_x)
sgd = SGDOptimizer(n_train_batches, n_valid_batches)
optimal_params = sgd.optimize(1000,
train_fn, validate_fn,
da.params)
epoch 1, minibatch 2500/2500, validation error 76.975506, patience 2499/25000 epoch 2, minibatch 2500/2500, validation error 73.006573, patience 4999/25000 epoch 3, minibatch 2500/2500, validation error 71.342713, patience 7499/25000 epoch 4, minibatch 2500/2500, validation error 69.922022, patience 9999/25000 epoch 5, minibatch 2500/2500, validation error 69.023672, patience 12499/25000 epoch 6, minibatch 2500/2500, validation error 68.464940, patience 14999/25000 epoch 7, minibatch 2500/2500, validation error 68.133056, patience 17499/29998 epoch 8, minibatch 2500/2500, validation error 67.737636, patience 19999/29998 epoch 9, minibatch 2500/2500, validation error 67.435959, patience 22499/39998 epoch 10, minibatch 2500/2500, validation error 67.298334, patience 24999/39998 epoch 11, minibatch 2500/2500, validation error 67.118313, patience 27499/39998 epoch 12, minibatch 2500/2500, validation error 67.114588, patience 29999/39998 epoch 13, minibatch 2500/2500, validation error 66.785414, patience 32499/39998 epoch 14, minibatch 2500/2500, validation error 66.724709, patience 34999/39998 epoch 15, minibatch 2500/2500, validation error 66.617774, patience 37499/39998 running out of patience ... optimization done with best validation: 66.617774
gray()
W, b_hid, b_vis = optimal_params
W2Img(W, img_shape = (28, 28), tile_shape = (20, 20))
<matplotlib.figure.Figure at 0x1068e110>
## stacked denoising autoencoder
class SdA(object):
def __init__(self, n_in, n_hids, n_out,
corruption_levels,
X = None, y = None):
self.n_in = n_in
self.n_hids = n_hids
self.n_out = n_out
self.corruption_levels = corruption_levels
assert len(self.n_hids) == len(self.corruption_levels)
self.X = X or T.matrix('X')
self.y = y or T.ivector('y')
## sigmoid_layers and dA_layers share weights
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(self.n_hids) # number of hidden layers
for i in xrange(self.n_layers):
## construct a sigmoid layer
n_in = self.n_in if i == 0 else self.n_hids[i-1]
n_out = self.n_hids[i]
X = self.X if i == 0 else self.sigmoid_layers[i-1].output
sigmoid_layer = HiddenLayer(n_in,
n_out,
X,
activation=T.nnet.sigmoid)
self.sigmoid_layers.append(sigmoid_layer)
## update param list
self.params.extend([sigmoid_layer.W, sigmoid_layer.b])
## construct a dA layer with shared weights as sigmoid_layer
dA_layer = dA(n_vis = n_in, n_hid = n_out,
corruption_level = self.corruption_levels[i],
X = X, W = sigmoid_layer.W,
b_hid = sigmoid_layer.b, b_vis = None)
self.dA_layers.append(dA_layer)
## logistic regression layer
self.logistic_layer = LogisticRegression(n_in = self.n_hids[-1],
n_out = self.n_out,
X = self.sigmoid_layers[-1].output,
y = self.y)
self.params.extend([self.logistic_layer.W, self.logistic_layer.b])
def get_da_train_fns(self, data_X,
batch_size = 20,
learning_rate = 0.1):
shared_X = share_data(data_X, borrow = True)
train_fns = []
index = T.lscalar('index')
for i in xrange(self.n_layers):
gparams = T.grad(self.dA_layers[i].cost,
self.dA_layers[i].params)
updates = [(p, p-learning_rate*gp)
for p, gp in zip(self.dA_layers[i].params,
gparams)]
train_fn = function(inputs = [index],
outputs = self.dA_layers[i].cost,
updates = updates,
givens = {
self.X: shared_X[index*batch_size:(index+1)*batch_size]
},
name = 'da_train%i'%i)
train_fns.append(train_fn)
return train_fns
def get_da_validate_fns(self, data_X):
shared_X = share_data(data_X, borrow = True)
validate_fns = []
for i in xrange(self.n_layers):
validate_fn = function(inputs = [],
outputs = self.dA_layers[i].cost,
givens = {
self.X: shared_X
},
name = 'da_validate%i'%i)
validate_fns.append(validate_fn)
return validate_fns
def get_mlp_train_fn(self, data_x, data_y,
batch_size = 20, learning_rate = 0.01):
shared_x = share_data(data_x)
shared_y = share_data(data_y, return_type = 'int32')
minibatch_index = T.lscalar('index')
cost = self.logistic_layer.nll
gparams = T.grad(cost, wrt = self.params)
updates = [(p, p - learning_rate * gp)
for p, gp in zip(self.params, gparams)]
return function(inputs = [minibatch_index],
outputs = cost,
updates = updates,
givens = {
self.X: shared_x[minibatch_index*batch_size:(minibatch_index+1)*batch_size],
self.y: shared_y[minibatch_index*batch_size:(minibatch_index+1)*batch_size]
},
name = 'mlp_train')
def get_mlp_validate_fn(self, data_x, data_y):
shared_x = share_data(data_x)
shared_y = share_data(data_y, return_type = 'int32')
cost = 1 - self.logistic_layer.classification_rate
return function(inputs = [],
outputs = cost,
givens = {self.X: shared_x,
self.y: shared_y},
name = 'mlp_validate')
def get_score_fn(self, X, y):
shared_x = share_data(X)
shared_y = share_data(y, return_type = 'int32')
return function(inputs = [],
outputs = [self.logistic_layer.classification_rate],
givens = {self.X: shared_x, self.y: shared_y})
n_train, n_in = train_x.shape
n_valid, n_in = valid_x.shape
batch_size = 20
n_train_batches = n_train / batch_size
n_valid_batches = n_valid / batch_size
n_hids = [400, 400, 400]
corruption_levels = [.1, .2, .3]
n_out = 10
sda = SdA(n_in, n_hids, n_out, corruption_levels)
print 'UNSUPERVISED LEARNING PHASE =========>'
da_train_fns = sda.get_da_train_fns(train_x, batch_size,
learning_rate = 0.001)
da_validate_fns = sda.get_da_validate_fns(valid_x)
for i, (da_train_fn, da_validate_fn) in enumerate(zip(da_train_fns, da_validate_fns)):
print '============= training layer %i ============= ' % i
sgd = SGDOptimizer(n_train_batches, n_valid_batches)
_ = sgd.optimize(15, da_train_fn, da_validate_fn, sda.dA_layers[i].params)
UNSUPERVISED LEARNING PHASE =========> ============= training layer 0 ============= epoch 1, minibatch 2500/2500, validation error 112.133206, patience 2499/25000 epoch 2, minibatch 2500/2500, validation error 99.522734, patience 4999/25000 epoch 3, minibatch 2500/2500, validation error 93.188617, patience 7499/25000 epoch 4, minibatch 2500/2500, validation error 89.121187, patience 9999/25000 epoch 5, minibatch 2500/2500, validation error 86.228872, patience 12499/25000 epoch 6, minibatch 2500/2500, validation error 84.021823, patience 14999/25000 epoch 7, minibatch 2500/2500, validation error 82.242831, patience 17499/29998 epoch 8, minibatch 2500/2500, validation error 80.821314, patience 19999/34998 epoch 9, minibatch 2500/2500, validation error 79.656603, patience 22499/39998 epoch 10, minibatch 2500/2500, validation error 78.570415, patience 24999/44998 epoch 11, minibatch 2500/2500, validation error 77.702440, patience 27499/49998 epoch 12, minibatch 2500/2500, validation error 76.885616, patience 29999/54998 epoch 13, minibatch 2500/2500, validation error 76.122938, patience 32499/59998 epoch 14, minibatch 2500/2500, validation error 75.452613, patience 34999/64998 epoch 15, minibatch 2500/2500, validation error 74.924232, patience 37499/69998 optimization done with best validation: 74.924232 ============= training layer 1 ============= epoch 1, minibatch 2500/2500, validation error 197.377227, patience 2499/25000 epoch 2, minibatch 2500/2500, validation error 186.748840, patience 4999/25000 epoch 3, minibatch 2500/2500, validation error 181.532341, patience 7499/25000 epoch 4, minibatch 2500/2500, validation error 178.265295, patience 9999/25000 epoch 5, minibatch 2500/2500, validation error 175.995613, patience 12499/25000 epoch 6, minibatch 2500/2500, validation error 174.277222, patience 14999/25000 epoch 7, minibatch 2500/2500, validation error 172.909617, patience 17499/29998 epoch 8, minibatch 2500/2500, validation error 171.882283, patience 19999/34998 epoch 9, minibatch 2500/2500, validation error 171.019469, patience 22499/39998 epoch 10, minibatch 2500/2500, validation error 170.304767, patience 24999/44998 epoch 11, minibatch 2500/2500, validation error 169.667782, patience 27499/44998 epoch 12, minibatch 2500/2500, validation error 169.180352, patience 29999/44998 epoch 13, minibatch 2500/2500, validation error 168.732582, patience 32499/44998 epoch 14, minibatch 2500/2500, validation error 168.303973, patience 34999/44998 epoch 15, minibatch 2500/2500, validation error 168.002929, patience 37499/44998 optimization done with best validation: 168.002929 ============= training layer 2 ============= epoch 1, minibatch 2500/2500, validation error 170.098534, patience 2499/25000 epoch 2, minibatch 2500/2500, validation error 159.376585, patience 4999/25000 epoch 3, minibatch 2500/2500, validation error 154.344309, patience 7499/25000 epoch 4, minibatch 2500/2500, validation error 151.361898, patience 9999/25000 epoch 5, minibatch 2500/2500, validation error 149.293972, patience 12499/25000 epoch 6, minibatch 2500/2500, validation error 147.720495, patience 14999/25000 epoch 7, minibatch 2500/2500, validation error 146.626033, patience 17499/29998 epoch 8, minibatch 2500/2500, validation error 145.640845, patience 19999/34998 epoch 9, minibatch 2500/2500, validation error 144.925428, patience 22499/39998 epoch 10, minibatch 2500/2500, validation error 144.269630, patience 24999/39998 epoch 11, minibatch 2500/2500, validation error 143.709638, patience 27499/39998 epoch 12, minibatch 2500/2500, validation error 143.292826, patience 29999/39998 epoch 13, minibatch 2500/2500, validation error 142.838126, patience 32499/39998 epoch 14, minibatch 2500/2500, validation error 142.503986, patience 34999/39998 epoch 15, minibatch 2500/2500, validation error 142.195094, patience 37499/39998 optimization done with best validation: 142.195094
print 'scoring based on unsupervised learning ...'
score_valid = sda.get_score_fn(valid_x, valid_y)
score_test = sda.get_score_fn(test_x, test_y)
print score_valid()
print score_test()
scoring based on unsupervised learning ... [array(0.0991)] [array(0.098)]
n_train, n_in = train_x.shape
n_valid, n_in = valid_x.shape
batch_size = 50
n_train_batches = n_train / batch_size
n_valid_batches = n_valid / batch_size
print 'SUPERVISED LEARNING PHASE =========>'
mlp_train_fn = sda.get_mlp_train_fn(train_x, train_y,
batch_size,
learning_rate = 0.1)
mlp_validate_fn = sda.get_mlp_validate_fn(valid_x, valid_y)
sgd = SGDOptimizer(n_train_batches, n_valid_batches)
_ = sgd.optimize(300, train_fn = mlp_train_fn, valid_fn = mlp_validate_fn, params = sda.params)
SUPERVISED LEARNING PHASE =========> epoch 1, minibatch 1000/1000, validation error 0.074700, patience 999/10000 epoch 2, minibatch 1000/1000, validation error 0.059800, patience 1999/10000 epoch 3, minibatch 1000/1000, validation error 0.051600, patience 2999/10000 epoch 4, minibatch 1000/1000, validation error 0.045900, patience 3999/10000 epoch 5, minibatch 1000/1000, validation error 0.041900, patience 4999/10000 epoch 6, minibatch 1000/1000, validation error 0.039400, patience 5999/10000 epoch 7, minibatch 1000/1000, validation error 0.037100, patience 6999/11998 epoch 8, minibatch 1000/1000, validation error 0.034900, patience 7999/13998 epoch 9, minibatch 1000/1000, validation error 0.033200, patience 8999/15998 epoch 10, minibatch 1000/1000, validation error 0.032000, patience 9999/17998 epoch 11, minibatch 1000/1000, validation error 0.031000, patience 10999/19998 epoch 12, minibatch 1000/1000, validation error 0.030400, patience 11999/21998 epoch 13, minibatch 1000/1000, validation error 0.029400, patience 12999/23998 epoch 14, minibatch 1000/1000, validation error 0.028300, patience 13999/25998 epoch 15, minibatch 1000/1000, validation error 0.027900, patience 14999/27998 epoch 16, minibatch 1000/1000, validation error 0.027400, patience 15999/29998 epoch 17, minibatch 1000/1000, validation error 0.026900, patience 16999/31998 epoch 18, minibatch 1000/1000, validation error 0.026500, patience 17999/33998 epoch 19, minibatch 1000/1000, validation error 0.025900, patience 18999/35998 epoch 20, minibatch 1000/1000, validation error 0.025500, patience 19999/37998 epoch 21, minibatch 1000/1000, validation error 0.024600, patience 20999/39998 epoch 22, minibatch 1000/1000, validation error 0.024200, patience 21999/41998 epoch 23, minibatch 1000/1000, validation error 0.023500, patience 22999/43998 epoch 24, minibatch 1000/1000, validation error 0.023500, patience 23999/45998 epoch 25, minibatch 1000/1000, validation error 0.023200, patience 24999/45998 epoch 26, minibatch 1000/1000, validation error 0.023200, patience 25999/49998 epoch 27, minibatch 1000/1000, validation error 0.023200, patience 26999/49998 epoch 28, minibatch 1000/1000, validation error 0.023200, patience 27999/49998 epoch 29, minibatch 1000/1000, validation error 0.023000, patience 28999/49998 epoch 30, minibatch 1000/1000, validation error 0.022800, patience 29999/57998 epoch 31, minibatch 1000/1000, validation error 0.022900, patience 30999/59998 epoch 32, minibatch 1000/1000, validation error 0.022600, patience 31999/59998 epoch 33, minibatch 1000/1000, validation error 0.022400, patience 32999/63998 epoch 34, minibatch 1000/1000, validation error 0.022500, patience 33999/65998 epoch 35, minibatch 1000/1000, validation error 0.022200, patience 34999/65998 epoch 36, minibatch 1000/1000, validation error 0.022000, patience 35999/69998 epoch 37, minibatch 1000/1000, validation error 0.021900, patience 36999/71998 epoch 38, minibatch 1000/1000, validation error 0.021900, patience 37999/71998 epoch 39, minibatch 1000/1000, validation error 0.021700, patience 38999/71998 epoch 40, minibatch 1000/1000, validation error 0.021600, patience 39999/77998 epoch 41, minibatch 1000/1000, validation error 0.021600, patience 40999/77998 epoch 42, minibatch 1000/1000, validation error 0.021500, patience 41999/77998 epoch 43, minibatch 1000/1000, validation error 0.021300, patience 42999/77998 epoch 44, minibatch 1000/1000, validation error 0.021400, patience 43999/85998 epoch 45, minibatch 1000/1000, validation error 0.021500, patience 44999/85998 epoch 46, minibatch 1000/1000, validation error 0.021400, patience 45999/85998 epoch 47, minibatch 1000/1000, validation error 0.021200, patience 46999/85998 epoch 48, minibatch 1000/1000, validation error 0.021000, patience 47999/85998 epoch 49, minibatch 1000/1000, validation error 0.020900, patience 48999/95998 epoch 50, minibatch 1000/1000, validation error 0.021000, patience 49999/95998 epoch 51, minibatch 1000/1000, validation error 0.020900, patience 50999/95998 epoch 52, minibatch 1000/1000, validation error 0.020700, patience 51999/95998 epoch 53, minibatch 1000/1000, validation error 0.020700, patience 52999/103998 epoch 54, minibatch 1000/1000, validation error 0.020500, patience 53999/103998 epoch 55, minibatch 1000/1000, validation error 0.020100, patience 54999/107998 epoch 56, minibatch 1000/1000, validation error 0.020100, patience 55999/109998 epoch 57, minibatch 1000/1000, validation error 0.020400, patience 56999/109998 epoch 58, minibatch 1000/1000, validation error 0.020300, patience 57999/109998 epoch 59, minibatch 1000/1000, validation error 0.020400, patience 58999/109998 epoch 60, minibatch 1000/1000, validation error 0.020200, patience 59999/109998 epoch 61, minibatch 1000/1000, validation error 0.020100, patience 60999/109998 epoch 62, minibatch 1000/1000, validation error 0.020000, patience 61999/109998 epoch 63, minibatch 1000/1000, validation error 0.020200, patience 62999/109998 epoch 64, minibatch 1000/1000, validation error 0.020200, patience 63999/109998 epoch 65, minibatch 1000/1000, validation error 0.020100, patience 64999/109998 epoch 66, minibatch 1000/1000, validation error 0.020100, patience 65999/109998 epoch 67, minibatch 1000/1000, validation error 0.020100, patience 66999/109998 epoch 68, minibatch 1000/1000, validation error 0.020100, patience 67999/109998 epoch 69, minibatch 1000/1000, validation error 0.020200, patience 68999/109998 epoch 70, minibatch 1000/1000, validation error 0.020200, patience 69999/109998 epoch 71, minibatch 1000/1000, validation error 0.020200, patience 70999/109998 epoch 72, minibatch 1000/1000, validation error 0.020100, patience 71999/109998 epoch 73, minibatch 1000/1000, validation error 0.020100, patience 72999/109998 epoch 74, minibatch 1000/1000, validation error 0.020200, patience 73999/109998 epoch 75, minibatch 1000/1000, validation error 0.020200, patience 74999/109998 epoch 76, minibatch 1000/1000, validation error 0.020200, patience 75999/109998 epoch 77, minibatch 1000/1000, validation error 0.020200, patience 76999/109998 epoch 78, minibatch 1000/1000, validation error 0.020200, patience 77999/109998 epoch 79, minibatch 1000/1000, validation error 0.020200, patience 78999/109998 epoch 80, minibatch 1000/1000, validation error 0.020300, patience 79999/109998 epoch 81, minibatch 1000/1000, validation error 0.020300, patience 80999/109998 epoch 82, minibatch 1000/1000, validation error 0.020300, patience 81999/109998 epoch 83, minibatch 1000/1000, validation error 0.020300, patience 82999/109998 epoch 84, minibatch 1000/1000, validation error 0.020300, patience 83999/109998 epoch 85, minibatch 1000/1000, validation error 0.020300, patience 84999/109998 epoch 86, minibatch 1000/1000, validation error 0.020200, patience 85999/109998 epoch 87, minibatch 1000/1000, validation error 0.020200, patience 86999/109998 epoch 88, minibatch 1000/1000, validation error 0.020200, patience 87999/109998 epoch 89, minibatch 1000/1000, validation error 0.020200, patience 88999/109998 epoch 90, minibatch 1000/1000, validation error 0.020000, patience 89999/109998 epoch 91, minibatch 1000/1000, validation error 0.020000, patience 90999/109998 epoch 92, minibatch 1000/1000, validation error 0.019800, patience 91999/109998 epoch 93, minibatch 1000/1000, validation error 0.019800, patience 92999/183998 epoch 94, minibatch 1000/1000, validation error 0.019700, patience 93999/183998 epoch 95, minibatch 1000/1000, validation error 0.019700, patience 94999/187998 epoch 96, minibatch 1000/1000, validation error 0.019700, patience 95999/187998 epoch 97, minibatch 1000/1000, validation error 0.019700, patience 96999/187998 epoch 98, minibatch 1000/1000, validation error 0.019700, patience 97999/187998 epoch 99, minibatch 1000/1000, validation error 0.019700, patience 98999/187998 epoch 100, minibatch 1000/1000, validation error 0.019700, patience 99999/187998 epoch 101, minibatch 1000/1000, validation error 0.019700, patience 100999/187998 epoch 102, minibatch 1000/1000, validation error 0.019700, patience 101999/187998 epoch 103, minibatch 1000/1000, validation error 0.019800, patience 102999/187998 epoch 104, minibatch 1000/1000, validation error 0.019800, patience 103999/187998 epoch 105, minibatch 1000/1000, validation error 0.019800, patience 104999/187998 epoch 106, minibatch 1000/1000, validation error 0.019700, patience 105999/187998 epoch 107, minibatch 1000/1000, validation error 0.019700, patience 106999/187998 epoch 108, minibatch 1000/1000, validation error 0.019600, patience 107999/187998 epoch 109, minibatch 1000/1000, validation error 0.019600, patience 108999/215998 epoch 110, minibatch 1000/1000, validation error 0.019400, patience 109999/215998 epoch 111, minibatch 1000/1000, validation error 0.019300, patience 110999/219998 epoch 112, minibatch 1000/1000, validation error 0.019300, patience 111999/221998 epoch 113, minibatch 1000/1000, validation error 0.019300, patience 112999/221998 epoch 114, minibatch 1000/1000, validation error 0.019300, patience 113999/221998 epoch 115, minibatch 1000/1000, validation error 0.019300, patience 114999/221998 epoch 116, minibatch 1000/1000, validation error 0.019300, patience 115999/221998 epoch 117, minibatch 1000/1000, validation error 0.019200, patience 116999/221998 epoch 118, minibatch 1000/1000, validation error 0.019200, patience 117999/233998 epoch 119, minibatch 1000/1000, validation error 0.019200, patience 118999/233998 epoch 120, minibatch 1000/1000, validation error 0.019200, patience 119999/233998 epoch 121, minibatch 1000/1000, validation error 0.019100, patience 120999/233998 epoch 122, minibatch 1000/1000, validation error 0.019100, patience 121999/241998 epoch 123, minibatch 1000/1000, validation error 0.019100, patience 122999/241998 epoch 124, minibatch 1000/1000, validation error 0.019000, patience 123999/241998 epoch 125, minibatch 1000/1000, validation error 0.019000, patience 124999/247998 epoch 126, minibatch 1000/1000, validation error 0.019000, patience 125999/247998 epoch 127, minibatch 1000/1000, validation error 0.019000, patience 126999/247998 epoch 128, minibatch 1000/1000, validation error 0.019000, patience 127999/247998 epoch 129, minibatch 1000/1000, validation error 0.019000, patience 128999/247998 epoch 130, minibatch 1000/1000, validation error 0.018900, patience 129999/247998 epoch 131, minibatch 1000/1000, validation error 0.018800, patience 130999/259998 epoch 132, minibatch 1000/1000, validation error 0.018800, patience 131999/261998 epoch 133, minibatch 1000/1000, validation error 0.018900, patience 132999/261998 epoch 134, minibatch 1000/1000, validation error 0.019000, patience 133999/261998 epoch 135, minibatch 1000/1000, validation error 0.018900, patience 134999/261998 epoch 136, minibatch 1000/1000, validation error 0.018900, patience 135999/261998 epoch 137, minibatch 1000/1000, validation error 0.019000, patience 136999/261998 epoch 138, minibatch 1000/1000, validation error 0.019000, patience 137999/261998 epoch 139, minibatch 1000/1000, validation error 0.019000, patience 138999/261998 epoch 140, minibatch 1000/1000, validation error 0.019000, patience 139999/261998 epoch 141, minibatch 1000/1000, validation error 0.019000, patience 140999/261998 epoch 142, minibatch 1000/1000, validation error 0.019000, patience 141999/261998 epoch 143, minibatch 1000/1000, validation error 0.019000, patience 142999/261998 epoch 144, minibatch 1000/1000, validation error 0.019000, patience 143999/261998 epoch 145, minibatch 1000/1000, validation error 0.019000, patience 144999/261998 epoch 146, minibatch 1000/1000, validation error 0.019000, patience 145999/261998 epoch 147, minibatch 1000/1000, validation error 0.019000, patience 146999/261998 epoch 148, minibatch 1000/1000, validation error 0.019000, patience 147999/261998 epoch 149, minibatch 1000/1000, validation error 0.018900, patience 148999/261998 epoch 150, minibatch 1000/1000, validation error 0.018900, patience 149999/261998 epoch 151, minibatch 1000/1000, validation error 0.018800, patience 150999/261998 epoch 152, minibatch 1000/1000, validation error 0.018800, patience 151999/261998 epoch 153, minibatch 1000/1000, validation error 0.018700, patience 152999/261998 epoch 154, minibatch 1000/1000, validation error 0.018700, patience 153999/305998 epoch 155, minibatch 1000/1000, validation error 0.018700, patience 154999/305998 epoch 156, minibatch 1000/1000, validation error 0.018700, patience 155999/305998 epoch 157, minibatch 1000/1000, validation error 0.018600, patience 156999/305998 epoch 158, minibatch 1000/1000, validation error 0.018600, patience 157999/313998 epoch 159, minibatch 1000/1000, validation error 0.018600, patience 158999/313998 epoch 160, minibatch 1000/1000, validation error 0.018600, patience 159999/313998 epoch 161, minibatch 1000/1000, validation error 0.018500, patience 160999/313998 epoch 162, minibatch 1000/1000, validation error 0.018500, patience 161999/321998 epoch 163, minibatch 1000/1000, validation error 0.018400, patience 162999/321998 epoch 164, minibatch 1000/1000, validation error 0.018400, patience 163999/325998 epoch 165, minibatch 1000/1000, validation error 0.018400, patience 164999/325998 epoch 166, minibatch 1000/1000, validation error 0.018400, patience 165999/325998 epoch 167, minibatch 1000/1000, validation error 0.018400, patience 166999/325998 epoch 168, minibatch 1000/1000, validation error 0.018400, patience 167999/325998 epoch 169, minibatch 1000/1000, validation error 0.018400, patience 168999/325998 epoch 170, minibatch 1000/1000, validation error 0.018400, patience 169999/325998 epoch 171, minibatch 1000/1000, validation error 0.018400, patience 170999/325998 epoch 172, minibatch 1000/1000, validation error 0.018400, patience 171999/325998 epoch 173, minibatch 1000/1000, validation error 0.018400, patience 172999/325998 epoch 174, minibatch 1000/1000, validation error 0.018400, patience 173999/325998 epoch 175, minibatch 1000/1000, validation error 0.018400, patience 174999/325998 epoch 176, minibatch 1000/1000, validation error 0.018400, patience 175999/325998 epoch 177, minibatch 1000/1000, validation error 0.018400, patience 176999/325998 epoch 178, minibatch 1000/1000, validation error 0.018400, patience 177999/325998 epoch 179, minibatch 1000/1000, validation error 0.018300, patience 178999/325998 epoch 180, minibatch 1000/1000, validation error 0.018300, patience 179999/357998 epoch 181, minibatch 1000/1000, validation error 0.018300, patience 180999/357998 epoch 182, minibatch 1000/1000, validation error 0.018300, patience 181999/357998 epoch 183, minibatch 1000/1000, validation error 0.018300, patience 182999/357998 epoch 184, minibatch 1000/1000, validation error 0.018300, patience 183999/357998 epoch 185, minibatch 1000/1000, validation error 0.018300, patience 184999/357998 epoch 186, minibatch 1000/1000, validation error 0.018300, patience 185999/357998 epoch 187, minibatch 1000/1000, validation error 0.018300, patience 186999/357998 epoch 188, minibatch 1000/1000, validation error 0.018300, patience 187999/357998 epoch 189, minibatch 1000/1000, validation error 0.018300, patience 188999/357998 epoch 190, minibatch 1000/1000, validation error 0.018300, patience 189999/357998 epoch 191, minibatch 1000/1000, validation error 0.018300, patience 190999/357998 epoch 192, minibatch 1000/1000, validation error 0.018400, patience 191999/357998 epoch 193, minibatch 1000/1000, validation error 0.018400, patience 192999/357998 epoch 194, minibatch 1000/1000, validation error 0.018400, patience 193999/357998 epoch 195, minibatch 1000/1000, validation error 0.018300, patience 194999/357998 epoch 196, minibatch 1000/1000, validation error 0.018300, patience 195999/357998 epoch 197, minibatch 1000/1000, validation error 0.018400, patience 196999/357998 epoch 198, minibatch 1000/1000, validation error 0.018400, patience 197999/357998 epoch 199, minibatch 1000/1000, validation error 0.018400, patience 198999/357998 epoch 200, minibatch 1000/1000, validation error 0.018400, patience 199999/357998 epoch 201, minibatch 1000/1000, validation error 0.018400, patience 200999/357998 epoch 202, minibatch 1000/1000, validation error 0.018400, patience 201999/357998 epoch 203, minibatch 1000/1000, validation error 0.018300, patience 202999/357998 epoch 204, minibatch 1000/1000, validation error 0.018300, patience 203999/357998 epoch 205, minibatch 1000/1000, validation error 0.018200, patience 204999/357998 epoch 206, minibatch 1000/1000, validation error 0.018200, patience 205999/409998 epoch 207, minibatch 1000/1000, validation error 0.018200, patience 206999/409998 epoch 208, minibatch 1000/1000, validation error 0.018200, patience 207999/409998 epoch 209, minibatch 1000/1000, validation error 0.018200, patience 208999/409998 epoch 210, minibatch 1000/1000, validation error 0.018200, patience 209999/409998 epoch 211, minibatch 1000/1000, validation error 0.018200, patience 210999/409998 epoch 212, minibatch 1000/1000, validation error 0.018100, patience 211999/409998 epoch 213, minibatch 1000/1000, validation error 0.018100, patience 212999/423998 epoch 214, minibatch 1000/1000, validation error 0.018100, patience 213999/423998 epoch 215, minibatch 1000/1000, validation error 0.018100, patience 214999/423998 epoch 216, minibatch 1000/1000, validation error 0.018000, patience 215999/423998 epoch 217, minibatch 1000/1000, validation error 0.017900, patience 216999/431998 epoch 218, minibatch 1000/1000, validation error 0.017900, patience 217999/433998 epoch 219, minibatch 1000/1000, validation error 0.017900, patience 218999/433998 epoch 220, minibatch 1000/1000, validation error 0.017900, patience 219999/433998 epoch 221, minibatch 1000/1000, validation error 0.017900, patience 220999/433998 epoch 222, minibatch 1000/1000, validation error 0.017900, patience 221999/433998 epoch 223, minibatch 1000/1000, validation error 0.017900, patience 222999/433998 epoch 224, minibatch 1000/1000, validation error 0.017900, patience 223999/433998 epoch 225, minibatch 1000/1000, validation error 0.017900, patience 224999/433998 epoch 226, minibatch 1000/1000, validation error 0.017900, patience 225999/433998 epoch 227, minibatch 1000/1000, validation error 0.017900, patience 226999/433998 epoch 228, minibatch 1000/1000, validation error 0.017900, patience 227999/433998 epoch 229, minibatch 1000/1000, validation error 0.017900, patience 228999/433998 epoch 230, minibatch 1000/1000, validation error 0.017900, patience 229999/433998 epoch 231, minibatch 1000/1000, validation error 0.017900, patience 230999/433998 epoch 232, minibatch 1000/1000, validation error 0.017900, patience 231999/433998 epoch 233, minibatch 1000/1000, validation error 0.017900, patience 232999/433998 epoch 234, minibatch 1000/1000, validation error 0.017900, patience 233999/433998 epoch 235, minibatch 1000/1000, validation error 0.017900, patience 234999/433998 epoch 236, minibatch 1000/1000, validation error 0.017900, patience 235999/433998 epoch 237, minibatch 1000/1000, validation error 0.017900, patience 236999/433998 epoch 238, minibatch 1000/1000, validation error 0.017900, patience 237999/433998 epoch 239, minibatch 1000/1000, validation error 0.017900, patience 238999/433998 epoch 240, minibatch 1000/1000, validation error 0.017900, patience 239999/433998 epoch 241, minibatch 1000/1000, validation error 0.017900, patience 240999/433998 epoch 242, minibatch 1000/1000, validation error 0.017900, patience 241999/433998 epoch 243, minibatch 1000/1000, validation error 0.017900, patience 242999/433998 epoch 244, minibatch 1000/1000, validation error 0.017900, patience 243999/433998 epoch 245, minibatch 1000/1000, validation error 0.017900, patience 244999/433998 epoch 246, minibatch 1000/1000, validation error 0.017900, patience 245999/433998 epoch 247, minibatch 1000/1000, validation error 0.017900, patience 246999/433998 epoch 248, minibatch 1000/1000, validation error 0.017900, patience 247999/433998 epoch 249, minibatch 1000/1000, validation error 0.017900, patience 248999/433998 epoch 250, minibatch 1000/1000, validation error 0.017900, patience 249999/433998 epoch 251, minibatch 1000/1000, validation error 0.017900, patience 250999/433998 epoch 252, minibatch 1000/1000, validation error 0.017800, patience 251999/433998 epoch 253, minibatch 1000/1000, validation error 0.017800, patience 252999/503998 epoch 254, minibatch 1000/1000, validation error 0.017800, patience 253999/503998 epoch 255, minibatch 1000/1000, validation error 0.017800, patience 254999/503998 epoch 256, minibatch 1000/1000, validation error 0.017800, patience 255999/503998 epoch 257, minibatch 1000/1000, validation error 0.017800, patience 256999/503998 epoch 258, minibatch 1000/1000, validation error 0.017800, patience 257999/503998 epoch 259, minibatch 1000/1000, validation error 0.017800, patience 258999/503998 epoch 260, minibatch 1000/1000, validation error 0.017800, patience 259999/503998 epoch 261, minibatch 1000/1000, validation error 0.017800, patience 260999/503998 epoch 262, minibatch 1000/1000, validation error 0.017800, patience 261999/503998 epoch 263, minibatch 1000/1000, validation error 0.017800, patience 262999/503998 epoch 264, minibatch 1000/1000, validation error 0.017800, patience 263999/503998 epoch 265, minibatch 1000/1000, validation error 0.017800, patience 264999/503998 epoch 266, minibatch 1000/1000, validation error 0.017800, patience 265999/503998 epoch 267, minibatch 1000/1000, validation error 0.017800, patience 266999/503998 epoch 268, minibatch 1000/1000, validation error 0.017800, patience 267999/503998 epoch 269, minibatch 1000/1000, validation error 0.017800, patience 268999/503998 epoch 270, minibatch 1000/1000, validation error 0.017900, patience 269999/503998 epoch 271, minibatch 1000/1000, validation error 0.017900, patience 270999/503998 epoch 272, minibatch 1000/1000, validation error 0.017900, patience 271999/503998 epoch 273, minibatch 1000/1000, validation error 0.017900, patience 272999/503998 epoch 274, minibatch 1000/1000, validation error 0.017900, patience 273999/503998 epoch 275, minibatch 1000/1000, validation error 0.017900, patience 274999/503998 epoch 276, minibatch 1000/1000, validation error 0.017900, patience 275999/503998 epoch 277, minibatch 1000/1000, validation error 0.017900, patience 276999/503998 epoch 278, minibatch 1000/1000, validation error 0.017900, patience 277999/503998 epoch 279, minibatch 1000/1000, validation error 0.017900, patience 278999/503998 epoch 280, minibatch 1000/1000, validation error 0.017900, patience 279999/503998 epoch 281, minibatch 1000/1000, validation error 0.017900, patience 280999/503998 epoch 282, minibatch 1000/1000, validation error 0.018000, patience 281999/503998 epoch 283, minibatch 1000/1000, validation error 0.018000, patience 282999/503998 epoch 284, minibatch 1000/1000, validation error 0.018000, patience 283999/503998 epoch 285, minibatch 1000/1000, validation error 0.018000, patience 284999/503998 epoch 286, minibatch 1000/1000, validation error 0.018000, patience 285999/503998 epoch 287, minibatch 1000/1000, validation error 0.018000, patience 286999/503998 epoch 288, minibatch 1000/1000, validation error 0.018000, patience 287999/503998 epoch 289, minibatch 1000/1000, validation error 0.018000, patience 288999/503998 epoch 290, minibatch 1000/1000, validation error 0.018000, patience 289999/503998 epoch 291, minibatch 1000/1000, validation error 0.018000, patience 290999/503998 epoch 292, minibatch 1000/1000, validation error 0.018000, patience 291999/503998 epoch 293, minibatch 1000/1000, validation error 0.018000, patience 292999/503998 epoch 294, minibatch 1000/1000, validation error 0.018000, patience 293999/503998 epoch 295, minibatch 1000/1000, validation error 0.018000, patience 294999/503998 epoch 296, minibatch 1000/1000, validation error 0.018000, patience 295999/503998 epoch 297, minibatch 1000/1000, validation error 0.018000, patience 296999/503998 epoch 298, minibatch 1000/1000, validation error 0.018000, patience 297999/503998 epoch 299, minibatch 1000/1000, validation error 0.018000, patience 298999/503998 epoch 300, minibatch 1000/1000, validation error 0.018000, patience 299999/503998 optimization done with best validation: 0.017800
gray()
W2Img(sda.dA_layers[0].W.get_value(), img_shape = (28, 28), tile_shape = (20, 20))
<matplotlib.figure.Figure at 0xab54810>
print 'scoring after supervised learning ...'
score_valid = sda.get_score_fn(valid_x, valid_y)
score_test = sda.get_score_fn(test_x, test_y)
print score_valid()
print score_test()
scoring after supervised learning ... [array(0.982)] [array(0.9811)]
$ p(x) = \frac{e^{-E(x)}}{Z} $ where $Z$ is the normalizing factor called partition function by analogy with physical systems $Z = \sum_x{e^{-E(x)}}$
-TODO