import theanoml2
reload(theanoml2)
<module 'theanoml2' from 'theanoml2/__init__.pyc'>
reload(theanoml2.formula)
reload(theanoml2.optimize)
import cPickle
import numpy as np
from sklearn.cross_validation import train_test_split
from functools import partial
X, y = cPickle.load(open('data/digits.pkl', 'rb'))
X = X / 255.
classes = np.unique(y)
print X.shape, y.shape
train_X, validation_X, train_y, validation_y = train_test_split(X, y, test_size = 0.2)
(42000, 784) (42000,)
## test SdA classifier
reload(theanoml2)
reload(theanoml2.lenet)
reload(theanoml2.formula)
reload(theanoml2.auto_encoder)
sda = theanoml2.auto_encoder.SdAClassifier(len(classes), pretrain_epochs = 15,
finetune_epochs = 100)
sda.formula_ = sda._create_formula(train_X, train_y)
sda.pretrain(X) ## train on whole X
sda.fit(train_X, train_y)
print sda.score(train_X, train_y)
print sda.score(validation_X, validation_y)
training epoch 0, train cost 122.080660 training epoch 1, train cost 97.409468 training epoch 2, train cost 91.860783 training epoch 3, train cost 88.531213 training epoch 4, train cost 86.063633 training epoch 5, train cost 84.116139 training epoch 6, train cost 82.559561 training epoch 7, train cost 81.350771 training epoch 8, train cost 80.438471 training epoch 9, train cost 79.705019 training epoch 10, train cost 79.149875 training epoch 11, train cost 78.653986 training epoch 12, train cost 78.222760 training epoch 13, train cost 77.884829 training epoch 14, train cost 77.607123 optimization complete with best train cost: 77.607123 training epoch 0, train cost 37.300914 training epoch 1, train cost 32.143227 training epoch 2, train cost 31.259929 training epoch 3, train cost 30.858785 training epoch 4, train cost 30.584925 training epoch 5, train cost 30.392978 training epoch 6, train cost 30.283089 training epoch 7, train cost 30.123150 training epoch 8, train cost 30.018182 training epoch 9, train cost 29.913434 training epoch 10, train cost 29.831760 training epoch 11, train cost 29.768833 training epoch 12, train cost 29.708539 training epoch 13, train cost 29.649420 training epoch 14, train cost 29.576363 optimization complete with best train cost: 29.576363 training epoch 0, train cost 44.067446 training epoch 1, train cost 40.254193 training epoch 2, train cost 39.663832 training epoch 3, train cost 39.421751 training epoch 4, train cost 39.224811 training epoch 5, train cost 39.083494 training epoch 6, train cost 38.955715 training epoch 7, train cost 38.871046 training epoch 8, train cost 38.763742 training epoch 9, train cost 38.659808 training epoch 10, train cost 38.616512 training epoch 11, train cost 38.561521 training epoch 12, train cost 38.500389 training epoch 13, train cost 38.431741 training epoch 14, train cost 38.374703 optimization complete with best train cost: 38.374703 epoch 1, minibatch 500/537, validation_error 21.746269 % epoch 2, minibatch 463/537, validation_error 12.880597 % epoch 3, minibatch 426/537, validation_error 10.328358 % epoch 4, minibatch 389/537, validation_error 9.552239 % epoch 5, minibatch 352/537, validation_error 8.567164 % epoch 6, minibatch 315/537, validation_error 8.432836 % epoch 7, minibatch 278/537, validation_error 7.686567 % epoch 8, minibatch 241/537, validation_error 7.000000 % epoch 9, minibatch 204/537, validation_error 6.567164 % epoch 10, minibatch 167/537, validation_error 6.432836 % epoch 11, minibatch 130/537, validation_error 6.313433 % epoch 12, minibatch 93/537, validation_error 5.880597 % epoch 13, minibatch 56/537, validation_error 5.611940 % epoch 14, minibatch 19/537, validation_error 5.492537 % epoch 14, minibatch 519/537, validation_error 5.223881 % epoch 15, minibatch 482/537, validation_error 5.328358 % epoch 16, minibatch 445/537, validation_error 5.149254 % epoch 17, minibatch 408/537, validation_error 5.119403 % epoch 18, minibatch 371/537, validation_error 5.000000 % epoch 19, minibatch 334/537, validation_error 4.820896 % epoch 20, minibatch 297/537, validation_error 4.880597 % epoch 21, minibatch 260/537, validation_error 4.791045 % epoch 22, minibatch 223/537, validation_error 4.522388 % epoch 23, minibatch 186/537, validation_error 4.611940 % epoch 24, minibatch 149/537, validation_error 4.641791 % epoch 25, minibatch 112/537, validation_error 4.462687 % epoch 26, minibatch 75/537, validation_error 4.492537 % epoch 27, minibatch 38/537, validation_error 4.313433 % epoch 28, minibatch 1/537, validation_error 4.134328 % epoch 28, minibatch 501/537, validation_error 4.179104 % epoch 29, minibatch 464/537, validation_error 4.194030 % epoch 30, minibatch 427/537, validation_error 4.164179 % epoch 31, minibatch 390/537, validation_error 4.149254 % epoch 32, minibatch 353/537, validation_error 3.985075 % epoch 33, minibatch 316/537, validation_error 4.029851 % epoch 34, minibatch 279/537, validation_error 3.985075 % epoch 35, minibatch 242/537, validation_error 3.791045 % epoch 36, minibatch 205/537, validation_error 3.955224 % epoch 37, minibatch 168/537, validation_error 3.865672 % epoch 38, minibatch 131/537, validation_error 4.238806 % epoch 39, minibatch 94/537, validation_error 3.776119 % epoch 40, minibatch 57/537, validation_error 3.865672 % epoch 41, minibatch 20/537, validation_error 3.985075 % epoch 41, minibatch 520/537, validation_error 3.761194 % epoch 42, minibatch 483/537, validation_error 3.910448 % epoch 43, minibatch 446/537, validation_error 3.597015 % epoch 44, minibatch 409/537, validation_error 3.626866 % epoch 45, minibatch 372/537, validation_error 3.671642 % epoch 46, minibatch 335/537, validation_error 3.671642 % epoch 47, minibatch 298/537, validation_error 3.850746 % epoch 48, minibatch 261/537, validation_error 3.656716 % epoch 49, minibatch 224/537, validation_error 3.626866 % epoch 50, minibatch 187/537, validation_error 3.626866 % epoch 51, minibatch 150/537, validation_error 3.656716 % epoch 52, minibatch 113/537, validation_error 3.492537 % epoch 53, minibatch 76/537, validation_error 3.641791 % epoch 54, minibatch 39/537, validation_error 3.776119 % epoch 55, minibatch 2/537, validation_error 3.507463 % epoch 55, minibatch 502/537, validation_error 3.626866 % epoch 56, minibatch 465/537, validation_error 3.582090 % epoch 57, minibatch 428/537, validation_error 3.641791 % epoch 58, minibatch 391/537, validation_error 3.522388 % epoch 59, minibatch 354/537, validation_error 3.507463 % epoch 60, minibatch 317/537, validation_error 3.626866 % epoch 61, minibatch 280/537, validation_error 3.462687 % epoch 62, minibatch 243/537, validation_error 3.582090 % epoch 63, minibatch 206/537, validation_error 3.582090 % epoch 64, minibatch 169/537, validation_error 3.552239 % epoch 65, minibatch 132/537, validation_error 3.761194 % epoch 66, minibatch 95/537, validation_error 3.492537 % epoch 67, minibatch 58/537, validation_error 3.671642 % epoch 68, minibatch 21/537, validation_error 3.611940 % epoch 68, minibatch 521/537, validation_error 3.552239 % epoch 69, minibatch 484/537, validation_error 3.582090 % epoch 70, minibatch 447/537, validation_error 3.462687 % epoch 71, minibatch 410/537, validation_error 3.507463 % epoch 72, minibatch 373/537, validation_error 3.567164 % epoch 73, minibatch 336/537, validation_error 3.552239 % epoch 74, minibatch 299/537, validation_error 3.582090 % epoch 75, minibatch 262/537, validation_error 3.402985 % epoch 76, minibatch 225/537, validation_error 3.477612 % epoch 77, minibatch 188/537, validation_error 3.402985 % epoch 78, minibatch 151/537, validation_error 3.597015 % epoch 79, minibatch 114/537, validation_error 3.507463 % epoch 80, minibatch 77/537, validation_error 3.492537 % epoch 81, minibatch 40/537, validation_error 3.597015 % epoch 82, minibatch 3/537, validation_error 3.507463 % epoch 82, minibatch 503/537, validation_error 3.552239 % epoch 83, minibatch 466/537, validation_error 3.582090 % epoch 84, minibatch 429/537, validation_error 3.477612 % epoch 85, minibatch 392/537, validation_error 3.507463 % epoch 86, minibatch 355/537, validation_error 3.522388 % epoch 87, minibatch 318/537, validation_error 3.492537 % epoch 88, minibatch 281/537, validation_error 3.477612 % epoch 89, minibatch 244/537, validation_error 3.507463 % epoch 90, minibatch 207/537, validation_error 3.477612 % epoch 91, minibatch 170/537, validation_error 3.417910 % epoch 92, minibatch 133/537, validation_error 3.537313 % epoch 93, minibatch 96/537, validation_error 3.432836 % epoch 94, minibatch 59/537, validation_error 3.507463 % epoch 95, minibatch 22/537, validation_error 3.537313 % epoch 95, minibatch 522/537, validation_error 3.507463 % epoch 96, minibatch 485/537, validation_error 3.507463 % epoch 97, minibatch 448/537, validation_error 3.328358 % epoch 98, minibatch 411/537, validation_error 3.507463 % epoch 99, minibatch 374/537, validation_error 3.522388 % epoch 100, minibatch 337/537, validation_error 3.507463 % optimization complete with best validation error: 3.328358 % 0.992857142857 0.968571428571
## Denoising Auto Encoder
reload(theanoml2)
reload(theanoml2.lenet)
reload(theanoml2.formula)
reload(theanoml2.auto_encoder)
d_auto_encoder = theanoml2.auto_encoder.DenoisingAutoEncoder(n_hidden = 500, corruption_level = 0.1, n_epochs=5)
d_auto_encoder.partial_fit(X)
print d_auto_encoder.transform(X).shape
training epoch 0, train cost -4661960859.348659 training epoch 1, train cost -14022858680.161507 training epoch 2, train cost -23384465235.135036 training epoch 3, train cost -32746071790.105610 training epoch 4, train cost -42107678345.075310 optimization complete with best train cost: -42107678345.075310 (42000, 500)
reload(theanoml2)
reload(theanoml2.lenet)
reload(theanoml2.formula)
reload(theanoml2.auto_encoder)
c_auto_encoder = theanoml2.auto_encoder.ContractiveAutoEncoder(n_hidden = 500,
contraction_level = 0.1, n_epochs=5)
c_auto_encoder.partial_fit(X[:1000])
print c_auto_encoder.transform(X[:1000]).shape
training epoch 0, train cost -103757114.201847 training epoch 1, train cost -334109909.187471 training epoch 2, train cost -564847972.077478 training epoch 3, train cost -795586036.173563 training epoch 4, train cost -1026324100.134109 optimization complete with best train cost: -1026324100.134109 (1000, 500)
## LeNet classification
import theanoml2
reload(theanoml2)
reload(theanoml2.lenet)
reload(theanoml2.formula)
lenet = theanoml2.lenet.LeNetClassifier(n_classes = len(classes), image_size = (28, 28),
n_hidden = 500, n_epochs = 5,)
lenet.partial_fit(train_X, train_y)
print lenet.score(train_X[:50], train_y[:50])
print np.mean([lenet.score(validation_X[i*50:(i+1)*50], validation_y[i*50:(i+1)*50])
for i in xrange(validation_X.shape[0] / 50)])
epoch 1, minibatch 500/537, validation_error 7.686567 % epoch 2, minibatch 463/537, validation_error 5.298507 % epoch 3, minibatch 426/537, validation_error 4.223881 % epoch 4, minibatch 389/537, validation_error 3.746269 % epoch 5, minibatch 352/537, validation_error 3.238806 % optimization complete with best validation error: 3.238806 % 0.94 0.966071428571
v_valid_X = theanoml2.formula.share_data(validation_X)
print v_valid_X.eval().shape
print lenet.batch_size, lenet.image_size
batch_size = 8400
layer0_input = v_valid_X.reshape((batch_size, 1, lenet.image_size[0], lenet.image_size[1]))
layer0_input.eval()
## MLP classification
reload(theanoml2.mlp)
reload(theanoml2.formula)
mlp = theanoml2.mlp.MLPClassifier(n_classes = len(classes), n_hidden = 500, n_epochs=5)
mlp.partial_fit(train_X, train_y)
print mlp.score(train_X, train_y)
mlp.partial_fit(train_X, train_y)
print mlp.score(train_X, train_y)
print mlp.score(validation_X, validation_y)
epoch 1, minibatch 500/537, validation_error 10.194030 % epoch 2, minibatch 463/537, validation_error 8.820896 % epoch 3, minibatch 426/537, validation_error 8.000000 % epoch 4, minibatch 389/537, validation_error 7.955224 % epoch 5, minibatch 352/537, validation_error 7.925373 % optimization complete with best validation error: 7.925373 % 0.92994047619 epoch 1, minibatch 500/537, validation_error 7.432836 % epoch 2, minibatch 463/537, validation_error 7.000000 % epoch 3, minibatch 426/537, validation_error 6.716418 % epoch 4, minibatch 389/537, validation_error 6.626866 % epoch 5, minibatch 352/537, validation_error 6.567164 % optimization complete with best validation error: 6.567164 % 0.942708333333 0.932857142857
## Use Logistic Regression Wrapper
reload(theanoml2)
reload(theanoml2.formula)
reload(theanoml2.linear_model)
reload(theanoml2.optimize)
lgr_model = theanoml2.linear_model.LogisticRegression(n_classes = len(classes), n_epochs = 50)
lgr_model.partial_fit(train_X, train_y)
#print lgr_model.predict(train_X)
print lgr_model.score(train_X, train_y)
print lgr_model.score(validation_X, validation_y)
epoch 1, minibatch 500/537, validation_error 16.298507 % epoch 2, minibatch 463/537, validation_error 13.059701 % epoch 3, minibatch 426/537, validation_error 10.000000 % epoch 4, minibatch 389/537, validation_error 12.985075 % epoch 5, minibatch 352/537, validation_error 13.208955 % epoch 6, minibatch 315/537, validation_error 10.850746 % epoch 7, minibatch 278/537, validation_error 14.119403 % epoch 8, minibatch 241/537, validation_error 13.223881 % epoch 9, minibatch 204/537, validation_error 10.522388 % epoch 10, minibatch 167/537, validation_error 10.104478 % epoch 11, minibatch 130/537, validation_error 10.417910 % epoch 12, minibatch 93/537, validation_error 10.477612 % epoch 13, minibatch 56/537, validation_error 11.000000 % epoch 14, minibatch 19/537, validation_error 10.835821 % epoch 14, minibatch 519/537, validation_error 13.791045 % epoch 15, minibatch 482/537, validation_error 11.746269 % epoch 16, minibatch 445/537, validation_error 11.970149 % epoch 17, minibatch 408/537, validation_error 12.910448 % epoch 18, minibatch 371/537, validation_error 21.910448 % epoch 19, minibatch 334/537, validation_error 12.791045 % epoch 20, minibatch 297/537, validation_error 11.029851 % epoch 21, minibatch 260/537, validation_error 11.537313 % epoch 22, minibatch 223/537, validation_error 12.268657 % epoch 23, minibatch 186/537, validation_error 14.567164 % epoch 24, minibatch 149/537, validation_error 10.014925 % epoch 25, minibatch 112/537, validation_error 12.343284 % epoch 26, minibatch 75/537, validation_error 11.477612 % epoch 27, minibatch 38/537, validation_error 11.029851 % epoch 28, minibatch 1/537, validation_error 11.567164 % epoch 28, minibatch 501/537, validation_error 14.253731 % epoch 29, minibatch 464/537, validation_error 13.194030 % epoch 30, minibatch 427/537, validation_error 11.238806 % epoch 31, minibatch 390/537, validation_error 11.388060 % epoch 32, minibatch 353/537, validation_error 13.432836 % epoch 33, minibatch 316/537, validation_error 11.880597 % epoch 34, minibatch 279/537, validation_error 10.940299 % epoch 35, minibatch 242/537, validation_error 12.537313 % epoch 36, minibatch 205/537, validation_error 10.925373 % epoch 37, minibatch 168/537, validation_error 10.089552 % epoch 38, minibatch 131/537, validation_error 11.477612 % epoch 39, minibatch 94/537, validation_error 12.283582 % epoch 40, minibatch 57/537, validation_error 11.522388 % epoch 41, minibatch 20/537, validation_error 11.253731 % epoch 41, minibatch 520/537, validation_error 12.268657 % epoch 42, minibatch 483/537, validation_error 14.388060 % epoch 43, minibatch 446/537, validation_error 13.597015 % epoch 44, minibatch 409/537, validation_error 12.119403 % epoch 45, minibatch 372/537, validation_error 12.238806 % epoch 46, minibatch 335/537, validation_error 12.865672 % epoch 47, minibatch 298/537, validation_error 13.761194 % epoch 48, minibatch 261/537, validation_error 17.865672 % epoch 49, minibatch 224/537, validation_error 12.865672 % epoch 50, minibatch 187/537, validation_error 17.283582 % optimization complete with best validation error: 10.000000 % 0.903125 0.88880952381
## Linear Regression Model
import theanoml2
reload(theanoml2)
reload(theanoml2.linear_model)
reload(theanoml2.formula)
from sklearn.datasets import load_diabetes
from sklearn.cross_validation import train_test_split
diabetes = load_diabetes()
X, y = diabetes.data, diabetes.target
X = np.random.uniform(size = (1000, 3))
y = X[:,0]*2. + X[:, 1] * 5. + 1.
print X.shape, y.shape
train_X, validation_X, train_y, validation_y = train_test_split(X, y, test_size = 0.2)
lr_model = theanoml2.linear_model.LinearRegression(n_epochs = 10, batch_size=10)
lr_model.partial_fit(train_X, train_y)
print lr_model.score(train_X, train_y)
print lr_model.score(validation_X, validation_y)
print map(lambda v: v.eval(), lr_model.formula_.params)
(1000, 3) (1000,) epoch 1, minibatch 64/64, validation_error 91.202448 % epoch 2, minibatch 64/64, validation_error 76.554709 % epoch 3, minibatch 64/64, validation_error 68.960607 % epoch 4, minibatch 64/64, validation_error 62.231556 % epoch 5, minibatch 64/64, validation_error 56.194271 % epoch 6, minibatch 64/64, validation_error 50.775172 % epoch 7, minibatch 64/64, validation_error 45.913820 % epoch 8, minibatch 64/64, validation_error 41.565633 % epoch 9, minibatch 64/64, validation_error 37.656044 % epoch 10, minibatch 64/64, validation_error 34.142120 % optimization complete with best validation error: 34.142120 % 0.928087113353 0.93059764046 [array([ 1.56063239, 3.64438407, 0.17407062]), array(1.8625258492463739)]
import theanoml2
reload(theanoml2.formula)
reload(theanoml2.optimize)
import cPickle
import numpy as np
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import MinMaxScaler
from functools import partial
X, y = cPickle.load(open('data/blackbox.pkl', 'rb'))
X = MinMaxScaler().fit_transform(X)
import features
from IPython.parallel import Client
n_features = X.shape[1]
tri = features.TriKmeansFeatures(n_clusters = 20, sparse_result=False,
feat_patches = features.bootstrap_seqs(range(n_features),
n_iter = 500, subsize=20),
client = Client(), )
X = tri.fit_transform(X)
#print np.min(X), np.max(X)
y = y - 1
classes = np.unique(y)
print X.shape, y.shape
train_X, validation_X, train_y, validation_y = train_test_split(X, y, test_size = 0.2)
4/4 tasks finished after 47 s done (1000, 10000) (1000,)
## stacked auto encoders on blackbox data
reload(theanoml2)
reload(theanoml2.lenet)
reload(theanoml2.formula)
reload(theanoml2.auto_encoder)
sda = theanoml2.auto_encoder.SdAClassifier(len(classes), pretrain_epochs = 30, pretrain_batch_size=1,
finetune_learning_rate=0.01,
finetune_epochs = 100, hidden_layer_sizes = (3000, 1000, 500, 100),
corruption_levels = (0.1, 0.2, 0.3, 0.3))
sda.formula_ = sda._create_formula(train_X, train_y)
sda.pretrain(X) ## train on whole X
sda.fit(train_X, train_y)
print sda.score(train_X, train_y)
print sda.score(validation_X, validation_y)
print sda.transform(X).shape
training epoch 0, train cost 3445.677260 training epoch 1, train cost 3084.838276 training epoch 2, train cost 2952.708848 training epoch 3, train cost 2875.915584 training epoch 4, train cost 2826.343480 training epoch 5, train cost 2792.545099 training epoch 6, train cost 2768.712079 training epoch 7, train cost 2750.969216 training epoch 8, train cost 2737.301677 training epoch 9, train cost 2726.588094 training epoch 10, train cost 2717.791846 training epoch 11, train cost 2710.277212 training epoch 12, train cost 2703.995309 training epoch 13, train cost 2698.469449 training epoch 14, train cost 2693.468839 training epoch 15, train cost 2689.044113 training epoch 16, train cost 2684.916942 training epoch 17, train cost 2681.083811 training epoch 18, train cost 2677.633932 training epoch 19, train cost 2674.365541 training epoch 20, train cost 2671.309701 training epoch 21, train cost 2668.376969 training epoch 22, train cost 2665.713755 training epoch 23, train cost 2663.069842 training epoch 24, train cost 2660.519091 training epoch 25, train cost 2658.184363 training epoch 26, train cost 2655.978615 training epoch 27, train cost 2653.744477 training epoch 28, train cost 2651.533174 training epoch 29, train cost 2649.620674 optimization complete with best train cost: 2649.620674 training epoch 0, train cost 1491.487965 training epoch 1, train cost 1374.793743 training epoch 2, train cost 1343.583601 training epoch 3, train cost 1326.428579 training epoch 4, train cost 1314.783143 training epoch 5, train cost 1306.003855 training epoch 6, train cost 1299.006819 training epoch 7, train cost 1292.962288 training epoch 8, train cost 1288.015381 training epoch 9, train cost 1283.396309 training epoch 10, train cost 1279.571486 training epoch 11, train cost 1275.909117 training epoch 12, train cost 1272.799425 training epoch 13, train cost 1269.893933 training epoch 14, train cost 1267.298578 training epoch 15, train cost 1264.789091 training epoch 16, train cost 1262.331509 training epoch 17, train cost 1260.316512 training epoch 18, train cost 1258.039868 training epoch 19, train cost 1256.336931 training epoch 20, train cost 1254.433314 training epoch 21, train cost 1252.699207 training epoch 22, train cost 1251.015170 training epoch 23, train cost 1249.360617 training epoch 24, train cost 1247.789795 training epoch 25, train cost 1246.483422 training epoch 26, train cost 1244.994625 training epoch 27, train cost 1243.640561 training epoch 28, train cost 1242.429469 training epoch 29, train cost 1241.212573 optimization complete with best train cost: 1241.212573 training epoch 0, train cost 316.818868 training epoch 1, train cost 249.854899 training epoch 2, train cost 230.941835 training epoch 3, train cost 219.916137 training epoch 4, train cost 212.443484 training epoch 5, train cost 207.302483 training epoch 6, train cost 203.410199 training epoch 7, train cost 199.959555 training epoch 8, train cost 197.292084 training epoch 9, train cost 195.270577 training epoch 10, train cost 193.038864 training epoch 11, train cost 191.157723 training epoch 12, train cost 189.551793 training epoch 13, train cost 187.935762 training epoch 14, train cost 186.899146 training epoch 15, train cost 185.333712 training epoch 16, train cost 184.362900 training epoch 17, train cost 183.259189 training epoch 18, train cost 182.278092 training epoch 19, train cost 181.393068 training epoch 20, train cost 180.430494 training epoch 21, train cost 179.944222 training epoch 22, train cost 179.113612 training epoch 23, train cost 178.383777 training epoch 24, train cost 177.698206 training epoch 25, train cost 177.067117 training epoch 26, train cost 176.356915 training epoch 27, train cost 175.855538 training epoch 28, train cost 175.364912 training epoch 29, train cost 174.895925 optimization complete with best train cost: 174.895925 training epoch 0, train cost 259.094671 training epoch 1, train cost 205.865195 training epoch 2, train cost 189.986593 training epoch 3, train cost 179.742673 training epoch 4, train cost 172.105566 training epoch 5, train cost 166.146700 training epoch 6, train cost 161.447090 training epoch 7, train cost 157.553290 training epoch 8, train cost 154.230319 training epoch 9, train cost 151.560145 training epoch 10, train cost 149.113159 training epoch 11, train cost 147.074910 training epoch 12, train cost 145.172795 training epoch 13, train cost 143.722784 training epoch 14, train cost 142.139697 training epoch 15, train cost 140.969293 training epoch 16, train cost 139.805747 training epoch 17, train cost 138.560454 training epoch 18, train cost 137.723139 training epoch 19, train cost 136.705694 training epoch 20, train cost 135.815412 training epoch 21, train cost 135.184513 training epoch 22, train cost 134.283806 training epoch 23, train cost 133.508631 training epoch 24, train cost 132.809381 training epoch 25, train cost 132.420527 training epoch 26, train cost 131.712802 training epoch 27, train cost 131.015996 training epoch 28, train cost 130.467072 training epoch 29, train cost 130.076170 optimization complete with best train cost: 130.076170 epoch 1, minibatch 12/12, validation_error 74.000000 % epoch 2, minibatch 12/12, validation_error 74.000000 % epoch 3, minibatch 12/12, validation_error 74.000000 % epoch 4, minibatch 12/12, validation_error 74.000000 % epoch 5, minibatch 12/12, validation_error 74.000000 % epoch 6, minibatch 12/12, validation_error 74.000000 % epoch 7, minibatch 12/12, validation_error 74.000000 % epoch 8, minibatch 12/12, validation_error 74.000000 % epoch 9, minibatch 12/12, validation_error 74.000000 % epoch 10, minibatch 12/12, validation_error 74.000000 % epoch 11, minibatch 12/12, validation_error 74.000000 % epoch 12, minibatch 12/12, validation_error 74.000000 % epoch 13, minibatch 12/12, validation_error 74.000000 % epoch 14, minibatch 12/12, validation_error 74.000000 % epoch 15, minibatch 12/12, validation_error 74.000000 % epoch 16, minibatch 12/12, validation_error 74.000000 % epoch 17, minibatch 12/12, validation_error 74.000000 % epoch 18, minibatch 12/12, validation_error 74.000000 % epoch 19, minibatch 12/12, validation_error 74.000000 % epoch 20, minibatch 12/12, validation_error 74.000000 % epoch 21, minibatch 12/12, validation_error 74.000000 % epoch 22, minibatch 12/12, validation_error 74.000000 % epoch 23, minibatch 12/12, validation_error 74.000000 % epoch 24, minibatch 12/12, validation_error 74.000000 % epoch 25, minibatch 12/12, validation_error 74.000000 % epoch 26, minibatch 12/12, validation_error 74.000000 % epoch 27, minibatch 12/12, validation_error 74.000000 % epoch 28, minibatch 12/12, validation_error 74.000000 % epoch 29, minibatch 12/12, validation_error 74.000000 % epoch 30, minibatch 12/12, validation_error 74.000000 % epoch 31, minibatch 12/12, validation_error 74.000000 % epoch 32, minibatch 12/12, validation_error 74.000000 % epoch 33, minibatch 12/12, validation_error 74.000000 % epoch 34, minibatch 12/12, validation_error 74.000000 % epoch 35, minibatch 12/12, validation_error 74.000000 % epoch 36, minibatch 12/12, validation_error 74.000000 % epoch 37, minibatch 12/12, validation_error 74.000000 % epoch 38, minibatch 12/12, validation_error 74.000000 % epoch 39, minibatch 12/12, validation_error 74.000000 % epoch 40, minibatch 12/12, validation_error 73.333333 % epoch 41, minibatch 12/12, validation_error 73.333333 % epoch 42, minibatch 12/12, validation_error 73.333333 % epoch 43, minibatch 12/12, validation_error 72.000000 % epoch 44, minibatch 12/12, validation_error 73.333333 % epoch 45, minibatch 12/12, validation_error 73.333333 % epoch 46, minibatch 12/12, validation_error 73.333333 % epoch 47, minibatch 12/12, validation_error 74.000000 % epoch 48, minibatch 12/12, validation_error 75.333333 % epoch 49, minibatch 12/12, validation_error 76.000000 % epoch 50, minibatch 12/12, validation_error 76.000000 % epoch 51, minibatch 12/12, validation_error 76.000000 % epoch 52, minibatch 12/12, validation_error 76.000000 % epoch 53, minibatch 12/12, validation_error 76.666667 % epoch 54, minibatch 12/12, validation_error 76.666667 % epoch 55, minibatch 12/12, validation_error 77.333333 % epoch 56, minibatch 12/12, validation_error 77.333333 % epoch 57, minibatch 12/12, validation_error 76.666667 % epoch 58, minibatch 12/12, validation_error 76.666667 % epoch 59, minibatch 12/12, validation_error 76.666667 % epoch 60, minibatch 12/12, validation_error 76.666667 % epoch 61, minibatch 12/12, validation_error 77.333333 % epoch 62, minibatch 12/12, validation_error 77.333333 % epoch 63, minibatch 12/12, validation_error 77.333333 % epoch 64, minibatch 12/12, validation_error 77.333333 % epoch 65, minibatch 12/12, validation_error 77.333333 % epoch 66, minibatch 12/12, validation_error 77.333333 % epoch 67, minibatch 12/12, validation_error 78.000000 % epoch 68, minibatch 12/12, validation_error 78.666667 % epoch 69, minibatch 12/12, validation_error 78.666667 % epoch 70, minibatch 12/12, validation_error 78.666667 % epoch 71, minibatch 12/12, validation_error 78.666667 % epoch 72, minibatch 12/12, validation_error 78.666667 % epoch 73, minibatch 12/12, validation_error 78.666667 % epoch 74, minibatch 12/12, validation_error 78.666667 % epoch 75, minibatch 12/12, validation_error 78.666667 % epoch 76, minibatch 12/12, validation_error 78.666667 % epoch 77, minibatch 12/12, validation_error 78.666667 % epoch 78, minibatch 12/12, validation_error 78.666667 % epoch 79, minibatch 12/12, validation_error 78.666667 % epoch 80, minibatch 12/12, validation_error 78.666667 % epoch 81, minibatch 12/12, validation_error 78.666667 % epoch 82, minibatch 12/12, validation_error 78.666667 % epoch 83, minibatch 12/12, validation_error 78.666667 % epoch 84, minibatch 12/12, validation_error 78.666667 % epoch 85, minibatch 12/12, validation_error 78.666667 % epoch 86, minibatch 12/12, validation_error 78.666667 % epoch 87, minibatch 12/12, validation_error 78.000000 % epoch 88, minibatch 12/12, validation_error 78.666667 % epoch 89, minibatch 12/12, validation_error 78.666667 % epoch 90, minibatch 12/12, validation_error 78.666667 % epoch 91, minibatch 12/12, validation_error 78.666667 % epoch 92, minibatch 12/12, validation_error 80.000000 % epoch 93, minibatch 12/12, validation_error 80.000000 % epoch 94, minibatch 12/12, validation_error 80.000000 % epoch 95, minibatch 12/12, validation_error 80.000000 % epoch 96, minibatch 12/12, validation_error 80.000000 % epoch 97, minibatch 12/12, validation_error 80.000000 % epoch 98, minibatch 12/12, validation_error 80.000000 % epoch 99, minibatch 12/12, validation_error 80.000000 % epoch 100, minibatch 12/12, validation_error 80.000000 % optimization complete with best validation error: 72.000000 % 0.21875 0.17 (1000, 100)
## trying deep learning tutorial mnist data
train_set, validation_set, test_set = cPickle.load(open('data/mnist.pkl', 'rb'))
train_X, train_y = train_set
validation_X, validation_y = validation_set
X = np.vstack([train_X, validation_X])
print train_X.shape, train_y.shape
print validation_X.shape, validation_y.shape
print X.shape
classes = range(10)
(50000, 784) (50000,) (10000, 784) (10000,) (60000, 784)
## Denoising Auto Encoder
reload(theanoml2)
reload(theanoml2.lenet)
reload(theanoml2.formula)
reload(theanoml2.auto_encoder)
d_auto_encoder = theanoml2.auto_encoder.DenoisingAutoEncoder(n_hidden = 500, corruption_level = 0.1, n_epochs=5)
d_auto_encoder.partial_fit(X)
print d_auto_encoder.transform(X).shape
training epoch 0, train cost 99.151071 training epoch 1, train cost 79.053245 training epoch 2, train cost 74.209851 training epoch 3, train cost 71.477951
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) <ipython-input-8-bb3dfc95b231> in <module>() 5 reload(theanoml2.auto_encoder) 6 d_auto_encoder = theanoml2.auto_encoder.DenoisingAutoEncoder(n_hidden = 500, corruption_level = 0.1, n_epochs=5) ----> 7 d_auto_encoder.partial_fit(X) 8 print d_auto_encoder.transform(X).shape /Users/lima/workspace/tutorials/ml-tutorials/theanoml2/formula.pyc in partial_fit(self, X) 104 def partial_fit(self, X): 105 if self.formula_ is None: --> 106 return self.fit(X) 107 else: 108 return self._optimize(X) /Users/lima/workspace/tutorials/ml-tutorials/theanoml2/formula.pyc in fit(self, X) 101 def fit(self, X): 102 self.formula_ = self._create_formula(X) --> 103 return self._optimize(X) 104 def partial_fit(self, X): 105 if self.formula_ is None: /Users/lima/workspace/tutorials/ml-tutorials/theanoml2/formula.pyc in _optimize(self, X) 120 model_infor = build_batch_fixed_iter_model_infor(self.formula_, v_X, 121 batch_size = self.batch_size, learning_rate = self.learning_rate) --> 122 best_params = batch_fixed_iter_optimize(model_infor, n_epochs = self.n_epochs) 123 for i in xrange(len(best_params)): 124 self.formula_.params[i].set_value(best_params[i]) /Users/lima/workspace/tutorials/ml-tutorials/theanoml2/optimize.pyc in batch_fixed_iter_optimize(model_infor, n_epochs, verbose) 67 ## iterative optimization for a fixed number of iterations 68 for epoch in xrange(n_epochs): ---> 69 train_cost = np.mean([train_model(i) for i in xrange(n_train_batches)]) 70 if verbose: 71 print 'training epoch %i, train cost %f' % (epoch, train_cost) /Library/Python/2.7/site-packages/Theano-0.6.0rc3-py2.7.egg/theano/compile/function_module.pyc in __call__(self, *args, **kwargs) 578 t0_fn = time.time() 579 try: --> 580 outputs = self.fn() 581 except Exception: 582 if hasattr(self.fn, 'position_of_error'): KeyboardInterrupt: