For tqdm progress bar to work correctly, before launching this notebook run:
$ jupyter nbextension enable --py --sys-prefix widgetsnbextension
Or simply run from project root:
$ make jupyter
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier as KNN
from sklearn.metrics import accuracy_score, confusion_matrix
import env
from boltzmann_machines.rbm import BernoulliRBM
from boltzmann_machines.utils import (Stopwatch,
im_plot, plot_confusion_matrix)
from boltzmann_machines.utils.dataset import load_mnist
%matplotlib inline
%load_ext autoreload
%autoreload 2
rbm = BernoulliRBM.load_model('../models/rbm_mnist/')
weights = rbm.get_tf_params(scope='weights')
W = weights['W']
hb = weights['hb']
INFO:tensorflow:Restoring parameters from ../models/rbm_mnist/model
fig = plt.figure(figsize=(10, 10))
im_plot(W.T, shape=(28, 28), title='First 100 filters extracted by RBM',
imshow_params={'cmap': plt.cm.gray});
plt.savefig('rbm_mnist.png', dpi=196, bbox_inches='tight');
X, y = load_mnist(mode='train', path='../data/')
X /= 255.
X_test, y_test = load_mnist(mode='test', path='../data/')
X_test /= 255.
print X.shape, y.shape, X_test.shape, y_test.shape
(60000, 784) (60000,) (10000, 784) (10000,)
Q = rbm.transform(X)
Q_test = rbm.transform(X_test)
print Q.shape, Q_test.shape
INFO:tensorflow:Restoring parameters from ../models/rbm_mnist/model
A Jupyter Widget
INFO:tensorflow:Restoring parameters from ../models/rbm_mnist/model
A Jupyter Widget
(60000, 1024) (10000, 1024)
knn = KNN(n_neighbors=3, p=2., weights='distance',
n_jobs=2)
with Stopwatch(verbose=True) as s:
knn.fit(Q, y)
print accuracy_score(y_test, knn.predict(Q_test))
0.9712 Elapsed time: 438.463 sec
logreg = LogisticRegression(multi_class='multinomial', solver='sag', max_iter=800,
n_jobs=2, verbose=10, random_state=1337) # displays progress in terminal
logreg.fit(Q, y)
print accuracy_score(y_test, logreg.predict(Q_test))
convergence after 720 epochs took 1930 seconds 0.9817
[Parallel(n_jobs=2)]: Done 1 tasks | elapsed: 32.2min [Parallel(n_jobs=2)]: Done 1 out of 1 | elapsed: 32.2min finished
svc = SVC(C=1e3, tol=1e-6,
verbose=10, random_state=1337) # displays progress in terminal
with Stopwatch(verbose=True) as s:
svc.fit(Q, y)
print accuracy_score(y_test, svc.predict(Q_test))
[LibSVM]0.982 Elapsed time: 337.938 sec
y_pred = np.load('../data/rbm_y_pred.npy')
y_test = np.load('../data/rbm_y_test.npy')
W_finetuned = np.load('../data/rbm_W_finetuned.npy')
print accuracy_score(y_test, y_pred)
0.9873
C = confusion_matrix(y_test, y_pred)
fig = plt.figure(figsize=(10, 8))
ax = plot_confusion_matrix(C, fmt='d')
plt.title('Confusion matrix for fine-tuned RBM\n', fontsize=20, y=0.97)
plt.savefig('rbm_mnist_confusion_matrix.png', dpi=144, bbox_inches='tight')
fig = plt.figure(figsize=(10, 10))
im_plot(W_finetuned.T, shape=(28, 28), title='First 100 filters after fine-tuning',
imshow_params={'cmap': plt.cm.gray});
plt.savefig('rbm_mnist_finetuned.png', dpi=196, bbox_inches='tight');
A third way to use RBMs for supervised problems is to jointly model distribution of the data along with the labels $\mathbb{P}(\mathbf{x},\;\mathbf{y})$. Such models are called classification RBMs (cRBMs). Currently, these are beyond of the scope of this repository.