# import all shogun classes
from modshogun import *
import scipy
import scipy.io
import numpy as np
import time
data=scipy.io.loadmat('./banana.mat')#('./usps3v5.mat')
x_train=data['X_training']
y_train=np.int_(data['y_training'].squeeze())
x_test=data['X_testing']
y_test=np.int_(data['y_testing'].squeeze())
#shogun's GPs
def get_inference(inference, kernel_func, features_train, mean_func, labels_train, likelihood, kernel_scale):
inf=inference(kernel_func, features_train, mean_func, labels_train, likelihood)
inf.set_scale(kernel_scale)
return inf
mean_func = ZeroMean()
kernel_sigma = 1.0
kernel_func = GaussianKernel(10, 2.0*kernel_sigma)
labels_train = BinaryLabels(y_train)
labels_test = BinaryLabels(y_test)
features_train=RealFeatures(x_train)
features_test=RealFeatures(x_test)
kernel_scale = 1.0
likelihood = LogitLikelihood()
start = time.time()
inf = get_inference(SingleLaplacianInferenceMethod, kernel_func, features_train, mean_func, labels_train, likelihood, kernel_scale)
inf.set_newton_iterations(100)
gp = GaussianProcessClassification(inf)
gp.train()
nlz=inf.get_negative_log_marginal_likelihood()
end = time.time()
print '%s training time:%.4f, nlz:%.6f'%(inf.get_name(), end-start, nlz),
start = time.time()
pred_labels_test = gp.apply_binary(features_test)
end = time.time()
print 'testing time:%.4f'%(end-start)
inf = get_inference(SingleLaplacianInferenceMethodWithLBFGS, kernel_func, features_train, mean_func, labels_train, likelihood, kernel_scale)
inf.set_lbfgs_parameters(100,80,3,80)
gp = GaussianProcessClassification(inf)
start = time.time()
gp.train()
nlz=inf.get_negative_log_marginal_likelihood()
end = time.time()
print '%s training time:%.4f, nlz:%.6f'%(inf.get_name(), end-start, nlz),
start = time.time()
pred_labels_test = gp.apply_binary(features_test)
end = time.time()
print 'testing time:%.4f'%(end-start)
SingleLaplacianInferenceMethod training time:5.0432, nlz:873.259883 testing time:3.3786 SingleLaplacianInferenceMethodWithLBFGS training time:1.9962, nlz:873.682773 testing time:3.1725
#sklearn's (0.18dev) GP
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
x_train2=x_train.T
x_test2=x_test.T
start = time.time()
clf = GaussianProcessClassifier(kernel=RBF(length_scale=1.0), warm_start=False, optimizer=None, n_jobs=-1,copy_X_train=False, max_iter_predict=100)
clf.fit(x_train2, y_train)
nlz, _=clf.log_marginal_likelihood(clf.kernel_.theta, eval_gradient=True)
end = time.time()
print 'training time:%.4f, nlz:%.6f'%(end-start, -nlz),
start = time.time()
pred = clf.predict_proba(x_test2)
end = time.time()
print 'testing time:%.4f'%(end-start)
training time:16.7133, nlz:873.259883 testing time:2.0571
#GPML 3.6 (Maltab 2015b Unix 64bit)
#load('usps3v5.mat')
#X_train=X_training';
#Y_train=y_training';
#X_test=X_testing';
#Y_test=y_testing';
#covfuncF = @covSEiso;
#meanfunc = @meanConst;
#hyp.mean = 0;
#likfunc = @likLogistic;
#hyp.lik=[];
#hyp.cov=[0.0,0.0];
#hyp.snu2=0;
#tic
#[nlZ, dnlZ] = gp(hyp, @infLaplace, meanfunc, covfuncF, likfunc, X_train, Y_train);
#toc
#nlZ
#note that if you run the Matlab code several times, Matlab uses cache to boost the performance.
import GPy
/home/wlin/python-cpu/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment. warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')
k = GPy.kern.RBF(x_train.shape[0], variance=1.0, lengthscale=1.0)
lik = GPy.likelihoods.Bernoulli() #gpy uses Bernoulli-Probit likelihood
vfunc = np.vectorize( lambda x: max(x,0) ) #gpy uses {0,1} instead of {-1,1}
x_train2=x_train.T
y_train2=vfunc(y_train).reshape(-1,1)
likelihood = GPy.likelihoods.Bernoulli()
laplace_inf = GPy.inference.latent_function_inference.Laplace()
start = time.time()
m = GPy.core.GP(x_train2, y_train2, kernel=k, likelihood=likelihood, inference_method=laplace_inf)
nlz = m.log_likelihood()
end = time.time()
print 'training time:%.4f, nlz:%.6f'%(end-start, -nlz)
training time:51.4373, nlz:743.695878
mean_func = ZeroMean()
kernel_sigma = 1.0
kernel_func = GaussianKernel(10, 2.0*kernel_sigma)
labels_train = BinaryLabels(y_train)
labels_test = BinaryLabels(y_test)
features_train=RealFeatures(x_train)
features_test=RealFeatures(x_test)
kernel_scale = 1.0
likelihood = ProbitLikelihood() #gpy uses Bernoulli-Probit likelihood
inf = get_inference(SingleLaplacianInferenceMethod, kernel_func, features_train, mean_func, labels_train, likelihood, kernel_scale)
inf.set_newton_iterations(100)
gp = GaussianProcessClassification(inf)
start = time.time()
gp.train()
nlz=inf.get_negative_log_marginal_likelihood()
end = time.time()
print '%s training time:%.4f, nlz:%.6f'%(inf.get_name(), end-start, nlz)
SingleLaplacianInferenceMethod training time:5.9478, nlz:743.695878
data=scipy.io.loadmat('8L.mat')
x_train=data['X_training']
y_train=data['y_training'].squeeze()
When the number of inducing points is 100
n_inducing=100
x_inducing=[]
np.random.seed(8)
for idx in xrange( x_train.shape[0]):
x_f_inducing = np.random.rand(n_inducing)*(x_train[idx,:].max()-x_train[idx,:].min())+x_train[idx,:].min()
x_inducing.append(x_f_inducing)
x_inducing=np.row_stack(x_inducing)
VarDTC method
feats_train=RealFeatures(x_train)
labels_train=RegressionLabels(y_train)
feats_inducing=RealFeatures(x_inducing)
mean_func = ZeroMean()
kernel_sigma = 1.0
likelihood = GaussianLikelihood()
kernel_func = GaussianKernel(10, 2.0*kernel_sigma)
inf = SparseVGInferenceMethod(kernel_func, feats_train, mean_func, labels_train, likelihood, feats_inducing)
gp = GaussianProcessRegression(inf)
start=time.time()
gp.train()
nlz=inf.get_negative_log_marginal_likelihood()
print "%s took %.2f seconds, nlz:%.6f" % ( inf.get_name(), time.time()-start, nlz)
SparseVGInferenceMethod took 0.24 seconds, nlz:60374864433908.093750
/home/wlin/python-cpu/lib/python2.7/site-packages/ipykernel/__main__.py:9: RuntimeWarning:[WARN] In file /home/wlin/dev/shogun-shogun_4.1.0/src/shogun/machine/gp/SingleSparseInferenceBase.cpp line 103: The provided kernel does not support to optimize inducing features
k = GPy.kern.RBF(x_train.shape[0], variance=1.0, lengthscale=1.0)
x_train2=x_train.T
y_train2=y_train.reshape(-1,1)
likelihood = GPy.likelihoods.Gaussian()
z_train2=x_inducing.T
inf = GPy.inference.latent_function_inference.VarDTC()
#inf = GPy.inference.latent_function_inference.FITC()
start = time.time()
m = GPy.core.SparseGP(x_train2, y_train2, z_train2, kernel=k, likelihood=likelihood, inference_method=inf)
nlz = m.log_likelihood()
end = time.time()
print 'training time:%.4f, nlz:%.6f'%(end-start, -nlz)
training time:0.4778, nlz:60374864433908.093750
FITC method
feats_train=RealFeatures(x_train)
labels_train=RegressionLabels(y_train)
feats_inducing=RealFeatures(x_inducing)
mean_func = ZeroMean()
kernel_sigma = 1.0
likelihood = GaussianLikelihood()
kernel_func = GaussianKernel(10, 2.0*kernel_sigma)
inf = FITCInferenceMethod(kernel_func, feats_train, mean_func, labels_train, likelihood, feats_inducing)
gp = GaussianProcessRegression(inf)
start=time.time()
gp.train()
nlz=inf.get_negative_log_marginal_likelihood()
print "%s took %.2f seconds, nlz:%.6f" % ( inf.get_name(), time.time()-start, nlz)
FITCInferenceMethod took 0.26 seconds, nlz:30187432229622.925781
/home/wlin/python-cpu/lib/python2.7/site-packages/ipykernel/__main__.py:9: RuntimeWarning:[WARN] In file /home/wlin/dev/shogun-shogun_4.1.0/src/shogun/machine/gp/SingleSparseInferenceBase.cpp line 103: The provided kernel does not support to optimize inducing features
k = GPy.kern.RBF(x_train.shape[0], variance=1.0, lengthscale=1.0)
x_train2=x_train.T
y_train2=y_train.reshape(-1,1)
likelihood = GPy.likelihoods.Gaussian()
z_train2=x_inducing.T
inf = GPy.inference.latent_function_inference.FITC()
start = time.time()
m = GPy.core.SparseGP(x_train2, y_train2, z_train2, kernel=k, likelihood=likelihood, inference_method=inf)
nlz = m.log_likelihood()
end = time.time()
print 'training time:%.4f, nlz:%.6f'%(end-start, -nlz)
training time:0.6308, nlz:30187432229622.929688
When the number of inducing points is 1000
n_inducing=1000
x_inducing=[]
np.random.seed(34)
for idx in xrange( x_train.shape[0]):
x_f_inducing = np.random.rand(n_inducing)*(x_train[idx,:].max()-x_train[idx,:].min())+x_train[idx,:].min()
x_inducing.append(x_f_inducing)
x_inducing=np.row_stack(x_inducing)
VarDTC method
feats_train=RealFeatures(x_train)
labels_train=RegressionLabels(y_train)
feats_inducing=RealFeatures(x_inducing)
mean_func = ZeroMean()
kernel_sigma = 1.0
likelihood = GaussianLikelihood()
kernel_func = GaussianKernel(10, 2.0*kernel_sigma)
inf = SparseVGInferenceMethod(kernel_func, feats_train, mean_func, labels_train, likelihood, feats_inducing)
gp = GaussianProcessRegression(inf)
start=time.time()
gp.train()
nlz=inf.get_negative_log_marginal_likelihood()
print "%s took %.2f seconds, nlz:%.6f" % ( inf.get_name(), time.time()-start, nlz)
SparseVGInferenceMethod took 8.23 seconds, nlz:60374861496783.296875
/home/wlin/python-cpu/lib/python2.7/site-packages/ipykernel/__main__.py:9: RuntimeWarning:[WARN] In file /home/wlin/dev/shogun-shogun_4.1.0/src/shogun/machine/gp/SingleSparseInferenceBase.cpp line 103: The provided kernel does not support to optimize inducing features
k = GPy.kern.RBF(x_train.shape[0], variance=1.0, lengthscale=1.0)
x_train2=x_train.T
y_train2=y_train.reshape(-1,1)
likelihood = GPy.likelihoods.Gaussian()
z_train2=x_inducing.T
inf = GPy.inference.latent_function_inference.VarDTC()
start = time.time()
m = GPy.core.SparseGP(x_train2, y_train2, z_train2, kernel=k, likelihood=likelihood, inference_method=inf)
nlz = m.log_likelihood()
end = time.time()
print 'training time:%.4f, nlz:%.6f'%(end-start, -nlz)
training time:8.2890, nlz:60374861496783.148438
FITC method
feats_train=RealFeatures(x_train)
labels_train=RegressionLabels(y_train)
feats_inducing=RealFeatures(x_inducing)
mean_func = ZeroMean()
kernel_sigma = 1.0
likelihood = GaussianLikelihood()
kernel_func = GaussianKernel(10, 2.0*kernel_sigma)
inf = FITCInferenceMethod(kernel_func, feats_train, mean_func, labels_train, likelihood, feats_inducing)
gp = GaussianProcessRegression(inf)
start=time.time()
gp.train()
nlz=inf.get_negative_log_marginal_likelihood()
print "%s took %.2f seconds, nlz:%.6f" % ( inf.get_name(), time.time()-start, nlz)
FITCInferenceMethod took 8.68 seconds, nlz:30187432223269.121094
/home/wlin/python-cpu/lib/python2.7/site-packages/ipykernel/__main__.py:9: RuntimeWarning:[WARN] In file /home/wlin/dev/shogun-shogun_4.1.0/src/shogun/machine/gp/SingleSparseInferenceBase.cpp line 103: The provided kernel does not support to optimize inducing features
k = GPy.kern.RBF(x_train.shape[0], variance=1.0, lengthscale=1.0)
x_train2=x_train.T
y_train2=y_train.reshape(-1,1)
likelihood = GPy.likelihoods.Gaussian()
z_train2=x_inducing.T
inf = GPy.inference.latent_function_inference.FITC()
start = time.time()
m = GPy.core.SparseGP(x_train2, y_train2, z_train2, kernel=k, likelihood=likelihood, inference_method=inf)
nlz = m.log_likelihood()
end = time.time()
print 'training time:%.4f, nlz:%.6f'%(end-start, -nlz)
training time:12.6938, nlz:30187432223269.128906