using AugmentedGaussianProcesses
using Distributions
using LinearAlgebra
using Plots
default(lw=3.0)
kernel = with_lengthscale(SqExponentialKernel(), 1.0)
N_train = 50
x = range(-10, 10; length=50)
x_test = range(-10, 10; length=500)
K = kernelmatrix(kernel, vcat(x, x_test))
f_all = rand(MvNormal(K + 1e-8I)) # Sample a random GP
f = f_all[1:N_train]
y = rand.(Bernoulli.(AGP.logistic.(f)))
y_sign = Int.(sign.(y .- 0.5));
function plot_data(x, y; size=(300, 500), kwargs...)
return Plots.scatter(x, y; alpha=0.5, markerstrokewidth=0.0, lab="", size=size, kwargs...)
end
plot_data(x, y; size=(600, 500), xlabel="x", ylabel="y")
@info "Running full model"
mfull = VGP(x, y_sign, kernel, LogisticLikelihood(), AnalyticVI(); optimiser=false)
@time train!(mfull, 5)
[ Info: Running full model 0.767139 seconds (1.84 M allocations: 97.731 MiB, 99.67% compilation time)
(Variational Gaussian Process with a BernoulliLikelihood{GPLikelihoods.LogisticLink}(GPLikelihoods.LogisticLink(LogExpFunctions.logistic)) infered by Analytic Variational Inference , (local_vars = (c = [0.8831610962048158, 0.7983048885763421, 0.7411179861808337, 0.7205905542174055, 0.7221377445865079, 0.7448142188811939, 0.7914378444211475, 0.8284343437766476, 0.8095400363364473, 0.7488703269720365 … 0.9041612000756052, 0.9579573239052209, 1.0207111359671175, 1.1146095217044207, 1.22784660204197, 1.3013116784278675, 1.268461966896905, 1.1189884318905525, 0.933589045988239, 0.8418220255784411], θ = [0.2349252591085695, 0.2375179788479699, 0.2391525829187937, 0.23971594806972904, 0.2396739271449476, 0.2390498067085787, 0.2377192114036301, 0.23661957013798524, 0.23718589761433848, 0.23893655927527493 … 0.23425455471260445, 0.23248701736576866, 0.2303404839029019, 0.22697407903437208, 0.22270001528671712, 0.2198210147114066, 0.22111766505081343, 0.2268129191628233, 0.2332962864373259, 0.23621241876184473]), opt_state = (NamedTuple(),), hyperopt_state = (NamedTuple(),), kernel_matrices = ((K = LinearAlgebra.Cholesky{Float64, Matrix{Float64}}([1.0000499987500624 0.9200303474974239 … 4.46855233855175e-84 1.3838273370995804e-87; 0.9200763478648182 0.3919746926572805 … 3.1154230234065755e-80 1.1397426112260664e-83; … ; 4.468775760583266e-84 1.2215781024732928e-80 … 0.09632063884559038 0.29129437964209737; 1.3838965267367376e-87 4.468775760583266e-84 … 0.9200763478648182 0.09632063884675626], 'U', 0),),)))
@info "Sampling from model"
mmcmc = MCGP(x, y, kernel, LogisticLikelihood(), GibbsSampling(); optimiser=false)
m = mmcmc
@time samples = sample(mmcmc, 1000);
[ Info: Sampling from model Sampling with Gibbs Sampler 0%| | ETA: N/A Sampling with Gibbs Sampler 0%|▏ | ETA: 0:04:53 Sampling with Gibbs Sampler 1%|▎ | ETA: 0:02:29 Sampling with Gibbs Sampler 1%|▍ | ETA: 0:01:39 Sampling with Gibbs Sampler 2%|▍ | ETA: 0:01:14 Sampling with Gibbs Sampler 2%|▌ | ETA: 0:00:59 Sampling with Gibbs Sampler 3%|▋ | ETA: 0:00:49 Sampling with Gibbs Sampler 3%|▊ | ETA: 0:00:42 Sampling with Gibbs Sampler 4%|▉ | ETA: 0:00:36 Sampling with Gibbs Sampler 4%|█ | ETA: 0:00:32 Sampling with Gibbs Sampler 5%|█ | ETA: 0:00:29 Sampling with Gibbs Sampler 5%|█▏ | ETA: 0:00:26 Sampling with Gibbs Sampler 5%|█▎ | ETA: 0:00:24 Sampling with Gibbs Sampler 6%|█▍ | ETA: 0:00:22 Sampling with Gibbs Sampler 6%|█▌ | ETA: 0:00:20 Sampling with Gibbs Sampler 7%|█▋ | ETA: 0:00:19 Sampling with Gibbs Sampler 7%|█▋ | ETA: 0:00:18 Sampling with Gibbs Sampler 8%|█▊ | ETA: 0:00:16 Sampling with Gibbs Sampler 8%|█▉ | ETA: 0:00:15 Sampling with Gibbs Sampler 9%|██ | ETA: 0:00:15 Sampling with Gibbs Sampler 9%|██▏ | ETA: 0:00:14 Sampling with Gibbs Sampler 10%|██▎ | ETA: 0:00:13 Sampling with Gibbs Sampler 10%|██▎ | ETA: 0:00:13 Sampling with Gibbs Sampler 10%|██▍ | ETA: 0:00:12 Sampling with Gibbs Sampler 11%|██▌ | ETA: 0:00:11 Sampling with Gibbs Sampler 11%|██▋ | ETA: 0:00:11 Sampling with Gibbs Sampler 12%|██▊ | ETA: 0:00:10 Sampling with Gibbs Sampler 12%|██▉ | ETA: 0:00:10 Sampling with Gibbs Sampler 13%|██▉ | ETA: 0:00:10 Sampling with Gibbs Sampler 13%|███ | ETA: 0:00:09 Sampling with Gibbs Sampler 14%|███▏ | ETA: 0:00:09 Sampling with Gibbs Sampler 14%|███▎ | ETA: 0:00:09 Sampling with Gibbs Sampler 15%|███▍ | ETA: 0:00:08 Sampling with Gibbs Sampler 15%|███▌ | ETA: 0:00:08 Sampling with Gibbs Sampler 15%|███▌ | ETA: 0:00:08 Sampling with Gibbs Sampler 16%|███▋ | ETA: 0:00:07 Sampling with Gibbs Sampler 16%|███▊ | ETA: 0:00:07 Sampling with Gibbs Sampler 17%|███▉ | ETA: 0:00:07 Sampling with Gibbs Sampler 17%|████ | ETA: 0:00:07 Sampling with Gibbs Sampler 18%|████▏ | ETA: 0:00:07 Sampling with Gibbs Sampler 18%|████▏ | ETA: 0:00:06 Sampling with Gibbs Sampler 19%|████▎ | ETA: 0:00:06 Sampling with Gibbs Sampler 19%|████▍ | ETA: 0:00:06 Sampling with Gibbs Sampler 20%|████▌ | ETA: 0:00:06 Sampling with Gibbs Sampler 20%|████▋ | ETA: 0:00:06 Sampling with Gibbs Sampler 20%|████▊ | ETA: 0:00:06 Sampling with Gibbs Sampler 21%|████▊ | ETA: 0:00:05 Sampling with Gibbs Sampler 21%|████▉ | ETA: 0:00:05 Sampling with Gibbs Sampler 22%|█████ | ETA: 0:00:05 Sampling with Gibbs Sampler 22%|█████▏ | ETA: 0:00:05 Sampling with Gibbs Sampler 23%|█████▎ | ETA: 0:00:05 Sampling with Gibbs Sampler 23%|█████▍ | ETA: 0:00:05 Sampling with Gibbs Sampler 24%|█████▍ | ETA: 0:00:05 Sampling with Gibbs Sampler 24%|█████▌ | ETA: 0:00:05 Sampling with Gibbs Sampler 25%|█████▋ | ETA: 0:00:04 Sampling with Gibbs Sampler 25%|█████▊ | ETA: 0:00:04 Sampling with Gibbs Sampler 25%|█████▉ | ETA: 0:00:04 Sampling with Gibbs Sampler 26%|██████ | ETA: 0:00:04 Sampling with Gibbs Sampler 26%|██████▏ | ETA: 0:00:04 Sampling with Gibbs Sampler 27%|██████▏ | ETA: 0:00:04 Sampling with Gibbs Sampler 27%|██████▎ | ETA: 0:00:04 Sampling with Gibbs Sampler 28%|██████▍ | ETA: 0:00:04 Sampling with Gibbs Sampler 28%|██████▌ | ETA: 0:00:04 Sampling with Gibbs Sampler 29%|██████▋ | ETA: 0:00:04 Sampling with Gibbs Sampler 29%|██████▊ | ETA: 0:00:04 Sampling with Gibbs Sampler 30%|██████▊ | ETA: 0:00:03 Sampling with Gibbs Sampler 30%|██████▉ | ETA: 0:00:03 Sampling with Gibbs Sampler 30%|███████ | ETA: 0:00:03 Sampling with Gibbs Sampler 31%|███████▏ | ETA: 0:00:03 Sampling with Gibbs Sampler 31%|███████▎ | ETA: 0:00:03 Sampling with Gibbs Sampler 32%|███████▍ | ETA: 0:00:03 Sampling with Gibbs Sampler 32%|███████▍ | ETA: 0:00:03 Sampling with Gibbs Sampler 33%|███████▌ | ETA: 0:00:03 Sampling with Gibbs Sampler 33%|███████▋ | ETA: 0:00:03 Sampling with Gibbs Sampler 34%|███████▊ | ETA: 0:00:03 Sampling with Gibbs Sampler 34%|███████▉ | ETA: 0:00:03 Sampling with Gibbs Sampler 35%|████████ | ETA: 0:00:03 Sampling with Gibbs Sampler 35%|████████ | ETA: 0:00:03 Sampling with Gibbs Sampler 35%|████████▏ | ETA: 0:00:03 Sampling with Gibbs Sampler 36%|████████▎ | ETA: 0:00:03 Sampling with Gibbs Sampler 36%|████████▍ | ETA: 0:00:03 Sampling with Gibbs Sampler 37%|████████▌ | ETA: 0:00:03 Sampling with Gibbs Sampler 37%|████████▋ | ETA: 0:00:02 Sampling with Gibbs Sampler 38%|████████▋ | ETA: 0:00:02 Sampling with Gibbs Sampler 38%|████████▊ | ETA: 0:00:02 Sampling with Gibbs Sampler 39%|████████▉ | ETA: 0:00:02 Sampling with Gibbs Sampler 39%|█████████ | ETA: 0:00:02 Sampling with Gibbs Sampler 40%|█████████▏ | ETA: 0:00:02 Sampling with Gibbs Sampler 40%|█████████▎ | ETA: 0:00:02 Sampling with Gibbs Sampler 40%|█████████▎ | ETA: 0:00:02 Sampling with Gibbs Sampler 41%|█████████▍ | ETA: 0:00:02 Sampling with Gibbs Sampler 41%|█████████▌ | ETA: 0:00:02 Sampling with Gibbs Sampler 42%|█████████▋ | ETA: 0:00:02 Sampling with Gibbs Sampler 42%|█████████▊ | ETA: 0:00:02 Sampling with Gibbs Sampler 43%|█████████▉ | ETA: 0:00:02 Sampling with Gibbs Sampler 43%|█████████▉ | ETA: 0:00:02 Sampling with Gibbs Sampler 44%|██████████ | ETA: 0:00:02 Sampling with Gibbs Sampler 44%|██████████▏ | ETA: 0:00:02 Sampling with Gibbs Sampler 45%|██████████▎ | ETA: 0:00:02 Sampling with Gibbs Sampler 45%|██████████▍ | ETA: 0:00:02 Sampling with Gibbs Sampler 45%|██████████▌ | ETA: 0:00:02 Sampling with Gibbs Sampler 46%|██████████▌ | ETA: 0:00:02 Sampling with Gibbs Sampler 46%|██████████▋ | ETA: 0:00:02 Sampling with Gibbs Sampler 47%|██████████▊ | ETA: 0:00:02 Sampling with Gibbs Sampler 47%|██████████▉ | ETA: 0:00:02 Sampling with Gibbs Sampler 48%|███████████ | ETA: 0:00:02 Sampling with Gibbs Sampler 48%|███████████▏ | ETA: 0:00:02 Sampling with Gibbs Sampler 49%|███████████▏ | ETA: 0:00:02 Sampling with Gibbs Sampler 49%|███████████▎ | ETA: 0:00:02 Sampling with Gibbs Sampler 50%|███████████▍ | ETA: 0:00:02 Sampling with Gibbs Sampler 50%|███████████▌ | ETA: 0:00:01 Sampling with Gibbs Sampler 50%|███████████▋ | ETA: 0:00:01 Sampling with Gibbs Sampler 51%|███████████▊ | ETA: 0:00:01 Sampling with Gibbs Sampler 51%|███████████▉ | ETA: 0:00:01 Sampling with Gibbs Sampler 52%|███████████▉ | ETA: 0:00:01 Sampling with Gibbs Sampler 52%|████████████ | ETA: 0:00:01 Sampling with Gibbs Sampler 53%|████████████▏ | ETA: 0:00:01 Sampling with Gibbs Sampler 53%|████████████▎ | ETA: 0:00:01 Sampling with Gibbs Sampler 54%|████████████▍ | ETA: 0:00:01 Sampling with Gibbs Sampler 54%|████████████▌ | ETA: 0:00:01 Sampling with Gibbs Sampler 55%|████████████▌ | ETA: 0:00:01 Sampling with Gibbs Sampler 55%|████████████▋ | ETA: 0:00:01 Sampling with Gibbs Sampler 55%|████████████▊ | ETA: 0:00:01 Sampling with Gibbs Sampler 56%|████████████▉ | ETA: 0:00:01 Sampling with Gibbs Sampler 56%|█████████████ | ETA: 0:00:01 Sampling with Gibbs Sampler 57%|█████████████▏ | ETA: 0:00:01 Sampling with Gibbs Sampler 57%|█████████████▏ | ETA: 0:00:01 Sampling with Gibbs Sampler 58%|█████████████▎ | ETA: 0:00:01 Sampling with Gibbs Sampler 58%|█████████████▍ | ETA: 0:00:01 Sampling with Gibbs Sampler 59%|█████████████▌ | ETA: 0:00:01 Sampling with Gibbs Sampler 59%|█████████████▋ | ETA: 0:00:01 Sampling with Gibbs Sampler 60%|█████████████▊ | ETA: 0:00:01 Sampling with Gibbs Sampler 60%|█████████████▊ | ETA: 0:00:01 Sampling with Gibbs Sampler 60%|█████████████▉ | ETA: 0:00:01 Sampling with Gibbs Sampler 61%|██████████████ | ETA: 0:00:01 Sampling with Gibbs Sampler 61%|██████████████▏ | ETA: 0:00:01 Sampling with Gibbs Sampler 62%|██████████████▎ | ETA: 0:00:01 Sampling with Gibbs Sampler 62%|██████████████▍ | ETA: 0:00:01 Sampling with Gibbs Sampler 63%|██████████████▍ | ETA: 0:00:01 Sampling with Gibbs Sampler 63%|██████████████▌ | ETA: 0:00:01 Sampling with Gibbs Sampler 64%|██████████████▋ | ETA: 0:00:01 Sampling with Gibbs Sampler 64%|██████████████▊ | ETA: 0:00:01 Sampling with Gibbs Sampler 65%|██████████████▉ | ETA: 0:00:01 Sampling with Gibbs Sampler 65%|███████████████ | ETA: 0:00:01 Sampling with Gibbs Sampler 65%|███████████████ | ETA: 0:00:01 Sampling with Gibbs Sampler 66%|███████████████▏ | ETA: 0:00:01 Sampling with Gibbs Sampler 66%|███████████████▎ | ETA: 0:00:01 Sampling with Gibbs Sampler 67%|███████████████▍ | ETA: 0:00:01 Sampling with Gibbs Sampler 67%|███████████████▌ | ETA: 0:00:01 Sampling with Gibbs Sampler 68%|███████████████▋ | ETA: 0:00:01 Sampling with Gibbs Sampler 68%|███████████████▋ | ETA: 0:00:01 Sampling with Gibbs Sampler 69%|███████████████▊ | ETA: 0:00:01 Sampling with Gibbs Sampler 69%|███████████████▉ | ETA: 0:00:01 Sampling with Gibbs Sampler 70%|████████████████ | ETA: 0:00:01 Sampling with Gibbs Sampler 70%|████████████████▏ | ETA: 0:00:01 Sampling with Gibbs Sampler 70%|████████████████▎ | ETA: 0:00:01 Sampling with Gibbs Sampler 71%|████████████████▎ | ETA: 0:00:01 Sampling with Gibbs Sampler 71%|████████████████▍ | ETA: 0:00:01 Sampling with Gibbs Sampler 72%|████████████████▌ | ETA: 0:00:01 Sampling with Gibbs Sampler 72%|████████████████▋ | ETA: 0:00:01 Sampling with Gibbs Sampler 73%|████████████████▊ | ETA: 0:00:01 Sampling with Gibbs Sampler 73%|████████████████▉ | ETA: 0:00:01 Sampling with Gibbs Sampler 74%|████████████████▉ | ETA: 0:00:01 Sampling with Gibbs Sampler 74%|█████████████████ | ETA: 0:00:01 Sampling with Gibbs Sampler 75%|█████████████████▏ | ETA: 0:00:01 Sampling with Gibbs Sampler 75%|█████████████████▎ | ETA: 0:00:01 Sampling with Gibbs Sampler 75%|█████████████████▍ | ETA: 0:00:01 Sampling with Gibbs Sampler 76%|█████████████████▌ | ETA: 0:00:01 Sampling with Gibbs Sampler 76%|█████████████████▋ | ETA: 0:00:00 Sampling with Gibbs Sampler 77%|█████████████████▋ | ETA: 0:00:00 Sampling with Gibbs Sampler 77%|█████████████████▊ | ETA: 0:00:00 Sampling with Gibbs Sampler 78%|█████████████████▉ | ETA: 0:00:00 Sampling with Gibbs Sampler 78%|██████████████████ | ETA: 0:00:00 Sampling with Gibbs Sampler 79%|██████████████████▏ | ETA: 0:00:00 Sampling with Gibbs Sampler 79%|██████████████████▎ | ETA: 0:00:00 Sampling with Gibbs Sampler 80%|██████████████████▎ | ETA: 0:00:00 Sampling with Gibbs Sampler 80%|██████████████████▍ | ETA: 0:00:00 Sampling with Gibbs Sampler 80%|██████████████████▌ | ETA: 0:00:00 Sampling with Gibbs Sampler 81%|██████████████████▋ | ETA: 0:00:00 Sampling with Gibbs Sampler 81%|██████████████████▊ | ETA: 0:00:00 Sampling with Gibbs Sampler 82%|██████████████████▉ | ETA: 0:00:00 Sampling with Gibbs Sampler 82%|██████████████████▉ | ETA: 0:00:00 Sampling with Gibbs Sampler 83%|███████████████████ | ETA: 0:00:00 Sampling with Gibbs Sampler 83%|███████████████████▏ | ETA: 0:00:00 Sampling with Gibbs Sampler 84%|███████████████████▎ | ETA: 0:00:00 Sampling with Gibbs Sampler 84%|███████████████████▍ | ETA: 0:00:00 Sampling with Gibbs Sampler 85%|███████████████████▌ | ETA: 0:00:00 Sampling with Gibbs Sampler 85%|███████████████████▌ | ETA: 0:00:00 Sampling with Gibbs Sampler 85%|███████████████████▋ | ETA: 0:00:00 Sampling with Gibbs Sampler 86%|███████████████████▊ | ETA: 0:00:00 Sampling with Gibbs Sampler 86%|███████████████████▉ | ETA: 0:00:00 Sampling with Gibbs Sampler 87%|████████████████████ | ETA: 0:00:00 Sampling with Gibbs Sampler 87%|████████████████████▏ | ETA: 0:00:00 Sampling with Gibbs Sampler 88%|████████████████████▏ | ETA: 0:00:00 Sampling with Gibbs Sampler 88%|████████████████████▎ | ETA: 0:00:00 Sampling with Gibbs Sampler 89%|████████████████████▍ | ETA: 0:00:00 Sampling with Gibbs Sampler 89%|████████████████████▌ | ETA: 0:00:00 Sampling with Gibbs Sampler 90%|████████████████████▋ | ETA: 0:00:00 Sampling with Gibbs Sampler 90%|████████████████████▊ | ETA: 0:00:00 Sampling with Gibbs Sampler 90%|████████████████████▊ | ETA: 0:00:00 Sampling with Gibbs Sampler 91%|████████████████████▉ | ETA: 0:00:00 Sampling with Gibbs Sampler 91%|█████████████████████ | ETA: 0:00:00 Sampling with Gibbs Sampler 92%|█████████████████████▏ | ETA: 0:00:00 Sampling with Gibbs Sampler 92%|█████████████████████▎ | ETA: 0:00:00 Sampling with Gibbs Sampler 93%|█████████████████████▍ | ETA: 0:00:00 Sampling with Gibbs Sampler 93%|█████████████████████▍ | ETA: 0:00:00 Sampling with Gibbs Sampler 94%|█████████████████████▌ | ETA: 0:00:00 Sampling with Gibbs Sampler 94%|█████████████████████▋ | ETA: 0:00:00 Sampling with Gibbs Sampler 95%|█████████████████████▊ | ETA: 0:00:00 Sampling with Gibbs Sampler 95%|█████████████████████▉ | ETA: 0:00:00 Sampling with Gibbs Sampler 95%|██████████████████████ | ETA: 0:00:00 Sampling with Gibbs Sampler 96%|██████████████████████ | ETA: 0:00:00 Sampling with Gibbs Sampler 96%|██████████████████████▏| ETA: 0:00:00 Sampling with Gibbs Sampler 97%|██████████████████████▎| ETA: 0:00:00 Sampling with Gibbs Sampler 97%|██████████████████████▍| ETA: 0:00:00 Sampling with Gibbs Sampler 98%|██████████████████████▌| ETA: 0:00:00 Sampling with Gibbs Sampler 98%|██████████████████████▋| ETA: 0:00:00 Sampling with Gibbs Sampler 99%|██████████████████████▋| ETA: 0:00:00 Sampling with Gibbs Sampler 99%|██████████████████████▊| ETA: 0:00:00 Sampling with Gibbs Sampler 100%|██████████████████████▉| ETA: 0:00:00 Sampling with Gibbs Sampler 100%|███████████████████████| Time: 0:00:01 Sampling with Gibbs Sampler 100%|███████████████████████| Time: 0:00:02 4.811782 seconds (9.68 M allocations: 682.888 MiB, 2.68% gc time, 94.01% compilation time)
p1 = plot(x, samples; label="", color=:black, alpha=0.01, lab="")
plot!(x, mean(mfull[1]); color=:blue, ribbon=sqrt.(var(mfull[1])), label="VI")
plot!(x_test, f_all[N_train+1:end]; color=:red, label="true f")
p2 = plot_data(x, y; size=(600, 400))
μ_vi, σ_vi = proba_y(mfull, x_test)
plot!(x_test, μ_vi; ribbon=σ_vi, label="VI")
μ_mcmc, σ_mcmc = proba_y(mmcmc, x_test)
plot!(x_test, μ_mcmc; ribbon=σ_mcmc, label="MCMC")
This notebook was generated using Literate.jl.