using Plots
#Make plots look like Bishop
Plots.default(markerstrokewidth=1.2, markerstrokecolor=:blue,
markeralpha=0, alpha=1, dpi=400)
pyplot()
Plots.PyPlotBackend()
using Distributions
using Printf
using LaTeXStrings
include("../src/linear_regression.jl")
include("../src/features.jl")
include("../src/data.jl")
include("../src/bayesian_regression.jl")
predict (generic function with 2 methods)
Some examples of basis functions.
μs = range(-1, 1, length=11)
x = range(-1, 1, length=100)
s = 0.05
p = plot(legend=false)
for μ in μs
plot!(x, x -> S(x, μ, s))
end
title!("Sigmoidal basis functions")
p
μs = range(-1, 1, length=11)
x = range(-1, 1, length=1000)
s = 0.05
p = plot(legend=false)
for μ in μs
plot!(x, x -> G(x, μ, s))
end
title!("Gaussian basis functions")
p
μs = range(-1, 1, length=11)
x = range(-1, 1, length=1000)
p = plot(legend=false)
degree = 10
for i in 1:degree
plot!(x, x -> x^i)
end
title!("Polynomial basis functions")
p
Fitting a sine function (and added noise) with a 9th order polynomial.
# Fitting sine function with a polynomial of order 9
# Multiple data points(N) are used
N = 15
X, y = make_sine(N)
Φ = poly_expansion(X, degree=9);
lm = LinearRegression()
fit!(lm, Φ, y)
x_range = range(0, 1, length=1000)
preds = predict(lm, poly_expansion(x_range, degree=9))
p1 = scatter(X, y, legend=false)
plot!(x_range, preds)
plot!(x -> sin(2π*x), color=:green)
xlims!(0, 1)
ylims!(-2, 2)
annotate!(0.8, 1, "N = $N")
N = 100
X, y = make_sine(N)
Φ = poly_expansion(X, degree=9);
lm = LinearRegression()
fit!(lm, Φ, y)
x_range = range(0, 1, length=1000)
preds = predict(lm, poly_expansion(x_range, degree=9))
p2 = scatter(X, y, legend=false)
plot!(x_range, preds)
plot!(x -> sin(2π*x), color=:green)
xlims!(0, 1)
ylims!(-2, 2)
annotate!(0.8, 1, "N = $N")
plot(p1, p2, layout=(1, 2), size=(800, 300), dpi=400)
#savefig("poly_fitting.png")
Generate some datasets to ilustrate the bias-variance decomposition
L = 100 # No. of datasets
N = 25 # Points in each dataset
datasets = []
for _ in 1:L
X, y = make_sine(N)
push!(datasets, [X, y])
end
Let's see one example of these datasets
ds = datasets[rand(1:L)]
scatter(ds[1], ds[2], legend=false)
plot!(x -> sin(2π*x))
Now, define a model to fit the data
# To generate the figure 3.5 from Bishop
M = 24
μs = range(0, 1, length=M)
s = 0.05
lambda_range = [exp(2.6), exp(-0.31), exp(-2.4)]
plots = []
for λ in lambda_range
models = []
for D in datasets
lin_reg = LinearRegression(λ=λ)
X, y = D
Φ = gaussian_expansion(X, μs=μs, s=s)
fit!(lin_reg, Φ, y)
push!(models, lin_reg)
end
X_ = range(0, 1, length=200)
Φ_ = gaussian_expansion(X_, μs=μs, s=s)
p = plot(dpi=100)
for model in models[rand(1:length(models), 20)]
preds = predict(model, Φ_)
plot!(X_, preds, color=:red, label="")
end
annotate!(.75, .7, L"ln $\lambda$ =" * @sprintf("%2.2f", log(λ)))
ylims!(-1.8, 1.8)
push!(plots, p)
p = plot(dpi=100)
y_av = mean([predict(model, Φ_) for model in models])
plot!(X_, y_av, color=:red, label="Average")
plot!(X_, sine, color=:green, label="h(x)")
push!(plots, p)
end
plot(plots..., layout=(3, 2), size=(1200, 1600))
#savefig("bivar_model_complexity.png")
X, y = make_sine(100)
# Model to fit
M = 10
s = 0.1
μs = range(0, 1, length=M)
Φ = gaussian_expansion(X, μs=μs, s=s)
br = BayesianRegression()
fit!(br, Φ, y)
X_ = range(0, 1, length=100)
Φ_ = gaussian_expansion(X_, μs=μs, s=s)
preds, vars = predict(br, Φ_)
scatter(X, y, legend=false)
plot!(X_, preds, ribbon=sqrt.(vars), fillcolor=:red, fillalpha=0.3, color=:red)
plot!(sine, color=:lightgreen)