In [1]:
# Generate data for SVM classifier with L1 regularization.
srand(1);
n = 20;
m = 1000;
TEST = m;
DENSITY = 0.2;
beta_true = randn(n,1);
idxs = randperm(n)[1:int((1-DENSITY)*n)];
for idx in idxs
    beta_true[idx] = 0;
end
offset = 0;
sigma = 45;
X = 5 * randn(m, n);
Y = sign(X * beta_true + offset + sigma * randn(m,1));
X_test = 5 * randn(TEST, n);
In [ ]:
# Form SVM with L1 regularization problem.
using Convex, SCS
set_default_solver(SCSSolver(verbose=0));
beta = Variable(n);
v = Variable();
loss = sum(pos(1 - Y .* (X*beta - v)));
reg = norm(beta, 1);

# Compute a trade-off curve and record train and test error.
TRIALS = 100
train_error = zeros(TRIALS);
test_error = zeros(TRIALS);
lambda_vals = logspace(-2, 0, TRIALS);
beta_vals = zeros(length(beta), TRIALS);
for i = 1:TRIALS
    lambda = lambda_vals[i];
    problem = minimize(loss/m + lambda*reg);
    solve!(problem);
    train_error[i] = sum(float(sign(X*beta_true + offset) .!= sign(evaluate(X*beta - v))))/m;
    test_error[i] = sum(float(sign(X_test*beta_true + offset) .!= sign(evaluate(X_test*beta - v))))/TEST;
    beta_vals[:, i] =  evaluate(beta);
end
In [3]:
# Plot the train and test error over the trade-off curve.
using Gadfly, DataFrames
df1 = DataFrame(λ=lambda_vals, errors=train_error, label="Train error");
df2 = DataFrame(λ=lambda_vals, errors=test_error, label="Test error");
df = vcat(df1, df2);

plot(df, x="λ", y="errors", color="label", Geom.line,
    Scale.x_log10, Scale.y_log10)
Out[3]:
λ 10-4.5 10-4.0 10-3.5 10-3.0 10-2.5 10-2.0 10-1.5 10-1.0 10-0.5 100.0 100.5 101.0 101.5 102.0 102.5 10-4.0 10-3.9 10-3.8 10-3.7 10-3.6 10-3.5 10-3.4 10-3.3 10-3.2 10-3.1 10-3.0 10-2.9 10-2.8 10-2.7 10-2.6 10-2.5 10-2.4 10-2.3 10-2.2 10-2.1 10-2.0 10-1.9 10-1.8 10-1.7 10-1.6 10-1.5 10-1.4 10-1.3 10-1.2 10-1.1 10-1.0 10-0.9 10-0.8 10-0.7 10-0.6 10-0.5 10-0.4 10-0.3 10-0.2 10-0.1 100.0 100.1 100.2 100.3 100.4 100.5 100.6 100.7 100.8 100.9 101.0 101.1 101.2 101.3 101.4 101.5 101.6 101.7 101.8 101.9 102.0 10-4 10-2 100 102 10-4.0 10-3.8 10-3.6 10-3.4 10-3.2 10-3.0 10-2.8 10-2.6 10-2.4 10-2.2 10-2.0 10-1.8 10-1.6 10-1.4 10-1.2 10-1.0 10-0.8 10-0.6 10-0.4 10-0.2 100.0 100.2 100.4 100.6 100.8 101.0 101.2 101.4 101.6 101.8 102.0 Train error Test error label 10-1.7 10-1.6 10-1.5 10-1.4 10-1.3 10-1.2 10-1.1 10-1.0 10-0.9 10-0.8 10-0.7 10-0.6 10-0.5 10-0.4 10-0.3 10-0.2 10-0.1 100.0 10-1.60 10-1.58 10-1.56 10-1.54 10-1.52 10-1.50 10-1.48 10-1.46 10-1.44 10-1.42 10-1.40 10-1.38 10-1.36 10-1.34 10-1.32 10-1.30 10-1.28 10-1.26 10-1.24 10-1.22 10-1.20 10-1.18 10-1.16 10-1.14 10-1.12 10-1.10 10-1.08 10-1.06 10-1.04 10-1.02 10-1.00 10-0.98 10-0.96 10-0.94 10-0.92 10-0.90 10-0.88 10-0.86 10-0.84 10-0.82 10-0.80 10-0.78 10-0.76 10-0.74 10-0.72 10-0.70 10-0.68 10-0.66 10-0.64 10-0.62 10-0.60 10-0.58 10-0.56 10-0.54 10-0.52 10-0.50 10-0.48 10-0.46 10-0.44 10-0.42 10-0.40 10-0.38 10-0.36 10-0.34 10-0.32 10-0.30 10-0.28 10-0.26 10-0.24 10-0.22 10-0.20 10-0.18 10-0.16 10-0.14 10-0.12 10-0.10 10-0.08 10-0.06 10-0.04 10-0.02 100.00 10-2.0 10-1.5 10-1.0 10-0.5 100.0 10-1.60 10-1.55 10-1.50 10-1.45 10-1.40 10-1.35 10-1.30 10-1.25 10-1.20 10-1.15 10-1.10 10-1.05 10-1.00 10-0.95 10-0.90 10-0.85 10-0.80 10-0.75 10-0.70 10-0.65 10-0.60 10-0.55 10-0.50 10-0.45 10-0.40 10-0.35 10-0.30 10-0.25 10-0.20 10-0.15 10-0.10 10-0.05 100.00 errors
In [4]:
# Plot the regularization path for beta.
df = DataFrame(λ=lambda_vals, betas=vec(beta_vals[1,:]), label="beta1")
for i=2:n
    df = vcat(df, DataFrame(λ=lambda_vals, betas=vec(beta_vals[i,:]), label=string("beta", i)));
end
plot(df, x="λ", y="betas", color="label", Geom.line, Scale.x_log10)
Out[4]:
λ 10-4.5 10-4.0 10-3.5 10-3.0 10-2.5 10-2.0 10-1.5 10-1.0 10-0.5 100.0 100.5 101.0 101.5 102.0 102.5 10-4.0 10-3.9 10-3.8 10-3.7 10-3.6 10-3.5 10-3.4 10-3.3 10-3.2 10-3.1 10-3.0 10-2.9 10-2.8 10-2.7 10-2.6 10-2.5 10-2.4 10-2.3 10-2.2 10-2.1 10-2.0 10-1.9 10-1.8 10-1.7 10-1.6 10-1.5 10-1.4 10-1.3 10-1.2 10-1.1 10-1.0 10-0.9 10-0.8 10-0.7 10-0.6 10-0.5 10-0.4 10-0.3 10-0.2 10-0.1 100.0 100.1 100.2 100.3 100.4 100.5 100.6 100.7 100.8 100.9 101.0 101.1 101.2 101.3 101.4 101.5 101.6 101.7 101.8 101.9 102.0 10-4 10-2 100 102 10-4.0 10-3.8 10-3.6 10-3.4 10-3.2 10-3.0 10-2.8 10-2.6 10-2.4 10-2.2 10-2.0 10-1.8 10-1.6 10-1.4 10-1.2 10-1.0 10-0.8 10-0.6 10-0.4 10-0.2 100.0 100.2 100.4 100.6 100.8 101.0 101.2 101.4 101.6 101.8 102.0 beta11 beta12 beta13 beta14 beta15 beta16 beta17 beta18 beta19 beta20 beta1 beta2 beta3 beta4 beta5 beta6 beta7 beta8 beta9 beta10 label -0.45 -0.40 -0.35 -0.30 -0.25 -0.20 -0.15 -0.10 -0.05 0.00 0.05 0.10 0.15 0.20 0.25 0.30 0.35 0.40 -0.40 -0.39 -0.38 -0.37 -0.36 -0.35 -0.34 -0.33 -0.32 -0.31 -0.30 -0.29 -0.28 -0.27 -0.26 -0.25 -0.24 -0.23 -0.22 -0.21 -0.20 -0.19 -0.18 -0.17 -0.16 -0.15 -0.14 -0.13 -0.12 -0.11 -0.10 -0.09 -0.08 -0.07 -0.06 -0.05 -0.04 -0.03 -0.02 -0.01 0.00 0.01 0.02 0.03 0.04 0.05 0.06 0.07 0.08 0.09 0.10 0.11 0.12 0.13 0.14 0.15 0.16 0.17 0.18 0.19 0.20 0.21 0.22 0.23 0.24 0.25 0.26 0.27 0.28 0.29 0.30 0.31 0.32 0.33 0.34 0.35 -0.4 -0.2 0.0 0.2 0.4 -0.40 -0.38 -0.36 -0.34 -0.32 -0.30 -0.28 -0.26 -0.24 -0.22 -0.20 -0.18 -0.16 -0.14 -0.12 -0.10 -0.08 -0.06 -0.04 -0.02 0.00 0.02 0.04 0.06 0.08 0.10 0.12 0.14 0.16 0.18 0.20 0.22 0.24 0.26 0.28 0.30 0.32 0.34 0.36 betas