import os
import warnings
import time
import copy
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error, explained_variance_score, r2_score
from sklearn.model_selection import train_test_split, KFold
from sklearn.linear_model import LinearRegression, Lasso, lasso_path, lars_path, LassoLarsIC
from sklearn.neural_network import MLPRegressor
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' #Hide messy TensorFlow warnings
warnings.filterwarnings("ignore") #Hide messy Numpy warnings
import keras
from keras.layers.core import Dense, Activation, Dropout
from keras.layers import Input
from keras.models import Model
from keras.layers.recurrent import LSTM, GRU
from keras.regularizers import l1
from keras.models import Sequential
from keras.models import load_model
#plotly charts
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
from plotly.graph_objs import *
import plotly.figure_factory as ff
init_notebook_mode(connected=True)
Using TensorFlow backend.
# splines chart
# R code
"""
testdata <- data.frame(seq(from=1, to = 1000))
testdata['x'] <- seq(from=0, to = 3*pi, length.out=1000)
testdata['y'] <- -cos(testdata$x) + testdata$x/(3*pi) + rnorm(1000)*0.25
nrows <- 300
indexes <- sample(nrow(testdata), nrows)
indexes <- indexes[order(indexes)]
testdata <- testdata[indexes, ]
plot(testdata$x, testdata$y, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=2)
testdata$fit2 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit2, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=3)
testdata$fit3 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit3, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=4)
testdata$fit4 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit4, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=5)
testdata$fit5 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit5, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=6)
testdata$fit6 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit6, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=20)
testdata$fit20 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit20, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=40)
testdata$fit40 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit40, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=80)
testdata$fit80 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit80, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=160)
testdata$fit160 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit160, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,df=240)
testdata$fit240 <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fit240, main="Scatterplot", xlab="x", ylab="y", pch=1)
fit <- smooth.spline(testdata$x,testdata$y,cv=TRUE)
testdata$fitcv <- predict(fit, newdata=testdata$x)[2]$y
plot(testdata$x, testdata$fitcv, main="Scatterplot", xlab="x", ylab="y", pch=1)
write.csv(testdata, "splines.csv")
print(fit)
"""
splines = pd.read_csv('splines.csv')
data = splines.values
x = data[:,2]
y = data[:,3]
fit2 = data[:,4]
fit3 = data[:,5]
fit4 = data[:,6]
fit5 = data[:,7]
fit6 = data[:,8]
fit20 = data[:,9]
fit40 = data[:,10]
fit80 = data[:,11]
fit160 = data[:,12]
fit240 = data[:,13]
fitcv = data[:,14]
plotdata = []
plotdata.append(Scatter(x=x,
y=y,
name='Raw data',
mode = 'markers',
marker = dict(
size = 3,
line = dict(
width = 2,
color = 'rgba(128, 128, 192, .75)',
)
)
)
)
plotdata.append(Scatter(x=x,
y=fit2,
name='Linear (high bias / low variance)',
mode = 'line',
line = dict(
color = ('orange'),
width = 2)
))
plotdata.append(Scatter(x=x,
y=fit4,
name='Underfit (high bias / low variance)',
mode = 'line',
line = dict(
width = 2,
color= ('blue'),
)
))
plotdata.append(Scatter(x=x,
y=fitcv,
name='Lowest CV error',
mode = 'line',
line = dict(
color = ('green'),
width = 5)
) )
plotdata.append(Scatter(x=x,
y=fit240,
name='Overfit (high variance / low bias)',
mode = 'line',
line = dict(
width = 2,
color= ('rgb(192,0,0)'))
))
layout = Layout(
yaxis=dict(
autorange=True))
fig = Figure(data=plotdata, layout=layout)
iplot(fig) # png to save notebook w/static image
# create a data set, sin wave plus trend plus random noise
nobs = 2000
x = np.linspace(0, 3*np.pi, num=nobs)
y = -np.cos(x) + x/(3*np.pi) + np.random.normal(0, 0.25, nobs)
x2 = np.linspace(0, 15*np.pi, num=nobs)
y1 = -np.sin(x2) + x2/(3*np.pi) + np.random.normal(0, 0.25, nobs)
y2 = -np.cos(x2) + x2/(3*np.pi) + np.random.normal(0, 0.25, nobs)
# sample 20% to make it more sparse
#arr = np.arange(nobs)
#np.random.shuffle(arr)
#x = arr[:200]
#y = y[x]
# chart it
def mychart(*args):
# pass some 2d n x 1 arrays, x, y, z
# 1st array is independent vars
# reshape to 1 dimensional array
x = args[0].reshape(-1)
# following are dependent vars plotted on y axis
data = []
for i in range(1, len(args)):
data.append(Scatter(x=x,
y=args[i].reshape(-1),
mode = 'markers'))
layout = Layout(
yaxis=dict(
autorange=True))
fig = Figure(data=data, layout=layout)
return iplot(fig) # png to save notebook w/static image
mychart(x,y)
mychart(x2,y1, y2)
import pandas as pd
pd.read_csv('splines.csv')
Unnamed: 0 | seq.from...1..to...1000. | x | y | fit2 | fit3 | fit4 | fit5 | fit6 | fit20 | fit40 | fit80 | fit160 | fit240 | fitcv | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 7 | 7 | 0.056605 | -0.894776 | -0.027209 | -0.231666 | -0.719842 | -1.027753 | -1.117166 | -0.879794 | -0.781292 | -0.858112 | -0.892830 | -0.892830 | -1.046363 |
1 | 14 | 14 | 0.122645 | -0.639564 | -0.019815 | -0.212231 | -0.676326 | -0.971100 | -1.059450 | -0.880533 | -0.784838 | -0.700174 | -0.645704 | -0.645704 | -1.004173 |
2 | 29 | 29 | 0.264158 | -0.858491 | -0.003971 | -0.170590 | -0.583081 | -0.849676 | -0.935686 | -0.877862 | -0.843645 | -0.775587 | -0.809460 | -0.809460 | -0.913081 |
3 | 32 | 32 | 0.292461 | -0.971105 | -0.000803 | -0.162264 | -0.564433 | -0.825382 | -0.910901 | -0.875433 | -0.863846 | -0.815897 | -0.824983 | -0.824983 | -0.894611 |
4 | 34 | 34 | 0.311329 | -0.441006 | 0.001310 | -0.156713 | -0.552002 | -0.809184 | -0.894369 | -0.873216 | -0.878066 | -0.846558 | -0.836715 | -0.836715 | -0.882226 |
5 | 35 | 35 | 0.320763 | -0.812196 | 0.002366 | -0.153938 | -0.545787 | -0.801084 | -0.886100 | -0.871895 | -0.885199 | -0.863802 | -0.847200 | -0.847200 | -0.876009 |
6 | 37 | 37 | 0.339632 | -1.107151 | 0.004479 | -0.148389 | -0.533356 | -0.784882 | -0.869555 | -0.868770 | -0.899093 | -0.902248 | -0.881082 | -0.881082 | -0.863520 |
7 | 42 | 42 | 0.386803 | -1.002422 | 0.009760 | -0.134517 | -0.502283 | -0.744365 | -0.828147 | -0.857574 | -0.925840 | -0.994434 | -0.994687 | -0.994687 | -0.831932 |
8 | 45 | 45 | 0.415105 | -1.195662 | 0.012929 | -0.126197 | -0.483642 | -0.720046 | -0.803265 | -0.848041 | -0.931322 | -1.022434 | -1.038721 | -1.038721 | -0.812685 |
9 | 48 | 48 | 0.443408 | -0.967867 | 0.016097 | -0.117879 | -0.465006 | -0.695720 | -0.778353 | -0.836071 | -0.925555 | -1.014256 | -1.035864 | -1.035864 | -0.793187 |
10 | 53 | 53 | 0.490579 | -0.757764 | 0.021378 | -0.104024 | -0.433956 | -0.655163 | -0.736759 | -0.810934 | -0.893917 | -0.940020 | -0.951372 | -0.951372 | -0.760116 |
11 | 60 | 60 | 0.556619 | -0.882360 | 0.028772 | -0.084649 | -0.390525 | -0.598362 | -0.678368 | -0.766014 | -0.817432 | -0.794839 | -0.783219 | -0.783219 | -0.712597 |
12 | 61 | 61 | 0.566053 | -0.747627 | 0.029828 | -0.081884 | -0.384326 | -0.590246 | -0.670011 | -0.758767 | -0.804596 | -0.777175 | -0.764024 | -0.764024 | -0.705691 |
13 | 66 | 66 | 0.613224 | -0.823935 | 0.035109 | -0.068069 | -0.353349 | -0.549667 | -0.628167 | -0.719926 | -0.736604 | -0.707282 | -0.696022 | -0.696022 | -0.670730 |
14 | 72 | 72 | 0.669829 | -0.523625 | 0.041445 | -0.051521 | -0.316238 | -0.500979 | -0.577829 | -0.668861 | -0.651602 | -0.642244 | -0.648028 | -0.648028 | -0.627878 |
15 | 92 | 92 | 0.858513 | -0.622196 | 0.062566 | 0.003297 | -0.193244 | -0.338960 | -0.409136 | -0.486504 | -0.415746 | -0.399757 | -0.434665 | -0.434665 | -0.478897 |
16 | 94 | 94 | 0.877382 | -0.275332 | 0.064678 | 0.008742 | -0.181025 | -0.322799 | -0.392199 | -0.468776 | -0.399977 | -0.378763 | -0.402004 | -0.402004 | -0.463549 |
17 | 98 | 98 | 0.915119 | -0.238423 | 0.068901 | 0.019608 | -0.156638 | -0.290509 | -0.358291 | -0.434364 | -0.374523 | -0.342332 | -0.328025 | -0.328025 | -0.432644 |
18 | 100 | 100 | 0.933987 | -0.123531 | 0.071013 | 0.025028 | -0.144473 | -0.274381 | -0.341322 | -0.417781 | -0.365058 | -0.328908 | -0.291046 | -0.291046 | -0.417092 |
19 | 101 | 101 | 0.943421 | -0.386109 | 0.072069 | 0.027735 | -0.138398 | -0.266322 | -0.332833 | -0.409645 | -0.361105 | -0.325847 | -0.280151 | -0.280151 | -0.409291 |
20 | 103 | 103 | 0.962290 | -0.727077 | 0.074181 | 0.033142 | -0.126262 | -0.250213 | -0.315848 | -0.393661 | -0.354619 | -0.332577 | -0.291137 | -0.291137 | -0.393637 |
21 | 105 | 105 | 0.981158 | -0.092702 | 0.076292 | 0.038539 | -0.114149 | -0.234118 | -0.298853 | -0.378022 | -0.349691 | -0.358963 | -0.356338 | -0.356338 | -0.377915 |
22 | 106 | 106 | 0.990592 | -0.178196 | 0.077348 | 0.041233 | -0.108100 | -0.226076 | -0.290352 | -0.370308 | -0.347513 | -0.374982 | -0.398651 | -0.398651 | -0.370026 |
23 | 109 | 109 | 1.018895 | -0.648567 | 0.080515 | 0.049303 | -0.089988 | -0.201973 | -0.264834 | -0.347455 | -0.340412 | -0.403746 | -0.490174 | -0.490174 | -0.346246 |
24 | 113 | 113 | 1.056632 | -0.366328 | 0.084738 | 0.060024 | -0.065924 | -0.169893 | -0.230780 | -0.317344 | -0.326463 | -0.352344 | -0.408403 | -0.408403 | -0.314260 |
25 | 115 | 115 | 1.075500 | -0.486962 | 0.086849 | 0.065368 | -0.053930 | -0.153881 | -0.213741 | -0.302373 | -0.318127 | -0.315264 | -0.327709 | -0.327709 | -0.298144 |
26 | 118 | 118 | 1.103803 | -0.263969 | 0.090016 | 0.073360 | -0.035991 | -0.129900 | -0.188169 | -0.279950 | -0.304810 | -0.282838 | -0.231644 | -0.231644 | -0.273814 |
27 | 120 | 120 | 1.122671 | 0.053620 | 0.092127 | 0.078673 | -0.024067 | -0.113939 | -0.171114 | -0.264966 | -0.295037 | -0.275382 | -0.191377 | -0.191377 | -0.257487 |
28 | 128 | 128 | 1.198145 | -0.612992 | 0.100570 | 0.099790 | 0.023316 | -0.050340 | -0.102857 | -0.203814 | -0.241514 | -0.288249 | -0.248066 | -0.248066 | -0.191293 |
29 | 130 | 130 | 1.217013 | -0.087615 | 0.102681 | 0.105033 | 0.035078 | -0.034507 | -0.085788 | -0.188020 | -0.222981 | -0.286090 | -0.319283 | -0.319283 | -0.174515 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
270 | 883 | 883 | 8.320975 | 1.090773 | 0.893429 | 0.977505 | 1.064809 | 1.134710 | 1.186539 | 1.277704 | 1.207429 | 1.028304 | 0.953941 | 0.953941 | 1.270916 |
271 | 885 | 885 | 8.339844 | 1.000214 | 0.895564 | 0.984115 | 1.077449 | 1.150509 | 1.203084 | 1.298065 | 1.242501 | 1.059962 | 0.978819 | 0.978819 | 1.287540 |
272 | 889 | 889 | 8.377580 | 1.070650 | 0.899835 | 0.997383 | 1.102828 | 1.182170 | 1.236144 | 1.339964 | 1.329076 | 1.247071 | 1.206912 | 1.206912 | 1.320577 |
273 | 897 | 897 | 8.453054 | 1.666208 | 0.908379 | 1.024105 | 1.153949 | 1.245697 | 1.302093 | 1.423501 | 1.510355 | 1.728304 | 1.809082 | 1.809082 | 1.385514 |
274 | 902 | 902 | 8.500225 | 2.164331 | 0.913720 | 1.040920 | 1.186119 | 1.285510 | 1.343161 | 1.471858 | 1.587764 | 1.832285 | 1.896199 | 1.896199 | 1.425103 |
275 | 910 | 910 | 8.575699 | 1.598121 | 0.922266 | 1.067986 | 1.237899 | 1.349337 | 1.408590 | 1.540403 | 1.640360 | 1.720218 | 1.699297 | 1.699297 | 1.486653 |
276 | 912 | 912 | 8.594567 | 1.674038 | 0.924403 | 1.074781 | 1.250897 | 1.365312 | 1.424890 | 1.555704 | 1.641260 | 1.671724 | 1.646877 | 1.646877 | 1.501679 |
277 | 914 | 914 | 8.613436 | 1.414879 | 0.926539 | 1.081586 | 1.263914 | 1.381293 | 1.441165 | 1.570257 | 1.638037 | 1.625459 | 1.610072 | 1.610072 | 1.516558 |
278 | 916 | 916 | 8.632304 | 1.742757 | 0.928676 | 1.088402 | 1.276950 | 1.397280 | 1.457418 | 1.584188 | 1.632486 | 1.585905 | 1.584810 | 1.584810 | 1.531294 |
279 | 926 | 926 | 8.726646 | 1.570245 | 0.939362 | 1.122621 | 1.342384 | 1.477277 | 1.538342 | 1.648121 | 1.609439 | 1.527509 | 1.535119 | 1.535119 | 1.603040 |
280 | 928 | 928 | 8.745515 | 1.643629 | 0.941499 | 1.129490 | 1.355517 | 1.493288 | 1.554462 | 1.660238 | 1.609349 | 1.537043 | 1.539531 | 1.539531 | 1.617032 |
281 | 932 | 932 | 8.783252 | 1.318655 | 0.945774 | 1.143251 | 1.381823 | 1.525316 | 1.586643 | 1.684328 | 1.617467 | 1.568683 | 1.562898 | 1.562898 | 1.644693 |
282 | 933 | 933 | 8.792686 | 1.618969 | 0.946843 | 1.146695 | 1.388408 | 1.533324 | 1.594676 | 1.690366 | 1.621550 | 1.578440 | 1.571771 | 1.571771 | 1.651545 |
283 | 950 | 950 | 8.953067 | 2.012024 | 0.965012 | 1.205484 | 1.500747 | 1.669515 | 1.730547 | 1.791749 | 1.773085 | 1.792390 | 1.810661 | 1.810661 | 1.764372 |
284 | 960 | 960 | 9.047409 | 1.624429 | 0.975702 | 1.240220 | 1.567092 | 1.749627 | 1.809931 | 1.845555 | 1.864767 | 1.904327 | 1.900205 | 1.900205 | 1.827779 |
285 | 961 | 961 | 9.056844 | 1.996643 | 0.976771 | 1.243698 | 1.573734 | 1.757636 | 1.817850 | 1.850507 | 1.871202 | 1.912575 | 1.902017 | 1.902017 | 1.834010 |
286 | 962 | 962 | 9.066278 | 1.777070 | 0.977840 | 1.247176 | 1.580376 | 1.765645 | 1.825766 | 1.855370 | 1.876986 | 1.919627 | 1.903155 | 1.903155 | 1.840221 |
287 | 965 | 965 | 9.094581 | 1.932855 | 0.981047 | 1.257615 | 1.600309 | 1.789670 | 1.849495 | 1.869428 | 1.890573 | 1.928173 | 1.912138 | 1.912138 | 1.858747 |
288 | 967 | 967 | 9.113449 | 2.122920 | 0.983185 | 1.264578 | 1.613602 | 1.805684 | 1.865300 | 1.878369 | 1.896750 | 1.919566 | 1.929082 | 1.929082 | 1.871011 |
289 | 971 | 971 | 9.151186 | 1.849789 | 0.987461 | 1.278507 | 1.640196 | 1.837708 | 1.896880 | 1.895414 | 1.905165 | 1.883622 | 1.952431 | 1.952431 | 1.895362 |
290 | 973 | 973 | 9.170054 | 2.085739 | 0.989599 | 1.285474 | 1.653497 | 1.853718 | 1.912656 | 1.903625 | 1.908997 | 1.867122 | 1.934527 | 1.934527 | 1.907461 |
291 | 978 | 978 | 9.217225 | 1.845186 | 0.994944 | 1.302898 | 1.686757 | 1.893738 | 1.952067 | 1.923569 | 1.921577 | 1.858799 | 1.789746 | 1.789746 | 1.937531 |
292 | 980 | 980 | 9.236094 | 1.432716 | 0.997082 | 1.309869 | 1.700063 | 1.909744 | 1.967820 | 1.931353 | 1.927733 | 1.868867 | 1.738972 | 1.738972 | 1.949500 |
293 | 986 | 986 | 9.292699 | 2.110915 | 1.003496 | 1.330786 | 1.739989 | 1.957758 | 2.015055 | 1.954210 | 1.948939 | 1.941426 | 1.871168 | 1.871168 | 1.985259 |
294 | 988 | 988 | 9.311567 | 2.150585 | 1.005634 | 1.337760 | 1.753298 | 1.973761 | 2.030794 | 1.961701 | 1.956536 | 1.976895 | 2.053107 | 2.053107 | 1.997144 |
295 | 990 | 990 | 9.330436 | 1.970253 | 1.007772 | 1.344733 | 1.766609 | 1.989764 | 2.046531 | 1.969119 | 1.963709 | 2.006993 | 2.207295 | 2.207295 | 2.009016 |
296 | 993 | 993 | 9.358738 | 2.289554 | 1.010979 | 1.355194 | 1.786574 | 2.013768 | 2.070134 | 1.980067 | 1.972180 | 2.016287 | 2.120013 | 2.120013 | 2.026805 |
297 | 994 | 994 | 9.368173 | 1.766312 | 1.012048 | 1.358681 | 1.793230 | 2.021769 | 2.078001 | 1.983659 | 1.974087 | 2.004609 | 1.957363 | 1.957363 | 2.032730 |
298 | 995 | 995 | 9.377607 | 1.850753 | 1.013117 | 1.362168 | 1.799885 | 2.029770 | 2.085867 | 1.987228 | 1.975583 | 1.986822 | 1.770286 | 1.770286 | 2.038654 |
299 | 999 | 999 | 9.415344 | 1.948860 | 1.017394 | 1.376116 | 1.826506 | 2.061773 | 2.117333 | 2.001399 | 1.979900 | 1.906551 | 1.950956 | 1.950956 | 2.062340 |
300 rows × 15 columns
# fit with sklearn MLPRegressor
layer1_sizes=[1,2,4,8]
layer2_sizes=[1,2,4,8]
import itertools
from plotly import tools
def run_grid(build_model_fn, layer1_sizes, layer2_sizes, x, y, epochs=None):
nrows = len(layer1_sizes)
ncols = len(layer2_sizes)
hyperparameter_list = list(itertools.product(layer1_sizes, layer2_sizes))
subplot_titles = ["%d units, %d units" %
(layer1_size, layer2_size) for (layer1_size, layer2_size) in hyperparameter_list]
fig = tools.make_subplots(rows=nrows,
cols=ncols,
subplot_titles=subplot_titles)
for count, (layer1_size, layer2_size) in enumerate(hyperparameter_list):
print("Layer 1 units: %d, Layer 2 units %d:" % (layer1_size, layer2_size))
print("Running experiment %d of %d : %d %d" % (count+1, len(hyperparameter_list), layer1_size, layer2_size))
model = build_model_fn(hidden_layer_sizes=(layer1_size, layer2_size),
activation='tanh',
max_iter=10000, tol=1e-10,
solver='lbfgs'
)
x = x.reshape(-1,1)
if epochs:
model.fit(x,y, epochs=EPOCHS)
else:
model.fit(x,y)
y_pred = model.predict(x)
train_score = mean_squared_error(y, y_pred)
print(train_score)
print("%s Train MSE: %s" % (time.strftime("%H:%M:%S"), str(train_score)))
print("%s Train R-squared: %.6f" % (time.strftime("%H:%M:%S"), 1-train_score/y.var()))
z = model.predict(x)
trace = Scatter(
x = x.reshape(-1),
y = z.reshape(-1),
name = 'fit',
mode = 'markers',
marker = dict(size = 2)
)
fig.append_trace(trace, count // nrows + 1, count % ncols +1)
return(iplot(fig))
run_grid(MLPRegressor, layer1_sizes, layer2_sizes, x, y)
This is the format of your plot grid: [ (1,1) x1,y1 ] [ (1,2) x2,y2 ] [ (1,3) x3,y3 ] [ (1,4) x4,y4 ] [ (2,1) x5,y5 ] [ (2,2) x6,y6 ] [ (2,3) x7,y7 ] [ (2,4) x8,y8 ] [ (3,1) x9,y9 ] [ (3,2) x10,y10 ] [ (3,3) x11,y11 ] [ (3,4) x12,y12 ] [ (4,1) x13,y13 ] [ (4,2) x14,y14 ] [ (4,3) x15,y15 ] [ (4,4) x16,y16 ] Layer 1 units: 1, Layer 2 units 1: Running experiment 1 of 16 : 1 1 0.45123687843071003 10:59:18 Train MSE: 0.45123687843071003 10:59:18 Train R-squared: 0.360194 Layer 1 units: 1, Layer 2 units 2: Running experiment 2 of 16 : 1 2 0.4501787438606833 10:59:18 Train MSE: 0.4501787438606833 10:59:18 Train R-squared: 0.361694 Layer 1 units: 1, Layer 2 units 4: Running experiment 3 of 16 : 1 4 0.449447662622746 10:59:19 Train MSE: 0.449447662622746 10:59:19 Train R-squared: 0.362731 Layer 1 units: 1, Layer 2 units 8: Running experiment 4 of 16 : 1 8 0.44960316888835167 10:59:19 Train MSE: 0.44960316888835167 10:59:19 Train R-squared: 0.362511 Layer 1 units: 2, Layer 2 units 1: Running experiment 5 of 16 : 2 1 0.4493035300677745 10:59:19 Train MSE: 0.4493035300677745 10:59:19 Train R-squared: 0.362935 Layer 1 units: 2, Layer 2 units 2: Running experiment 6 of 16 : 2 2 0.3769890459200242 10:59:19 Train MSE: 0.3769890459200242 10:59:19 Train R-squared: 0.465470 Layer 1 units: 2, Layer 2 units 4: Running experiment 7 of 16 : 2 4 0.06425241086551657 10:59:20 Train MSE: 0.06425241086551657 10:59:20 Train R-squared: 0.908897 Layer 1 units: 2, Layer 2 units 8: Running experiment 8 of 16 : 2 8 0.06460457742750453 10:59:20 Train MSE: 0.06460457742750453 10:59:20 Train R-squared: 0.908398 Layer 1 units: 4, Layer 2 units 1: Running experiment 9 of 16 : 4 1 0.0646415331604307 10:59:20 Train MSE: 0.0646415331604307 10:59:20 Train R-squared: 0.908345 Layer 1 units: 4, Layer 2 units 2: Running experiment 10 of 16 : 4 2 0.44931457571832617 10:59:20 Train MSE: 0.44931457571832617 10:59:20 Train R-squared: 0.362920 Layer 1 units: 4, Layer 2 units 4: Running experiment 11 of 16 : 4 4 0.06389679796982471 10:59:20 Train MSE: 0.06389679796982471 10:59:20 Train R-squared: 0.909401 Layer 1 units: 4, Layer 2 units 8: Running experiment 12 of 16 : 4 8 0.0636990840102081 10:59:21 Train MSE: 0.0636990840102081 10:59:21 Train R-squared: 0.909681 Layer 1 units: 8, Layer 2 units 1: Running experiment 13 of 16 : 8 1 0.4493031378114045 10:59:21 Train MSE: 0.4493031378114045 10:59:21 Train R-squared: 0.362936 Layer 1 units: 8, Layer 2 units 2: Running experiment 14 of 16 : 8 2 0.06367152233486137 10:59:21 Train MSE: 0.06367152233486137 10:59:21 Train R-squared: 0.909721 Layer 1 units: 8, Layer 2 units 4: Running experiment 15 of 16 : 8 4 0.06352021208219046 10:59:22 Train MSE: 0.06352021208219046 10:59:22 Train R-squared: 0.909935 Layer 1 units: 8, Layer 2 units 8: Running experiment 16 of 16 : 8 8 0.06369116044117601 10:59:22 Train MSE: 0.06369116044117601 10:59:22 Train R-squared: 0.909693
# sklearn MLP regression didn't work as well as expected
# let's build our own keras model by wrappping it in sklearn interface
def build_ols_model(input_size = 1, hidden_layer_sizes=[4]):
main_input = Input(shape=(input_size,), name='main_input')
lastlayer=main_input
for layer_size in hidden_layer_sizes:
lastlayer = Dense(layer_size,
kernel_initializer=keras.initializers.glorot_normal(seed=None),
bias_initializer=keras.initializers.glorot_normal(seed=None),
activation='tanh')(lastlayer)
output = Dense(1, activation='linear')(lastlayer)
model = Model(inputs=[main_input], outputs=[output])
model.compile(loss="mse",
optimizer='rmsprop'
)
print(model.summary())
return model
EPOCHS=5000
BATCH_SIZE=1000
def run_experiment (model, x, y):
models = []
losses = []
for epoch in range(EPOCHS):
fit = model.fit(
x,
y,
batch_size=BATCH_SIZE,
shuffle=True,
epochs=1,
verbose=0
)
train_loss = fit.history['loss'][-1]
losses.append(train_loss)
models.append(copy.copy(model))
bestloss_index = np.argmin(losses)
bestloss_value = losses[bestloss_index]
if epoch % 100 == 99:
print("%s Epoch %d of %d Loss %.6f Best Loss %.6f" % (time.strftime("%H:%M:%S"),
epoch+1,
EPOCHS, train_loss,
bestloss_value))
# stop if loss rises by 20% from best
#if train_loss / bestloss_value > 1.2:
# print("%s Stopping..." % (time.strftime("%H:%M:%S")))
# break
print ("%s Best training loss epoch %d, value %f" % (time.strftime("%H:%M:%S"), bestloss_index, bestloss_value))
model = models[bestloss_index]
train_score = model.evaluate(x, y)
print(train_score)
print("%s Train MSE: %s" % (time.strftime("%H:%M:%S"), str(train_score)))
print("%s Train R-squared: %.6f" % (time.strftime("%H:%M:%S"), 1-train_score/y.var()))
return mychart(x, y, model.predict(x))
model = build_ols_model(hidden_layer_sizes=[4,4])
run_experiment(model, x, y)
_________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_1 (Dense) (None, 4) 8 _________________________________________________________________ dense_2 (Dense) (None, 4) 20 _________________________________________________________________ dense_3 (Dense) (None, 1) 5 ================================================================= Total params: 33 Trainable params: 33 Non-trainable params: 0 _________________________________________________________________ None 10:59:28 Epoch 100 of 5000 Loss 0.454866 Best Loss 0.454866 10:59:30 Epoch 200 of 5000 Loss 0.440300 Best Loss 0.440300 10:59:31 Epoch 300 of 5000 Loss 0.430287 Best Loss 0.430266 10:59:32 Epoch 400 of 5000 Loss 0.423292 Best Loss 0.423292 10:59:34 Epoch 500 of 5000 Loss 0.419050 Best Loss 0.419013 10:59:35 Epoch 600 of 5000 Loss 0.415203 Best Loss 0.415203 10:59:37 Epoch 700 of 5000 Loss 0.411413 Best Loss 0.411413 10:59:38 Epoch 800 of 5000 Loss 0.407389 Best Loss 0.407328 10:59:39 Epoch 900 of 5000 Loss 0.402804 Best Loss 0.402804 10:59:41 Epoch 1000 of 5000 Loss 0.398626 Best Loss 0.398626 10:59:42 Epoch 1100 of 5000 Loss 0.395037 Best Loss 0.394761 10:59:43 Epoch 1200 of 5000 Loss 0.391516 Best Loss 0.391516 10:59:45 Epoch 1300 of 5000 Loss 0.389361 Best Loss 0.389119 10:59:46 Epoch 1400 of 5000 Loss 0.387390 Best Loss 0.387390 10:59:47 Epoch 1500 of 5000 Loss 0.386087 Best Loss 0.386087 10:59:48 Epoch 1600 of 5000 Loss 0.385108 Best Loss 0.385108 10:59:49 Epoch 1700 of 5000 Loss 0.384455 Best Loss 0.384379 10:59:50 Epoch 1800 of 5000 Loss 0.383898 Best Loss 0.383654 10:59:51 Epoch 1900 of 5000 Loss 0.382989 Best Loss 0.382975 10:59:52 Epoch 2000 of 5000 Loss 0.382186 Best Loss 0.382186 10:59:53 Epoch 2100 of 5000 Loss 0.382344 Best Loss 0.381253 10:59:55 Epoch 2200 of 5000 Loss 0.379681 Best Loss 0.379681 10:59:56 Epoch 2300 of 5000 Loss 0.376322 Best Loss 0.375842 10:59:57 Epoch 2400 of 5000 Loss 0.357396 Best Loss 0.357396 10:59:58 Epoch 2500 of 5000 Loss 0.242111 Best Loss 0.242111 10:59:59 Epoch 2600 of 5000 Loss 0.097278 Best Loss 0.097278 11:00:00 Epoch 2700 of 5000 Loss 0.078167 Best Loss 0.076910 11:00:01 Epoch 2800 of 5000 Loss 0.073316 Best Loss 0.070604 11:00:02 Epoch 2900 of 5000 Loss 0.069464 Best Loss 0.067773 11:00:03 Epoch 3000 of 5000 Loss 0.068371 Best Loss 0.066624 11:00:04 Epoch 3100 of 5000 Loss 0.066509 Best Loss 0.066264 11:00:05 Epoch 3200 of 5000 Loss 0.067152 Best Loss 0.066055 11:00:06 Epoch 3300 of 5000 Loss 0.067296 Best Loss 0.065978 11:00:07 Epoch 3400 of 5000 Loss 0.066435 Best Loss 0.065866 11:00:09 Epoch 3500 of 5000 Loss 0.065987 Best Loss 0.065753 11:00:10 Epoch 3600 of 5000 Loss 0.068419 Best Loss 0.065646 11:00:11 Epoch 3700 of 5000 Loss 0.067809 Best Loss 0.065501 11:00:12 Epoch 3800 of 5000 Loss 0.065614 Best Loss 0.065393 11:00:13 Epoch 3900 of 5000 Loss 0.065301 Best Loss 0.065238 11:00:14 Epoch 4000 of 5000 Loss 0.065297 Best Loss 0.065232 11:00:16 Epoch 4100 of 5000 Loss 0.065132 Best Loss 0.065132 11:00:17 Epoch 4200 of 5000 Loss 0.067115 Best Loss 0.065132 11:00:20 Epoch 4300 of 5000 Loss 0.065320 Best Loss 0.065060 11:00:21 Epoch 4400 of 5000 Loss 0.065299 Best Loss 0.065060 11:00:22 Epoch 4500 of 5000 Loss 0.068100 Best Loss 0.065059 11:00:23 Epoch 4600 of 5000 Loss 0.066901 Best Loss 0.065020 11:00:24 Epoch 4700 of 5000 Loss 0.068272 Best Loss 0.065020 11:00:25 Epoch 4800 of 5000 Loss 0.066469 Best Loss 0.065017 11:00:26 Epoch 4900 of 5000 Loss 0.068633 Best Loss 0.064932 11:00:27 Epoch 5000 of 5000 Loss 0.065355 Best Loss 0.064932 11:00:27 Best training loss epoch 4861, value 0.064932 2000/2000 [==============================] - 0s 71us/step 0.06788910901546479 11:00:27 Train MSE: 0.06788910901546479 11:00:27 Train R-squared: 0.903740
# wrap our keras model in sklearn wrapper so we can run grid like above MLPRegressor
from keras.wrappers.scikit_learn import KerasRegressor
# closure for sklearn wrapper
def make_build(layer_sizes):
def myclosure():
return build_ols_model(input_size=1, hidden_layer_sizes=layer_sizes)
return myclosure
def make_keras_model(**kwargs):
# make a function that takes hidden layer sizes kwarg and returns estimator of correct layer sizes
build_fn = make_build(kwargs['hidden_layer_sizes'])
keras_estimator = KerasRegressor(build_fn=build_fn,
epochs=500,
batch_size=32,
verbose=0)
return keras_estimator
run_grid(make_keras_model, layer1_sizes, layer2_sizes, x, y)
# works but not as conistently as expected
# expected all to work well but show increasing degrees of freedom
# takeaway: feedforward NN a bit hinky for regression
This is the format of your plot grid: [ (1,1) x1,y1 ] [ (1,2) x2,y2 ] [ (1,3) x3,y3 ] [ (1,4) x4,y4 ] [ (2,1) x5,y5 ] [ (2,2) x6,y6 ] [ (2,3) x7,y7 ] [ (2,4) x8,y8 ] [ (3,1) x9,y9 ] [ (3,2) x10,y10 ] [ (3,3) x11,y11 ] [ (3,4) x12,y12 ] [ (4,1) x13,y13 ] [ (4,2) x14,y14 ] [ (4,3) x15,y15 ] [ (4,4) x16,y16 ] Layer 1 units: 1, Layer 2 units 1: Running experiment 1 of 16 : 1 1 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_4 (Dense) (None, 1) 2 _________________________________________________________________ dense_5 (Dense) (None, 1) 2 _________________________________________________________________ dense_6 (Dense) (None, 1) 2 ================================================================= Total params: 6 Trainable params: 6 Non-trainable params: 0 _________________________________________________________________ None 0.4519322147661302 11:02:40 Train MSE: 0.4519322147661302 11:02:40 Train R-squared: 0.359208 Layer 1 units: 1, Layer 2 units 2: Running experiment 2 of 16 : 1 2 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_7 (Dense) (None, 1) 2 _________________________________________________________________ dense_8 (Dense) (None, 2) 4 _________________________________________________________________ dense_9 (Dense) (None, 1) 3 ================================================================= Total params: 9 Trainable params: 9 Non-trainable params: 0 _________________________________________________________________ None 0.45160741952506867 11:04:51 Train MSE: 0.45160741952506867 11:04:51 Train R-squared: 0.359669 Layer 1 units: 1, Layer 2 units 4: Running experiment 3 of 16 : 1 4 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_10 (Dense) (None, 1) 2 _________________________________________________________________ dense_11 (Dense) (None, 4) 8 _________________________________________________________________ dense_12 (Dense) (None, 1) 5 ================================================================= Total params: 15 Trainable params: 15 Non-trainable params: 0 _________________________________________________________________ None 0.7053004490393621 11:07:17 Train MSE: 0.7053004490393621 11:07:17 Train R-squared: -0.000041 Layer 1 units: 1, Layer 2 units 8: Running experiment 4 of 16 : 1 8 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_13 (Dense) (None, 1) 2 _________________________________________________________________ dense_14 (Dense) (None, 8) 16 _________________________________________________________________ dense_15 (Dense) (None, 1) 9 ================================================================= Total params: 27 Trainable params: 27 Non-trainable params: 0 _________________________________________________________________ None 0.4510370600120921 11:09:26 Train MSE: 0.4510370600120921 11:09:26 Train R-squared: 0.360477 Layer 1 units: 2, Layer 2 units 1: Running experiment 5 of 16 : 2 1 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_16 (Dense) (None, 2) 4 _________________________________________________________________ dense_17 (Dense) (None, 1) 3 _________________________________________________________________ dense_18 (Dense) (None, 1) 2 ================================================================= Total params: 9 Trainable params: 9 Non-trainable params: 0 _________________________________________________________________ None 0.45114841289054414 11:11:41 Train MSE: 0.45114841289054414 11:11:41 Train R-squared: 0.360320 Layer 1 units: 2, Layer 2 units 2: Running experiment 6 of 16 : 2 2 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_19 (Dense) (None, 2) 4 _________________________________________________________________ dense_20 (Dense) (None, 2) 6 _________________________________________________________________ dense_21 (Dense) (None, 1) 3 ================================================================= Total params: 13 Trainable params: 13 Non-trainable params: 0 _________________________________________________________________ None 0.06812825100574675 11:14:05 Train MSE: 0.06812825100574675 11:14:05 Train R-squared: 0.903401 Layer 1 units: 2, Layer 2 units 4: Running experiment 7 of 16 : 2 4 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_22 (Dense) (None, 2) 4 _________________________________________________________________ dense_23 (Dense) (None, 4) 12 _________________________________________________________________ dense_24 (Dense) (None, 1) 5 ================================================================= Total params: 21 Trainable params: 21 Non-trainable params: 0 _________________________________________________________________ None 0.3792664590602151 11:16:15 Train MSE: 0.3792664590602151 11:16:15 Train R-squared: 0.462241 Layer 1 units: 2, Layer 2 units 8: Running experiment 8 of 16 : 2 8 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_25 (Dense) (None, 2) 4 _________________________________________________________________ dense_26 (Dense) (None, 8) 24 _________________________________________________________________ dense_27 (Dense) (None, 1) 9 ================================================================= Total params: 37 Trainable params: 37 Non-trainable params: 0 _________________________________________________________________ None 0.4000896874315654 11:18:35 Train MSE: 0.4000896874315654 11:18:35 Train R-squared: 0.432715 Layer 1 units: 4, Layer 2 units 1: Running experiment 9 of 16 : 4 1 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_28 (Dense) (None, 4) 8 _________________________________________________________________ dense_29 (Dense) (None, 1) 5 _________________________________________________________________ dense_30 (Dense) (None, 1) 2 ================================================================= Total params: 15 Trainable params: 15 Non-trainable params: 0 _________________________________________________________________ None 0.4504766956211491 11:20:56 Train MSE: 0.4504766956211491 11:20:56 Train R-squared: 0.361272 Layer 1 units: 4, Layer 2 units 2: Running experiment 10 of 16 : 4 2 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_31 (Dense) (None, 4) 8 _________________________________________________________________ dense_32 (Dense) (None, 2) 10 _________________________________________________________________ dense_33 (Dense) (None, 1) 3 ================================================================= Total params: 21 Trainable params: 21 Non-trainable params: 0 _________________________________________________________________ None 0.06517304845517428 11:23:05 Train MSE: 0.06517304845517428 11:23:05 Train R-squared: 0.907592 Layer 1 units: 4, Layer 2 units 4: Running experiment 11 of 16 : 4 4 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_34 (Dense) (None, 4) 8 _________________________________________________________________ dense_35 (Dense) (None, 4) 20 _________________________________________________________________ dense_36 (Dense) (None, 1) 5 ================================================================= Total params: 33 Trainable params: 33 Non-trainable params: 0 _________________________________________________________________ None 0.08572741041763401 11:25:31 Train MSE: 0.08572741041763401 11:25:31 Train R-squared: 0.878448 Layer 1 units: 4, Layer 2 units 8: Running experiment 12 of 16 : 4 8 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_37 (Dense) (None, 4) 8 _________________________________________________________________ dense_38 (Dense) (None, 8) 40 _________________________________________________________________ dense_39 (Dense) (None, 1) 9 ================================================================= Total params: 57 Trainable params: 57 Non-trainable params: 0 _________________________________________________________________ None 0.07093969956793557 11:27:46 Train MSE: 0.07093969956793557 11:27:46 Train R-squared: 0.899415 Layer 1 units: 8, Layer 2 units 1: Running experiment 13 of 16 : 8 1 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_40 (Dense) (None, 8) 16 _________________________________________________________________ dense_41 (Dense) (None, 1) 9 _________________________________________________________________ dense_42 (Dense) (None, 1) 2 ================================================================= Total params: 27 Trainable params: 27 Non-trainable params: 0 _________________________________________________________________ None 0.06676349956404776 11:29:59 Train MSE: 0.06676349956404776 11:29:59 Train R-squared: 0.905336 Layer 1 units: 8, Layer 2 units 2: Running experiment 14 of 16 : 8 2 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_43 (Dense) (None, 8) 16 _________________________________________________________________ dense_44 (Dense) (None, 2) 18 _________________________________________________________________ dense_45 (Dense) (None, 1) 3 ================================================================= Total params: 37 Trainable params: 37 Non-trainable params: 0 _________________________________________________________________ None 0.06960820754959089 11:32:26 Train MSE: 0.06960820754959089 11:32:26 Train R-squared: 0.901303 Layer 1 units: 8, Layer 2 units 4: Running experiment 15 of 16 : 8 4 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_46 (Dense) (None, 8) 16 _________________________________________________________________ dense_47 (Dense) (None, 4) 36 _________________________________________________________________ dense_48 (Dense) (None, 1) 5 ================================================================= Total params: 57 Trainable params: 57 Non-trainable params: 0 _________________________________________________________________ None 0.06459001586471468 11:34:40 Train MSE: 0.06459001586471468 11:34:40 Train R-squared: 0.908418 Layer 1 units: 8, Layer 2 units 8: Running experiment 16 of 16 : 8 8 _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= main_input (InputLayer) (None, 1) 0 _________________________________________________________________ dense_49 (Dense) (None, 8) 16 _________________________________________________________________ dense_50 (Dense) (None, 8) 72 _________________________________________________________________ dense_51 (Dense) (None, 1) 9 ================================================================= Total params: 97 Trainable params: 97 Non-trainable params: 0 _________________________________________________________________ None 0.06490714871662286 11:36:52 Train MSE: 0.06490714871662286 11:36:52 Train R-squared: 0.907969
# try RandomForestRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import RandomizedSearchCV
rf = RandomForestRegressor(random_state = 42)
from pprint import pprint
# Look at parameters used by our current forest
print('Parameters currently in use:\n')
pprint(rf.get_params())
# First 2 are most important
# Number of trees in random forest
n_estimators = [int(a) for a in np.linspace(start = 200, stop = 2000, num = 10)]
# Number of features to consider at every split
max_features = ['auto', 'sqrt']
# Maximum number of levels in tree
max_depth = [int(a) for a in np.linspace(10, 110, num = 11)]
max_depth.append(None)
# Minimum number of samples required to split a node
min_samples_split = [2, 5, 10]
# Minimum number of samples required at each leaf node
min_samples_leaf = [1, 2, 4]
# Method of selecting samples for training each tree
bootstrap = [True, False]
# Create the random grid
print("Search grid:")
random_grid = {'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap}
pprint(random_grid)
Parameters currently in use: {'bootstrap': True, 'criterion': 'mse', 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_impurity_split': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 10, 'n_jobs': 1, 'oob_score': False, 'random_state': 42, 'verbose': 0, 'warm_start': False} Search grid: {'bootstrap': [True, False], 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None], 'max_features': ['auto', 'sqrt'], 'min_samples_leaf': [1, 2, 4], 'min_samples_split': [2, 5, 10], 'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000]}
np.linspace(start = 200, stop = 2000, num = 10)
array([ 200., 400., 600., 800., 1000., 1200., 1400., 1600., 1800., 2000.])
x=x.reshape(x.shape[0],1)
y=y.reshape(y.shape[0],1)
rf = RandomForestRegressor()
rf.fit(x, y)
z = rf.predict(x)
mychart(x, y, z)
# Use the random grid to search for best hyperparameters
# First create the base model to tune
rf = RandomForestRegressor()
# Random search of parameters, using 3 fold cross validation,
# search across 100 different combinations, and use all available cores
rf_random = RandomizedSearchCV(estimator = rf,
param_distributions = random_grid,
n_iter = 100,
cv = 3, verbose=2,
random_state=42,
n_jobs = 1)
# Fit the random search model
rf_random.fit(x, y)
Fitting 3 folds for each of 100 candidates, totalling 300 fits [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=30 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=30, total= 0.7s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=30
[Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.7s remaining: 0.0s
[CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=30, total= 0.6s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=30 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=30, total= 0.7s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=10, total= 3.1s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=10, total= 2.9s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=10, total= 3.1s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=10, total= 2.0s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=10, total= 1.9s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=10, total= 2.0s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=30, total= 4.1s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=30, total= 3.2s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=30, total= 3.3s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=10 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=10, total= 2.3s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=10 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=10, total= 2.2s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=10 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=10, total= 2.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=30, total= 1.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=30, total= 1.4s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=30, total= 1.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=100, total= 1.8s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=100, total= 1.7s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=100, total= 1.7s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=5, max_depth=60 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=5, max_depth=60, total= 1.1s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=5, max_depth=60 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=5, max_depth=60, total= 1.2s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=5, max_depth=60 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=5, max_depth=60, total= 1.1s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=50 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=50, total= 2.3s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=50 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=50, total= 2.4s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=50 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=50, total= 2.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=5, max_depth=10 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=5, max_depth=10, total= 2.8s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=5, max_depth=10 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=5, max_depth=10, total= 2.7s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=5, max_depth=10 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=5, max_depth=10, total= 2.8s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=70 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=70, total= 0.6s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=70 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=70, total= 0.6s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=70 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=70, total= 0.6s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=90 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=90, total= 1.5s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=90 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=90, total= 1.5s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=90 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=90, total= 1.4s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=10, max_depth=10 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=10, max_depth=10, total= 3.3s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=10, max_depth=10 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=10, max_depth=10, total= 3.2s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=10, max_depth=10 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=10, max_depth=10, total= 3.1s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=10, total= 2.5s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=10, total= 2.5s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=10, total= 2.6s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=30 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=30, total= 1.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=30 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=30, total= 1.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=30 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=30, total= 1.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=50 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=50, total= 3.2s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=50 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=50, total= 3.0s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=50 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=50, total= 3.0s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=70 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=70, total= 1.1s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=70 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=70, total= 1.0s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=70 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=70, total= 1.0s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=20, total= 1.8s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=20, total= 1.6s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=20, total= 1.8s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=10, max_depth=110 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=10, max_depth=110, total= 3.2s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=10, max_depth=110 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=10, max_depth=110, total= 2.9s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=10, max_depth=110 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=10, max_depth=110, total= 3.0s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=80 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=80, total= 0.9s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=80 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=80, total= 1.0s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=80 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=auto, min_samples_split=5, max_depth=80, total= 0.9s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=sqrt, min_samples_split=10, max_depth=30 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=sqrt, min_samples_split=10, max_depth=30, total= 3.3s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=sqrt, min_samples_split=10, max_depth=30 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=sqrt, min_samples_split=10, max_depth=30, total= 2.9s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=sqrt, min_samples_split=10, max_depth=30 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=sqrt, min_samples_split=10, max_depth=30, total= 3.0s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=70 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=70, total= 2.8s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=70 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=70, total= 2.7s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=70 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=5, max_depth=70, total= 2.8s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80, total= 1.9s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80, total= 1.9s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80, total= 1.9s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=None [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=None, total= 2.6s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=None [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=None, total= 2.6s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=None [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=None, total= 2.6s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80, total= 2.4s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80, total= 2.3s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=80, total= 2.4s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=None [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=None, total= 0.8s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=None [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=None, total= 0.7s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=None [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=None, total= 0.8s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=40 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=40, total= 2.7s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=40 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=40, total= 2.6s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=40 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=40, total= 2.7s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=20, total= 1.4s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=20, total= 1.4s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=20, total= 1.4s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=100 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=100, total= 3.1s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=100 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=100, total= 3.1s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=100 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=100, total= 3.1s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=20, total= 1.7s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=20, total= 1.7s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=20, total= 1.7s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=10, max_depth=20 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=10, max_depth=20, total= 1.9s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=10, max_depth=20 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=10, max_depth=20, total= 1.8s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=10, max_depth=20 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=10, max_depth=20, total= 1.9s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=50 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=50, total= 1.4s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=50 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=50, total= 1.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=50 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=50, total= 1.4s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=100, total= 1.4s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=100, total= 1.4s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=100, total= 1.4s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=50 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=50, total= 1.1s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=50 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=50, total= 1.1s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=50 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=50, total= 1.1s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1800, max_features=sqrt, min_samples_split=2, max_depth=90 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1800, max_features=sqrt, min_samples_split=2, max_depth=90, total= 2.4s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1800, max_features=sqrt, min_samples_split=2, max_depth=90 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1800, max_features=sqrt, min_samples_split=2, max_depth=90, total= 2.4s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1800, max_features=sqrt, min_samples_split=2, max_depth=90 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1800, max_features=sqrt, min_samples_split=2, max_depth=90, total= 2.4s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=20 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=20, total= 1.5s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=20 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=20, total= 1.7s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=20 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=20, total= 1.8s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=20, total= 2.0s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=20, total= 2.8s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1200, max_features=sqrt, min_samples_split=5, max_depth=20, total= 2.0s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=auto, min_samples_split=2, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=auto, min_samples_split=2, max_depth=100, total= 1.3s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=auto, min_samples_split=2, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=auto, min_samples_split=2, max_depth=100, total= 1.3s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=auto, min_samples_split=2, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=auto, min_samples_split=2, max_depth=100, total= 1.2s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=5, max_depth=None [CV] bootstrap=True, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=5, max_depth=None, total= 1.1s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=5, max_depth=None [CV] bootstrap=True, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=5, max_depth=None, total= 1.1s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=5, max_depth=None [CV] bootstrap=True, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=5, max_depth=None, total= 1.1s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=60 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=60, total= 1.4s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=60 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=60, total= 1.4s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=60 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=60, total= 1.4s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=10, total= 0.3s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=10, total= 0.3s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=10, total= 0.3s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=60 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=60, total= 0.8s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=60 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=60, total= 0.8s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=60 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=60, total= 0.8s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=90 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=90, total= 1.2s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=90 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=90, total= 1.1s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=90 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=90, total= 1.2s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90, total= 0.6s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90, total= 0.6s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90, total= 0.6s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=90 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=90, total= 0.4s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=90 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=90, total= 0.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=90 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=90, total= 0.3s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=110 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=110, total= 1.5s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=110 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=110, total= 1.5s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=110 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=2, max_depth=110, total= 1.5s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=90 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=90, total= 3.0s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=90 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=90, total= 2.8s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=90 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=90, total= 2.8s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=80 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=80, total= 0.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=80 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=80, total= 0.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=80 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=80, total= 0.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=70 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=70, total= 1.9s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=70 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=70, total= 1.8s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=70 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1200, max_features=sqrt, min_samples_split=2, max_depth=70, total= 1.9s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=110 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=110, total= 1.0s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=110 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=110, total= 1.0s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=110 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=110, total= 1.0s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=20 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=20, total= 3.3s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=20 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=20, total= 3.3s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=20 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=20, total= 3.4s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=50 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=50, total= 3.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=50 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=50, total= 3.1s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=50 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=50, total= 3.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=50 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=50, total= 1.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=50 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=50, total= 1.5s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=50 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=50, total= 1.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=30, total= 1.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=30, total= 1.5s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=30, total= 1.6s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=None [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=None, total= 2.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=None [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=None, total= 2.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=None [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=5, max_depth=None, total= 2.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=60 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=60, total= 1.0s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=60 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=60, total= 0.9s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=60 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=60, total= 1.0s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=10, total= 2.2s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=10, total= 2.2s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=10, total= 2.2s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=80 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=80, total= 2.6s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=80 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=80, total= 2.5s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=80 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=80, total= 2.6s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=100, total= 2.2s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=100, total= 2.2s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=100, total= 2.2s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=10, max_depth=80 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=10, max_depth=80, total= 1.9s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=10, max_depth=80 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=10, max_depth=80, total= 2.1s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=10, max_depth=80 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=sqrt, min_samples_split=10, max_depth=80, total= 2.0s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=200, max_features=sqrt, min_samples_split=2, max_depth=50 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=200, max_features=sqrt, min_samples_split=2, max_depth=50, total= 0.3s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=200, max_features=sqrt, min_samples_split=2, max_depth=50 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=200, max_features=sqrt, min_samples_split=2, max_depth=50, total= 0.3s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=200, max_features=sqrt, min_samples_split=2, max_depth=50 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=200, max_features=sqrt, min_samples_split=2, max_depth=50, total= 0.4s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=10 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=10, total= 0.7s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=10 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=10, total= 0.6s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=10 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=400, max_features=sqrt, min_samples_split=2, max_depth=10, total= 0.7s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=80 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=80, total= 1.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=80 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=80, total= 1.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=80 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=10, max_depth=80, total= 1.6s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=auto, min_samples_split=10, max_depth=None [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=auto, min_samples_split=10, max_depth=None, total= 2.1s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=auto, min_samples_split=10, max_depth=None [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=auto, min_samples_split=10, max_depth=None, total= 1.9s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=auto, min_samples_split=10, max_depth=None [CV] bootstrap=False, min_samples_leaf=2, n_estimators=1200, max_features=auto, min_samples_split=10, max_depth=None, total= 2.1s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=20 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=20, total= 2.6s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=20 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=20, total= 2.6s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=20 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=20, total= 2.5s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1600, max_features=auto, min_samples_split=10, max_depth=None [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1600, max_features=auto, min_samples_split=10, max_depth=None, total= 2.5s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1600, max_features=auto, min_samples_split=10, max_depth=None [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1600, max_features=auto, min_samples_split=10, max_depth=None, total= 2.5s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1600, max_features=auto, min_samples_split=10, max_depth=None [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1600, max_features=auto, min_samples_split=10, max_depth=None, total= 2.2s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=10 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=10, total= 3.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=10 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=10, total= 3.4s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=10 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1800, max_features=auto, min_samples_split=2, max_depth=10, total= 2.9s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=70 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=70, total= 2.2s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=70 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=70, total= 2.1s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=70 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1400, max_features=auto, min_samples_split=2, max_depth=70, total= 2.1s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=80 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=80, total= 1.7s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=80 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=80, total= 1.6s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=80 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=80, total= 1.7s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=60 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=60, total= 3.5s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=60 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=60, total= 3.2s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=60 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=60, total= 3.9s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=2, max_depth=80 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=2, max_depth=80, total= 2.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=2, max_depth=80 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=2, max_depth=80, total= 2.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=2, max_depth=80 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=1400, max_features=sqrt, min_samples_split=2, max_depth=80, total= 2.5s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=20, total= 1.5s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=20, total= 1.4s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=2, max_depth=20, total= 1.3s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=sqrt, min_samples_split=5, max_depth=60 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=sqrt, min_samples_split=5, max_depth=60, total= 2.7s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=sqrt, min_samples_split=5, max_depth=60 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=sqrt, min_samples_split=5, max_depth=60, total= 2.7s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=sqrt, min_samples_split=5, max_depth=60 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1800, max_features=sqrt, min_samples_split=5, max_depth=60, total= 2.7s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=auto, min_samples_split=5, max_depth=90 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=auto, min_samples_split=5, max_depth=90, total= 0.7s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=auto, min_samples_split=5, max_depth=90 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=auto, min_samples_split=5, max_depth=90, total= 0.7s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=auto, min_samples_split=5, max_depth=90 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=400, max_features=auto, min_samples_split=5, max_depth=90, total= 0.8s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=90 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=90, total= 2.5s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=90 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=90, total= 2.3s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=90 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=auto, min_samples_split=5, max_depth=90, total= 2.6s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90, total= 0.7s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90, total= 0.7s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=sqrt, min_samples_split=10, max_depth=90, total= 0.8s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=None [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=None, total= 2.6s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=None [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=None, total= 2.5s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=None [CV] bootstrap=True, min_samples_leaf=1, n_estimators=1600, max_features=sqrt, min_samples_split=10, max_depth=None, total= 2.3s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=100, total= 3.0s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=100, total= 3.0s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=100, total= 3.1s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=10, total= 1.4s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=10, total= 1.5s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=10 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=5, max_depth=10, total= 1.4s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=80 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=80, total= 0.3s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=80 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=80, total= 0.3s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=80 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=80, total= 0.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=60 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=60, total= 3.4s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=60 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=60, total= 3.1s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=60 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=auto, min_samples_split=2, max_depth=60, total= 3.5s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=10, max_depth=100 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=10, max_depth=100, total= 0.8s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=10, max_depth=100 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=10, max_depth=100, total= 0.8s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=10, max_depth=100 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=600, max_features=auto, min_samples_split=10, max_depth=100, total= 0.8s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=100 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=100, total= 1.4s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=100 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=100, total= 1.4s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=100 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1000, max_features=auto, min_samples_split=5, max_depth=100, total= 1.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=10, max_depth=None [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=10, max_depth=None, total= 1.3s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=10, max_depth=None [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=10, max_depth=None, total= 1.2s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=10, max_depth=None [CV] bootstrap=False, min_samples_leaf=2, n_estimators=800, max_features=auto, min_samples_split=10, max_depth=None, total= 1.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=70 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=70, total= 1.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=70 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=70, total= 1.2s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=70 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=70, total= 1.3s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=40 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=40, total= 1.0s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=40 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=40, total= 0.9s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=40 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=40, total= 1.0s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=100, total= 0.6s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=100, total= 0.6s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=100 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=400, max_features=sqrt, min_samples_split=5, max_depth=100, total= 0.6s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=80 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=80, total= 2.3s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=80 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=80, total= 2.1s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=80 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=1600, max_features=sqrt, min_samples_split=2, max_depth=80, total= 2.1s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=100 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=100, total= 1.1s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=100 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=100, total= 1.1s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=100 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=800, max_features=sqrt, min_samples_split=10, max_depth=100, total= 1.1s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=10 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=10, total= 1.3s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=10 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=10, total= 1.3s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=10 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=1000, max_features=sqrt, min_samples_split=10, max_depth=10, total= 1.3s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=110 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=110, total= 0.8s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=110 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=110, total= 0.8s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=110 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=600, max_features=sqrt, min_samples_split=10, max_depth=110, total= 0.8s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=None [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=None, total= 3.2s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=None [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=None, total= 3.0s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=None [CV] bootstrap=False, min_samples_leaf=4, n_estimators=2000, max_features=sqrt, min_samples_split=5, max_depth=None, total= 3.3s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=40 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=40, total= 1.2s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=40 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=40, total= 1.2s [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=40 [CV] bootstrap=True, min_samples_leaf=1, n_estimators=800, max_features=sqrt, min_samples_split=5, max_depth=40, total= 1.2s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=600, max_features=auto, min_samples_split=2, max_depth=40 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=600, max_features=auto, min_samples_split=2, max_depth=40, total= 0.8s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=600, max_features=auto, min_samples_split=2, max_depth=40 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=600, max_features=auto, min_samples_split=2, max_depth=40, total= 0.8s [CV] bootstrap=True, min_samples_leaf=4, n_estimators=600, max_features=auto, min_samples_split=2, max_depth=40 [CV] bootstrap=True, min_samples_leaf=4, n_estimators=600, max_features=auto, min_samples_split=2, max_depth=40, total= 0.8s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=30, total= 1.0s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=30, total= 0.9s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=30 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=600, max_features=sqrt, min_samples_split=2, max_depth=30, total= 1.0s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=40 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=40, total= 0.7s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=40 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=40, total= 0.6s [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=40 [CV] bootstrap=False, min_samples_leaf=2, n_estimators=400, max_features=auto, min_samples_split=10, max_depth=40, total= 0.6s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=10 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=10, total= 1.6s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=10 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=10, total= 1.6s [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=10 [CV] bootstrap=False, min_samples_leaf=1, n_estimators=1000, max_features=auto, min_samples_split=2, max_depth=10, total= 1.6s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=100 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=100, total= 0.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=100 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=100, total= 0.3s [CV] bootstrap=False, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=100 [CV] bootstrap=False, min_samples_leaf=4, n_estimators=200, max_features=auto, min_samples_split=5, max_depth=100, total= 0.3s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=sqrt, min_samples_split=2, max_depth=20, total= 2.8s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=sqrt, min_samples_split=2, max_depth=20, total= 2.8s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=sqrt, min_samples_split=2, max_depth=20 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=sqrt, min_samples_split=2, max_depth=20, total= 3.0s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=40 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=40, total= 2.7s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=40 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=40, total= 2.7s [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=40 [CV] bootstrap=True, min_samples_leaf=2, n_estimators=2000, max_features=auto, min_samples_split=10, max_depth=40, total= 2.7s
[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed: 9.4min finished
RandomizedSearchCV(cv=3, error_score='raise', estimator=RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None, max_features='auto', max_leaf_nodes=None, min_impurity_decrease=0.0, min_impurity_split=None, min_samples_leaf=1, min_samples_split=2, min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1, oob_score=False, random_state=None, verbose=0, warm_start=False), fit_params=None, iid=True, n_iter=100, n_jobs=1, param_distributions={'bootstrap': [True, False], 'min_samples_leaf': [1, 2, 4], 'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000], 'min_samples_split': [2, 5, 10], 'max_features': ['auto', 'sqrt'], 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None]}, pre_dispatch='2*n_jobs', random_state=42, refit=True, return_train_score='warn', scoring=None, verbose=2)
rf_random.best_params_
{'bootstrap': False, 'max_depth': 10, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 1200}
rf = RandomForestRegressor(bootstrap=False,
n_estimators=1200,
max_features='sqrt',
max_depth=10,
min_samples_leaf=2,
min_samples_split=5,
)
rf.fit(x, y)
z = rf.predict(x)
mychart(x, y, z)
rf = RandomForestRegressor(n_estimators=600,
max_features='auto',
max_depth=None,
min_samples_leaf=4,
min_samples_split=2,
)
rf.fit(x, y)
z = rf.predict(x)
mychart(x, y, z)
from scipy.interpolate import UnivariateSpline
spl = UnivariateSpline(x, y)
z = spl(x)
mychart(x, y, z)
spl.set_smoothing_factor(101.0)
#print(spl.get_knots())
z = spl(x)
mychart(x, y, z)
def spline_cv_test(smoothing_factor):
kf = KFold(n_splits=5)
errors = []
for train_index, test_index in kf.split(x):
x_train, x_test = x[train_index], x[test_index]
y_train, y_test = y[train_index], y[test_index]
spline = UnivariateSpline(x_train, y_train)
spline.set_smoothing_factor(smoothing_factor)
y_pred_test = spline(x_test)
errors.append(mean_squared_error(y_test, y_pred_test))
return np.mean(np.array(errors))
print(spline_cv_test(49.856))
for sf in np.linspace(10, 110, num=101):
print(sf, spline_cv_test(sf))
4146407058.9861517 (10.0, 33247092630076.83) (11.0, 29475614624086.59) (12.0, 29558362233432.53) (13.0, 27736768535424.715) (14.0, 26414569025152.24) (15.0, 25709370408353.465) (16.0, 24634112258771.12) (17.0, 22070573209369.37) (18.0, 34762440049306.203) (19.0, 28995785902109.6) (20.0, 25770131493033.906) (21.0, 26265698632396.156) (22.0, 23697753834507.715) (23.0, 28310324440186.1) (24.0, 4059767967770.148) (25.0, 3916644608906.1465) (26.0, 4824453016586.01) (27.0, 56376321055.55424) (28.0, 55562653311.56177) (29.0, 52594047044.55418) (30.0, 61455155857.17878) (31.0, 56449841551.18958) (32.0, 57019560639.7848) (33.0, 53770603740.183395) (34.0, 57195783200.04348) (35.0, 58654224000.13448) (36.0, 68212678411.61669) (37.0, 78709333835.38673) (38.0, 84730456041.5764) (39.0, 6011720459.68354) (40.0, 6231192279.7697) (41.0, 5760476682.374887) (42.0, 5992944422.7285185) (43.0, 5368627878.031981) (44.0, 5106394139.494643) (45.0, 4798101260.433854) (46.0, 4692589740.948821) (47.0, 4383231967.811253) (48.0, 4169875675.4103723) (49.0, 4087905363.489609) (50.0, 4119941566.538889) (51.0, 304534373059.7632) (52.0, 319659183461.79486) (53.0, 332016260700.8769) (54.0, 342055577738.312) (55.0, 348457597114.5676) (56.0, 352151813819.3777) (57.0, 353037412881.7689) (58.0, 351325948727.9931) (59.0, 347297681624.6608) (60.0, 341278589697.5876) (61.0, 333526211145.6957) (62.0, 441726740163.7495) (63.0, 430653018385.7642) (64.0, 413905328309.2967) (65.0, 395040356994.88696) (66.0, 371131700725.1827) (67.0, 347834956206.02094) (68.0, 341390951622.9299) (69.0, 323821608636.58655) (70.0, 256146963309.88522) (71.0, 254264662682.2758) (72.0, 253581454777.15063) (73.0, 255192642640.8915) (74.0, 249116269545.6131) (75.0, 255390352518.3961) (76.0, 247069897775.83905) (77.0, 238388147339.28958) (78.0, 199245200724.46246) (79.0, 1390347540.060241) (80.0, 1460575869.1399848) (81.0, 507637087.6948515) (82.0, 24313904.205273505) (83.0, 35902735.79875658) (84.0, 108113116.80639756) (85.0, 601348749.000735) (86.0, 660088695.3629776) (87.0, 450959.1560572498) (88.0, 818997.3135701473) (89.0, 445227.5400082507) (90.0, 435738.8472340024) (91.0, 414369.39111643395) (92.0, 814941.6070000506) (93.0, 383793.93787504005) (94.0, 7175365.42675651) (95.0, 840549.2243547873) (96.0, 1539370.3420854488) (97.0, 115513.70913762086) (98.0, 56918.81033437237) (99.0, 15364.46737816732) (100.0, 2552.420702278856) (101.0, 0.1968513560366662) (102.0, 2.8564629246408826) (103.0, 7.01178867660263) (104.0, 11.827185995071453) (105.0, 11.940081221867297) (106.0, 11.380104011537451) (107.0, 10.819879403613854) (108.0, 10.225661266430416) (109.0, 9.689135004107914) (110.0, 12.782728637570663)
from xgboost import XGBRegressor
from sklearn.model_selection import RandomizedSearchCV
import scipy.stats as st
xgbreg = XGBRegressor(nthreads=1)
one_to_left = st.beta(10, 1)
from_zero_positive = st.expon(0, 50)
param_grid = {
'silent': [False],
'max_depth': [6, 10, 15, 20],
'learning_rate': [0.001, 0.01, 0.1, 0.2, 0,3],
'subsample': [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
'colsample_bytree': [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
'colsample_bylevel': [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
'min_child_weight': [0.5, 1.0, 3.0, 5.0, 7.0, 10.0],
'gamma': [0, 0.25, 0.5, 1.0],
'reg_lambda': [0.1, 1.0, 5.0, 10.0, 50.0, 100.0],
'n_estimators': [100]}
params = {
"n_estimators": [10, 20, 40, 80, 160, 320, 640],
"max_depth": [4, 8, 16, 32, 64, 128],
"learning_rate": [0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50],
"colsample_bytree": [1.0, 0.99, 0.95, 0.90, 0.85, 0.80, 0.60],
"subsample": [1.0, 0.99, 0.95, 0.90, 0.85, 0.80, 0.60],
"gamma": [0, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0],
'reg_alpha': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100],
"min_child_weight": [10, 20, 30, 40, 50, 60, 70, 80, 90, 100],
}
gs = RandomizedSearchCV(xgbreg, params, n_jobs=1)
gs.fit(x, y)
gs.best_params_
{'colsample_bytree': 0.6, 'gamma': 3.0, 'learning_rate': 0.05, 'max_depth': 64, 'min_child_weight': 90, 'n_estimators': 10, 'reg_alpha': 100, 'subsample': 0.99}
xgbreg = XGBRegressor(nthreads=1,
colsample_bytree=0.6,
gamma=3.0,
learning_rate=0.05,
max_depth=64,
min_child_weight=90,
n_estimators=10,
reg_alpha=100,
subsample=0.99)
xgbreg.fit(x, y)
z = xgbreg.predict(x)
mychart(x, y, z)
#LSTM/GRU
def generator_factory(data, # Xs
lookback, # sequence length
min_index=0, # start index
max_index=None, # max index for e.g. train/xval/text, if None use all of X
shuffle=False, # if True, just repeatedly sample randomly
batch_size=128): # how many (predictors, responses) tuples to generate
if max_index == None:
max_index = data.shape[0] - 2 # last row of predictors that has a y to predict
first_row = min_index + lookback # first row of responses to return
i = first_row
while True: # keep generating forever
if shuffle: # pick rows at random
rows = np.random.choice(range(first_row,max_index), size=batch_size, replace=False)
else:
if i + batch_size >= max_index: # start from beginning
i = first_row
rows = np.arange(i, i + batch_size)
predictors = np.zeros((len(rows), lookback, data.shape[-1]))
responses = np.zeros((len(rows), data.shape[-1]))
for j, row in enumerate(rows):
indices = range(rows[j] - lookback, rows[j])
predictors[j] = data[indices]
responses[j] = data[rows[j]]
yield predictors, [responses[:,0], responses[:,1]]
y2 = y2.reshape(y2.shape[0],1)
y1 = y1.reshape(y1.shape[0],1)
X = np.hstack((y1,y2))
X.shape
mygen = generator_factory(X, 10, batch_size=16)
next(mygen)
(array([[[ 0.06066273, -0.5962953 ], [ 0.0154961 , -0.97396613], [ 0.29961459, -0.72818719], [ 0.26859928, -1.18538647], [-0.46767877, -1.30893672], [-0.33681018, -1.30955189], [-0.07610397, -0.83522199], [-0.06501045, -0.96951715], [-0.57310981, -1.13934823], [ 0.01873465, -1.02707313]], [[ 0.0154961 , -0.97396613], [ 0.29961459, -0.72818719], [ 0.26859928, -1.18538647], [-0.46767877, -1.30893672], [-0.33681018, -1.30955189], [-0.07610397, -0.83522199], [-0.06501045, -0.96951715], [-0.57310981, -1.13934823], [ 0.01873465, -1.02707313], [-0.33741347, -1.28745857]], [[ 0.29961459, -0.72818719], [ 0.26859928, -1.18538647], [-0.46767877, -1.30893672], [-0.33681018, -1.30955189], [-0.07610397, -0.83522199], [-0.06501045, -0.96951715], [-0.57310981, -1.13934823], [ 0.01873465, -1.02707313], [-0.33741347, -1.28745857], [-0.10820698, -0.84829179]], [[ 0.26859928, -1.18538647], [-0.46767877, -1.30893672], [-0.33681018, -1.30955189], [-0.07610397, -0.83522199], [-0.06501045, -0.96951715], [-0.57310981, -1.13934823], [ 0.01873465, -1.02707313], [-0.33741347, -1.28745857], [-0.10820698, -0.84829179], [ 0.09578219, -1.10095065]], [[-0.46767877, -1.30893672], [-0.33681018, -1.30955189], [-0.07610397, -0.83522199], [-0.06501045, -0.96951715], [-0.57310981, -1.13934823], [ 0.01873465, -1.02707313], [-0.33741347, -1.28745857], [-0.10820698, -0.84829179], [ 0.09578219, -1.10095065], [-0.40179584, -1.13244153]], [[-0.33681018, -1.30955189], [-0.07610397, -0.83522199], [-0.06501045, -0.96951715], [-0.57310981, -1.13934823], [ 0.01873465, -1.02707313], [-0.33741347, -1.28745857], [-0.10820698, -0.84829179], [ 0.09578219, -1.10095065], [-0.40179584, -1.13244153], [-0.26829727, -0.63113494]], [[-0.07610397, -0.83522199], [-0.06501045, -0.96951715], [-0.57310981, -1.13934823], [ 0.01873465, -1.02707313], [-0.33741347, -1.28745857], [-0.10820698, -0.84829179], [ 0.09578219, -1.10095065], [-0.40179584, -1.13244153], [-0.26829727, -0.63113494], [-0.14780634, -1.15089659]], [[-0.06501045, -0.96951715], [-0.57310981, -1.13934823], [ 0.01873465, -1.02707313], [-0.33741347, -1.28745857], [-0.10820698, -0.84829179], [ 0.09578219, -1.10095065], [-0.40179584, -1.13244153], [-0.26829727, -0.63113494], [-0.14780634, -1.15089659], [-0.52769753, -0.65231528]], [[-0.57310981, -1.13934823], [ 0.01873465, -1.02707313], [-0.33741347, -1.28745857], [-0.10820698, -0.84829179], [ 0.09578219, -1.10095065], [-0.40179584, -1.13244153], [-0.26829727, -0.63113494], [-0.14780634, -1.15089659], [-0.52769753, -0.65231528], [-0.67783355, -0.93284959]], [[ 0.01873465, -1.02707313], [-0.33741347, -1.28745857], [-0.10820698, -0.84829179], [ 0.09578219, -1.10095065], [-0.40179584, -1.13244153], [-0.26829727, -0.63113494], [-0.14780634, -1.15089659], [-0.52769753, -0.65231528], [-0.67783355, -0.93284959], [-0.25965463, -0.66408584]], [[-0.33741347, -1.28745857], [-0.10820698, -0.84829179], [ 0.09578219, -1.10095065], [-0.40179584, -1.13244153], [-0.26829727, -0.63113494], [-0.14780634, -1.15089659], [-0.52769753, -0.65231528], [-0.67783355, -0.93284959], [-0.25965463, -0.66408584], [-0.21045462, -1.04332307]], [[-0.10820698, -0.84829179], [ 0.09578219, -1.10095065], [-0.40179584, -1.13244153], [-0.26829727, -0.63113494], [-0.14780634, -1.15089659], [-0.52769753, -0.65231528], [-0.67783355, -0.93284959], [-0.25965463, -0.66408584], [-0.21045462, -1.04332307], [-0.34603607, -0.8710234 ]], [[ 0.09578219, -1.10095065], [-0.40179584, -1.13244153], [-0.26829727, -0.63113494], [-0.14780634, -1.15089659], [-0.52769753, -0.65231528], [-0.67783355, -0.93284959], [-0.25965463, -0.66408584], [-0.21045462, -1.04332307], [-0.34603607, -0.8710234 ], [-0.11911523, -0.90433275]], [[-0.40179584, -1.13244153], [-0.26829727, -0.63113494], [-0.14780634, -1.15089659], [-0.52769753, -0.65231528], [-0.67783355, -0.93284959], [-0.25965463, -0.66408584], [-0.21045462, -1.04332307], [-0.34603607, -0.8710234 ], [-0.11911523, -0.90433275], [-0.24633775, -0.50097024]], [[-0.26829727, -0.63113494], [-0.14780634, -1.15089659], [-0.52769753, -0.65231528], [-0.67783355, -0.93284959], [-0.25965463, -0.66408584], [-0.21045462, -1.04332307], [-0.34603607, -0.8710234 ], [-0.11911523, -0.90433275], [-0.24633775, -0.50097024], [-0.10660787, -1.11483104]], [[-0.14780634, -1.15089659], [-0.52769753, -0.65231528], [-0.67783355, -0.93284959], [-0.25965463, -0.66408584], [-0.21045462, -1.04332307], [-0.34603607, -0.8710234 ], [-0.11911523, -0.90433275], [-0.24633775, -0.50097024], [-0.10660787, -1.11483104], [-0.79584369, -0.71976902]]]), [array([-0.33741347, -0.10820698, 0.09578219, -0.40179584, -0.26829727, -0.14780634, -0.52769753, -0.67783355, -0.25965463, -0.21045462, -0.34603607, -0.11911523, -0.24633775, -0.10660787, -0.79584369, -0.35641191]), array([-1.28745857, -0.84829179, -1.10095065, -1.13244153, -0.63113494, -1.15089659, -0.65231528, -0.93284959, -0.66408584, -1.04332307, -0.8710234 , -0.90433275, -0.50097024, -1.11483104, -0.71976902, -1.03635705])])
OUTPUT_DIM = 2
def build_model(hidden_layers = [[16, 0.0001, 0.2],
[16, 0.0001, 0.2],
[1, 0.0, 0.0]]):
"""Take list of [layer_size, reg_penalty, dropout], last layer is linear, rest LSTM"""
main_input = Input(shape=(None,
OUTPUT_DIM),
dtype='float32',
name='main_input')
lastlayer=main_input
n_lstms = len(hidden_layers)-1
for i in range(n_lstms):
layer_size, reg_penalty, dropout = hidden_layers[i]
# first n-1 layers are LSTM, return_sequences=True
return_sequences = True
if i == n_lstms-1:
return_sequences= False
LSTM_layer = GRU(layer_size,
return_sequences=return_sequences,
kernel_regularizer=keras.regularizers.l1(reg_penalty),
dropout=dropout,
recurrent_dropout=dropout,
name = "GRU%02d" % i
)(lastlayer)
lastlayer = LSTM_layer
layer_size, reg_penalty, dropout = hidden_layers[-1]
if dropout:
lastlayer = Dropout(dropout)(lastlayer)
outputs = []
for i in range(OUTPUT_DIM):
# OUTPUT_DIM outputs
output01 = Dense(1,
activation='linear',
kernel_regularizer=keras.regularizers.l1(reg_penalty),
name='output%02d' % i)(lastlayer)
outputs.append(output01)
model = Model(inputs=[main_input], outputs=outputs)
print(model.summary())
model.compile(loss="mae", metrics=['mae'], optimizer="rmsprop", loss_weights=[1.]*OUTPUT_DIM)
return model
model = build_model()
__________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ================================================================================================== main_input (InputLayer) (None, None, 2) 0 __________________________________________________________________________________________________ GRU00 (GRU) (None, None, 16) 912 main_input[0][0] __________________________________________________________________________________________________ GRU01 (GRU) (None, 16) 1584 GRU00[0][0] __________________________________________________________________________________________________ output00 (Dense) (None, 1) 17 GRU01[0][0] __________________________________________________________________________________________________ output01 (Dense) (None, 1) 17 GRU01[0][0] ================================================================================================== Total params: 2,530 Trainable params: 2,530 Non-trainable params: 0 __________________________________________________________________________________________________ None
LOOKBACK = 128
BATCH_SIZE = 64
mygen = generator_factory(X, LOOKBACK, batch_size=BATCH_SIZE, shuffle=True)
history = model.fit_generator(mygen,
steps_per_epoch=32,
epochs=40,
)
Epoch 1/40 32/32 [==============================] - 11s 346ms/step - loss: 3.7321 - output00_loss: 1.8919 - output01_loss: 1.8273 - output00_mean_absolute_error: 1.8919 - output01_mean_absolute_error: 1.8273 Epoch 2/40 32/32 [==============================] - 10s 326ms/step - loss: 2.0985 - output00_loss: 1.0409 - output01_loss: 1.0450 - output00_mean_absolute_error: 1.0409 - output01_mean_absolute_error: 1.0450 Epoch 3/40 32/32 [==============================] - 10s 324ms/step - loss: 1.8702 - output00_loss: 0.9141 - output01_loss: 0.9439 - output00_mean_absolute_error: 0.9141 - output01_mean_absolute_error: 0.9439 Epoch 4/40 32/32 [==============================] - 10s 324ms/step - loss: 1.7142 - output00_loss: 0.8325 - output01_loss: 0.8698 - output00_mean_absolute_error: 0.8325 - output01_mean_absolute_error: 0.8698 Epoch 5/40 32/32 [==============================] - 10s 320ms/step - loss: 1.6417 - output00_loss: 0.7942 - output01_loss: 0.8359 - output00_mean_absolute_error: 0.7942 - output01_mean_absolute_error: 0.8359 Epoch 6/40 32/32 [==============================] - 10s 323ms/step - loss: 1.5260 - output00_loss: 0.7415 - output01_loss: 0.7731 - output00_mean_absolute_error: 0.7415 - output01_mean_absolute_error: 0.7731 Epoch 7/40 32/32 [==============================] - 10s 313ms/step - loss: 1.4786 - output00_loss: 0.7208 - output01_loss: 0.7465 - output00_mean_absolute_error: 0.7208 - output01_mean_absolute_error: 0.7465 Epoch 8/40 32/32 [==============================] - 10s 313ms/step - loss: 1.3636 - output00_loss: 0.6567 - output01_loss: 0.6957 - output00_mean_absolute_error: 0.6567 - output01_mean_absolute_error: 0.6957 Epoch 9/40 32/32 [==============================] - 10s 314ms/step - loss: 1.3350 - output00_loss: 0.6556 - output01_loss: 0.6684 - output00_mean_absolute_error: 0.6556 - output01_mean_absolute_error: 0.6684 Epoch 10/40 32/32 [==============================] - 10s 313ms/step - loss: 1.3357 - output00_loss: 0.6492 - output01_loss: 0.6755 - output00_mean_absolute_error: 0.6492 - output01_mean_absolute_error: 0.6755 Epoch 11/40 32/32 [==============================] - 10s 324ms/step - loss: 1.2915 - output00_loss: 0.6241 - output01_loss: 0.6566 - output00_mean_absolute_error: 0.6241 - output01_mean_absolute_error: 0.6566 Epoch 12/40 32/32 [==============================] - 12s 379ms/step - loss: 1.3242 - output00_loss: 0.6344 - output01_loss: 0.6790 - output00_mean_absolute_error: 0.6344 - output01_mean_absolute_error: 0.6790 Epoch 13/40 32/32 [==============================] - 12s 385ms/step - loss: 1.2485 - output00_loss: 0.5992 - output01_loss: 0.6385 - output00_mean_absolute_error: 0.5992 - output01_mean_absolute_error: 0.6385 Epoch 14/40 32/32 [==============================] - 12s 388ms/step - loss: 1.2267 - output00_loss: 0.5955 - output01_loss: 0.6204 - output00_mean_absolute_error: 0.5955 - output01_mean_absolute_error: 0.6204 Epoch 15/40 32/32 [==============================] - 12s 378ms/step - loss: 1.2452 - output00_loss: 0.6054 - output01_loss: 0.6290 - output00_mean_absolute_error: 0.6054 - output01_mean_absolute_error: 0.6290 Epoch 16/40 32/32 [==============================] - 12s 379ms/step - loss: 1.2232 - output00_loss: 0.5869 - output01_loss: 0.6254 - output00_mean_absolute_error: 0.5869 - output01_mean_absolute_error: 0.6254 Epoch 17/40 32/32 [==============================] - 12s 382ms/step - loss: 1.1922 - output00_loss: 0.5702 - output01_loss: 0.6111 - output00_mean_absolute_error: 0.5702 - output01_mean_absolute_error: 0.6111 Epoch 18/40 32/32 [==============================] - 12s 383ms/step - loss: 1.1964 - output00_loss: 0.5760 - output01_loss: 0.6094 - output00_mean_absolute_error: 0.5760 - output01_mean_absolute_error: 0.6094 Epoch 19/40 32/32 [==============================] - 12s 384ms/step - loss: 1.1761 - output00_loss: 0.5770 - output01_loss: 0.5881 - output00_mean_absolute_error: 0.5770 - output01_mean_absolute_error: 0.5881 Epoch 20/40 32/32 [==============================] - 12s 379ms/step - loss: 1.1678 - output00_loss: 0.5629 - output01_loss: 0.5940 - output00_mean_absolute_error: 0.5629 - output01_mean_absolute_error: 0.5940 Epoch 21/40 32/32 [==============================] - 12s 381ms/step - loss: 1.1443 - output00_loss: 0.5468 - output01_loss: 0.5865 - output00_mean_absolute_error: 0.5468 - output01_mean_absolute_error: 0.5865 Epoch 22/40 32/32 [==============================] - 12s 389ms/step - loss: 1.1685 - output00_loss: 0.5754 - output01_loss: 0.5820 - output00_mean_absolute_error: 0.5754 - output01_mean_absolute_error: 0.5820 Epoch 23/40 32/32 [==============================] - 12s 379ms/step - loss: 1.1917 - output00_loss: 0.5690 - output01_loss: 0.6117 - output00_mean_absolute_error: 0.5690 - output01_mean_absolute_error: 0.6117 Epoch 24/40 32/32 [==============================] - 12s 386ms/step - loss: 1.1651 - output00_loss: 0.5730 - output01_loss: 0.5809 - output00_mean_absolute_error: 0.5730 - output01_mean_absolute_error: 0.5809 Epoch 25/40 32/32 [==============================] - 10s 323ms/step - loss: 1.1103 - output00_loss: 0.5364 - output01_loss: 0.5627 - output00_mean_absolute_error: 0.5364 - output01_mean_absolute_error: 0.5627 Epoch 26/40 32/32 [==============================] - 10s 310ms/step - loss: 1.1416 - output00_loss: 0.5571 - output01_loss: 0.5733 - output00_mean_absolute_error: 0.5571 - output01_mean_absolute_error: 0.5733 Epoch 27/40 32/32 [==============================] - 10s 312ms/step - loss: 1.0820 - output00_loss: 0.5221 - output01_loss: 0.5487 - output00_mean_absolute_error: 0.5221 - output01_mean_absolute_error: 0.5487 Epoch 28/40 32/32 [==============================] - 10s 313ms/step - loss: 1.1204 - output00_loss: 0.5620 - output01_loss: 0.5472 - output00_mean_absolute_error: 0.5620 - output01_mean_absolute_error: 0.5472 Epoch 29/40 32/32 [==============================] - 10s 312ms/step - loss: 1.1241 - output00_loss: 0.5423 - output01_loss: 0.5705 - output00_mean_absolute_error: 0.5423 - output01_mean_absolute_error: 0.5705 Epoch 30/40 32/32 [==============================] - 10s 314ms/step - loss: 1.0786 - output00_loss: 0.5265 - output01_loss: 0.5408 - output00_mean_absolute_error: 0.5265 - output01_mean_absolute_error: 0.5408 Epoch 31/40 32/32 [==============================] - 10s 315ms/step - loss: 1.0626 - output00_loss: 0.5176 - output01_loss: 0.5337 - output00_mean_absolute_error: 0.5176 - output01_mean_absolute_error: 0.5337 Epoch 32/40 32/32 [==============================] - 10s 314ms/step - loss: 1.1126 - output00_loss: 0.5407 - output01_loss: 0.5605 - output00_mean_absolute_error: 0.5407 - output01_mean_absolute_error: 0.5605 Epoch 33/40 32/32 [==============================] - 10s 311ms/step - loss: 1.0792 - output00_loss: 0.5205 - output01_loss: 0.5472 - output00_mean_absolute_error: 0.5205 - output01_mean_absolute_error: 0.5472 Epoch 34/40 32/32 [==============================] - 10s 317ms/step - loss: 1.0806 - output00_loss: 0.5274 - output01_loss: 0.5417 - output00_mean_absolute_error: 0.5274 - output01_mean_absolute_error: 0.5417 Epoch 35/40 32/32 [==============================] - 10s 312ms/step - loss: 1.0872 - output00_loss: 0.5187 - output01_loss: 0.5571 - output00_mean_absolute_error: 0.5187 - output01_mean_absolute_error: 0.5571 Epoch 36/40 32/32 [==============================] - 11s 349ms/step - loss: 1.0806 - output00_loss: 0.5205 - output01_loss: 0.5485 - output00_mean_absolute_error: 0.5205 - output01_mean_absolute_error: 0.5485 Epoch 37/40 32/32 [==============================] - 10s 316ms/step - loss: 1.0773 - output00_loss: 0.5148 - output01_loss: 0.5509 - output00_mean_absolute_error: 0.5148 - output01_mean_absolute_error: 0.5509 Epoch 38/40 32/32 [==============================] - 10s 313ms/step - loss: 1.0587 - output00_loss: 0.5243 - output01_loss: 0.5228 - output00_mean_absolute_error: 0.5243 - output01_mean_absolute_error: 0.5228 Epoch 39/40 32/32 [==============================] - 10s 307ms/step - loss: 1.0545 - output00_loss: 0.5165 - output01_loss: 0.5263 - output00_mean_absolute_error: 0.5165 - output01_mean_absolute_error: 0.5263 Epoch 40/40 32/32 [==============================] - 10s 314ms/step - loss: 1.0867 - output00_loss: 0.5333 - output01_loss: 0.5416 - output00_mean_absolute_error: 0.5333 - output01_mean_absolute_error: 0.5416
model.predict(X[:128].reshape(1,LOOKBACK,OUTPUT_DIM))
[array([[0.54338735]], dtype=float32), array([[1.1517216]], dtype=float32)]
X[129]
array([-0.25599367, 1.20011222])
# predict using X
X_pred = np.zeros_like(X)
# copy first 128 rows to X_pred
for i in range(LOOKBACK):
X_pred[i]=X[i]
# forecast remainder of rows using X
for i in range(LOOKBACK, X.shape[0]):
predlist = model.predict(X[i-LOOKBACK:i].reshape(1,LOOKBACK,OUTPUT_DIM))
X_pred[i] = np.array([predlist[0][0][0],predlist[1][0][0]])
from sklearn.metrics import mean_absolute_error
print("%.6f" % mean_absolute_error(y1, X_pred[:,0]))
print("%.6f" % mean_absolute_error(y2, X_pred[:,1]))
0.487040 0.512351
mychart(x2,X_pred[:,0],X_pred[:,1])
# predict recursively after bootstrapping first 128 rows of X
X_pred = np.zeros_like(X)
# copy first 128 rows to X_pred
for i in range(LOOKBACK):
X_pred[i]=X[i]
# recursively forecast remainder of rows using X_pred
for i in range(LOOKBACK, X.shape[0]):
predlist = model.predict(X_pred[i-LOOKBACK:i].reshape(1,LOOKBACK,OUTPUT_DIM))
X_pred[i] = np.array([predlist[0][0][0],predlist[1][0][0]])
mychart(x2,X_pred[:,0],X_pred[:,1])