%load_ext watermark
%watermark -a 'Christian Schuhegger' -u -d -v -p numpy,scipy,pandas,matplotlib,seaborn,mxnet
Christian Schuhegger last updated: 2019-02-27 CPython 3.6.8 IPython 7.3.0 numpy 1.14.6 scipy 1.2.0 pandas 0.24.1 matplotlib 3.0.2 seaborn 0.9.0 mxnet 1.3.1
%matplotlib inline
import numpy as np, scipy, scipy.stats as stats, pandas as pd, matplotlib.pyplot as plt, seaborn as sns
import sklearn, sklearn.pipeline, sklearn.model_selection, sklearn.preprocessing
import logging, time, datetime, tqdm
import mxnet as mx
from mxnet import gluon, nd, autograd, metric
import sys,os,subprocess,glob,multiprocessing
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
# pd.set_option('display.float_format', lambda x: '%.2f' % x)
np.set_printoptions(edgeitems=10)
np.set_printoptions(linewidth=1000)
np.set_printoptions(suppress=True)
np.core.arrayprint._line_width = 180
sns.set()
def get_gpu_name():
try:
out_str = subprocess.run(["nvidia-smi", "--query-gpu=gpu_name", "--format=csv"], stdout=subprocess.PIPE).stdout
out_list = out_str.decode("utf-8").split('\n')
out_list = out_list[1:-1]
return out_list
except Exception as e:
print(e)
def get_cuda_version():
"""Get CUDA version"""
if sys.platform == 'win32':
raise NotImplementedError("Implement this!")
# This breaks on linux:
#cuda=!ls "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA"
#path = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\" + str(cuda[0]) +"\\version.txt"
elif sys.platform == 'linux' or sys.platform == 'darwin':
path = '/usr/local/cuda/version.txt'
else:
raise ValueError("Not in Windows, Linux or Mac")
if os.path.isfile(path):
with open(path, 'r') as f:
data = f.read().replace('\n','')
return data
else:
return "No CUDA in this machine"
def get_cudnn_version():
"""Get CUDNN version"""
if sys.platform == 'win32':
raise NotImplementedError("Implement this!")
# This breaks on linux:
#cuda=!ls "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA"
#candidates = ["C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\" + str(cuda[0]) +"\\include\\cudnn.h"]
elif sys.platform == 'linux':
candidates = ['/usr/include/x86_64-linux-gnu/cudnn_v[0-99].h',
'/usr/local/cuda/include/cudnn.h',
'/usr/include/cudnn.h']
elif sys.platform == 'darwin':
candidates = ['/usr/local/cuda/include/cudnn.h',
'/usr/include/cudnn.h']
else:
raise ValueError("Not in Windows, Linux or Mac")
for c in candidates:
file = glob.glob(c)
if file: break
if file:
with open(file[0], 'r') as f:
version = ''
for line in f:
if "#define CUDNN_MAJOR" in line:
version = line.split()[-1]
if "#define CUDNN_MINOR" in line:
version += '.' + line.split()[-1]
if "#define CUDNN_PATCHLEVEL" in line:
version += '.' + line.split()[-1]
if version:
return version
else:
return "Cannot find CUDNN version"
else:
return "No CUDNN in this machine"
print("OS: ", sys.platform)
print("Python: ", sys.version)
print("MXNet: ", mx.__version__)
print("Numpy: ", np.__version__)
print("GPU: ", get_gpu_name())
print("CPU cores:", multiprocessing.cpu_count())
print(get_cuda_version())
print("CuDNN Version ", get_cudnn_version())
OS: linux Python: 3.6.8 |Anaconda, Inc.| (default, Dec 30 2018, 01:22:34) [GCC 7.3.0] MXNet: 1.3.1 Numpy: 1.14.6 GPU: ['GeForce GTX 1080'] CPU cores: 12 CUDA Version 9.0.176 CuDNN Version 7.4.2
from IPython.display import display, HTML
from IPython.display import display_html
def display_side_by_side(*args):
html_str=''
for df in args:
if type(df) == np.ndarray:
df = pd.DataFrame(df)
html_str+=df.to_html()
html_str = html_str.replace('table','table style="display:inline"')
# print(html_str)
display_html(html_str,raw=True)
CSS = """
.output {
flex-direction: row;
}
"""
def display_graphs_side_by_side(*args):
html_str='<table><tr>'
for g in args:
html_str += '<td>'
html_str += g._repr_svg_()
html_str += '</td>'
html_str += '</tr></table>'
display_html(html_str,raw=True)
display(HTML("<style>.container { width:70% !important; }</style>"))
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s:%(name)s:%(levelname)s: %(message)s')
log = logging.getLogger('std')
N_in = 1000
N_subjects = 260 * 10 # 100000
N_subjects = 100000
W = stats.norm(loc=0, scale=1).rvs(size=(2,N_in), random_state=np.random.RandomState(42))
X = stats.norm(loc=0, scale=1).rvs(size=(N_subjects,N_in), random_state=np.random.RandomState(43))
y = np.sum(W[1:,:] * X + W[0,:], axis=1)
y.shape, X.shape
((100000,), (100000, 1000))
pd.Series(y).describe()
count 100000.000000 mean 19.234805 std 31.516783 min -147.380827 25% -2.102370 50% 19.273695 75% 40.475011 max 147.253509 dtype: float64
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, test_size = 0.1, random_state = 42)
# Maximize training performance with Gluon data loader workers
# https://aws.amazon.com/blogs/machine-learning/maximize-training-performance-with-gluon-data-loader-workers/
def to_gluon_iter(x_in, y_in, batch_size=256, workers=(multiprocessing.cpu_count()//2)):
x_nd = nd.array(x_in)
y_nd = nd.array(y_in)
dataset = mx.gluon.data.ArrayDataset(x_nd, y_nd)
itr = mx.gluon.data.DataLoader(dataset, batch_size = batch_size, shuffle = None, num_workers=workers)# , last_batch = 'rollover'
return itr
def to_sym_iter(x_in, y_in, batch_size=256):
itr = mx.io.NDArrayIter(x_in, y_in, batch_size, shuffle=None , label_name='lin_reg_label')
return itr
class DataIterLoader():
def __init__(self, data_iter):
self.data_iter = data_iter
def __iter__(self):
self.data_iter.reset()
return self
def __next__(self):
batch = self.data_iter.__next__()
assert len(batch.data) == len(batch.label) == 1
# print('len(batch_data): {}; batch.data[0].shape: {}'.format(len(batch.data), batch.data[0].shape))
data = batch.data[0]
label = batch.label[0]
return data, label
def next(self):
return self.__next__() # for Python 2
# See:
# Appendix: Upgrading from Module DataIter to Gluon DataLoader
# https://mxnet.incubator.apache.org/versions/master/tutorials/gluon/datasets.html
batch_size=256
gluon_train_iter = to_gluon_iter(X_train, y_train, batch_size=batch_size)
gluon_valid_iter = to_gluon_iter(X_test , y_test, batch_size=batch_size)
sym_train_iter = to_sym_iter(X_train, y_train, batch_size=batch_size)
sym_valid_iter = to_sym_iter(X_test, y_test, batch_size=batch_size)
gluon_train_iter = DataIterLoader(sym_train_iter)
gluon_valid_iter = DataIterLoader(sym_valid_iter)
def create_aux():
epochs=20
model_ctx=mx.cpu()
loss_function = mx.gluon.loss.L2Loss()
init_function = mx.init.Xavier()
optimizer = mx.optimizer.Adam()
return epochs, model_ctx, loss_function, init_function, optimizer
def create_gluon_model():
ACTIVATION = 'relu'
net = mx.gluon.nn.HybridSequential(prefix='MLP_')
with net.name_scope():
net.add(
mx.gluon.nn.Dense(300, activation=ACTIVATION, prefix='fc-1_'),
mx.gluon.nn.Dense(100, activation=ACTIVATION, prefix='fc-2_'),
mx.gluon.nn.Dense(1 , activation=None , prefix='predictions')
)
return net
def create_sym_model():
ACTIVATION = 'relu'
data = mx.sym.Variable('data')
Y = mx.sym.Variable('lin_reg_label')
fc1 = mx.sym.FullyConnected(data, name='fc1', num_hidden=300)
act1 = mx.sym.Activation(fc1, name='relu1', act_type=ACTIVATION)
fc2 = mx.sym.FullyConnected(act1, name='fc2', num_hidden=100)
act2 = mx.sym.Activation(fc2, name='relu2', act_type=ACTIVATION)
fc3 = mx.sym.FullyConnected(act2, name='fc3', num_hidden=1)
lro = mx.sym.LinearRegressionOutput(data=fc3, label=Y, name="lro")
return lro
epochs, model_ctx, loss_function, init_function, optimizer = create_aux()
sym_train_iter = to_sym_iter(X_train, y_train, batch_size=batch_size)
sym_valid_iter = to_sym_iter(X_test, y_test, batch_size=batch_size)
gluon_model = create_gluon_model()
# gluon_model.hybridize()
gluon_model.hybridize(static_shape=True, static_alloc=True)
gluon_model.collect_params().initialize(init_function, ctx=model_ctx)
trainer = gluon.Trainer(gluon_model.collect_params(), optimizer)
nr_batches = len(X_train) // batch_size
total = epochs * (nr_batches + 1)
def train(num_workers):
mx.random.seed(1)
if num_workers > 1:
gluon_train_iter = to_gluon_iter(X_train, y_train, batch_size=batch_size, workers=num_workers)
gluon_valid_iter = to_gluon_iter(X_test , y_test, batch_size=batch_size, workers=num_workers)
else:
gluon_train_iter = DataIterLoader(sym_train_iter)
gluon_valid_iter = DataIterLoader(sym_valid_iter)
time1 = time.time()
for e in range(epochs):
for i, (x_, y_) in enumerate(gluon_train_iter):
x_ = x_.as_in_context(model_ctx)
y_ = y_.as_in_context(model_ctx)
if num_workers > 1:
nd.waitall()
with autograd.record():
output = gluon_model(x_)
loss = loss_function(output, y_)
loss.backward()
last_batch_loss = nd.mean(loss).asscalar()
trainer.step(x_.shape[0])
if num_workers > 1:
nd.waitall()
t = time.time()
print([t-time1, e, last_batch_loss])
time2 = time.time()
print('workers: {}; time: {}'.format(num_workers, time2-time1))
for workers in range(0, 6, 2):
train(num_workers=workers)
[1.0114946365356445, 0, 2.688013] [2.0326144695281982, 1, 0.9665312] [3.0164339542388916, 2, 0.48773876] [3.9574451446533203, 3, 0.28398213] [4.913841009140015, 4, 0.19583277] [5.915135383605957, 5, 0.15492727] [6.879735946655273, 6, 0.14247555] [7.859589576721191, 7, 0.18341143] [8.832339763641357, 8, 0.8992266] [9.830835580825806, 9, 3.629622] [10.792657375335693, 10, 1.9046185] [11.763952732086182, 11, 1.7743697] [12.71415376663208, 12, 0.5882967] [13.694055318832397, 13, 1.5400267] [14.645780086517334, 14, 1.0786914] [15.640945672988892, 15, 1.2148588] [16.607487201690674, 16, 1.0880853] [17.586325645446777, 17, 0.6401208] [18.567651748657227, 18, 0.7938393] [19.549959897994995, 19, 0.47012243] workers: 0; time: 19.550041675567627 [2.080505847930908, 0, 0.46424496] [3.9421205520629883, 1, 0.48203924] [6.162583112716675, 2, 0.49990627] [8.28538727760315, 3, 0.3972701] [10.105726480484009, 4, 0.4251598] [11.934930562973022, 5, 0.70309675] [13.798903703689575, 6, 1.5159985] [15.691807985305786, 7, 2.4252243] [17.546621561050415, 8, 1.616506] [19.324228763580322, 9, 3.8505468] [21.208154916763306, 10, 1.9401183] [23.062612056732178, 11, 1.0572857] [24.97093892097473, 12, 0.78012973] [26.971498727798462, 13, 0.39198607] [28.860776901245117, 14, 0.2522775] [30.770732879638672, 15, 0.4382887] [32.62504196166992, 16, 0.74195063] [34.49631190299988, 17, 0.6759308] [36.76907277107239, 18, 1.1210217] [38.66449570655823, 19, 0.9971001] workers: 2; time: 38.66468834877014 [1.8759772777557373, 0, 0.5457086] [3.9425806999206543, 1, 0.27232322] [5.816941976547241, 2, 0.14699128] [8.007057428359985, 3, 0.10915594] [10.081457376480103, 4, 0.12385155] [11.983157634735107, 5, 0.17104238] [13.958155155181885, 6, 0.23911487] [15.885420083999634, 7, 0.3623786] [17.843082189559937, 8, 0.5243365] [19.805946111679077, 9, 0.45486805] [21.74500060081482, 10, 1.5060887] [23.67216157913208, 11, 1.9099464] [25.843996047973633, 12, 0.714357] [27.72542119026184, 13, 0.28021395] [29.729376316070557, 14, 0.17045738] [31.75821566581726, 15, 0.110555805] [33.6689875125885, 16, 0.17122094] [35.82129526138306, 17, 0.42039445] [37.803547620773315, 18, 0.34069037] [39.806894302368164, 19, 0.5471134] workers: 4; time: 39.807068824768066
gluon_predict_iter = mx.gluon.data.DataLoader(mx.gluon.data.ArrayDataset(nd.array(X_test)), batch_size=batch_size)
y_gluon_pred = nd.zeros(X_test.shape[0])
for i, (data) in enumerate(gluon_predict_iter):
data = data.as_in_context(model_ctx)
output = gluon_model(data)
y_gluon_pred[i * batch_size : i * batch_size + output.shape[0]] = output[:,0]
s = sklearn.metrics.mean_squared_error(y_test, y_gluon_pred.asnumpy())
s
2.737390181114425
sklearn.metrics.explained_variance_score(y_test, y_gluon_pred.asnumpy())
0.9976410425625697
epochs, model_ctx, loss_function, init_function, optimizer = create_aux()
sym_train_iter = to_sym_iter(X_train, y_train, batch_size=batch_size)
sym_valid_iter = to_sym_iter(X_test, y_test, batch_size=batch_size)
sym_model = create_sym_model()
sym_model_module = mx.mod.Module(symbol = sym_model, data_names = ['data'], label_names = ['lin_reg_label'], context = model_ctx)
freq = int((len(X_train) * epochs / batch_size) // 10)
batch_end_callback = mx.callback.Speedometer(batch_size, frequent=freq, auto_reset=False)
time1 = time.time()
sym_model_module.fit(sym_train_iter,
sym_valid_iter,
optimizer=optimizer,
initializer=init_function,
num_epoch=epochs,
eval_metric='mse',
batch_end_callback=batch_end_callback
)
time2 = time.time()
/home/local/cs/local/install/anaconda3-5.3.1-Linux-x86_64/envs/mxnet/lib/python3.6/site-packages/mxnet/module/base_module.py:504: UserWarning: Optimizer created manually outside Module but rescale_grad is not normalized to 1.0/batch_size/num_workers (1.0 vs. 0.00390625). Is this intended? optimizer_params=optimizer_params) 2019-02-27 14:51:46,594:root:INFO: Epoch[0] Train-mse=151.881052 2019-02-27 14:51:46,595:root:INFO: Epoch[0] Time cost=1.018 2019-02-27 14:51:46,664:root:INFO: Epoch[0] Validation-mse=4.889890 2019-02-27 14:51:47,685:root:INFO: Epoch[1] Train-mse=2.949202 2019-02-27 14:51:47,686:root:INFO: Epoch[1] Time cost=1.021 2019-02-27 14:51:47,746:root:INFO: Epoch[1] Validation-mse=3.650367 2019-02-27 14:51:48,753:root:INFO: Epoch[2] Train-mse=1.537151 2019-02-27 14:51:48,754:root:INFO: Epoch[2] Time cost=1.007 2019-02-27 14:51:48,835:root:INFO: Epoch[2] Validation-mse=3.233742 2019-02-27 14:51:49,828:root:INFO: Epoch[3] Train-mse=0.924725 2019-02-27 14:51:49,829:root:INFO: Epoch[3] Time cost=0.993 2019-02-27 14:51:49,911:root:INFO: Epoch[3] Validation-mse=3.063358 2019-02-27 14:51:50,909:root:INFO: Epoch[4] Train-mse=0.610477 2019-02-27 14:51:50,910:root:INFO: Epoch[4] Time cost=0.998 2019-02-27 14:51:50,992:root:INFO: Epoch[4] Validation-mse=3.018220 2019-02-27 14:51:51,994:root:INFO: Epoch[5] Train-mse=0.441097 2019-02-27 14:51:51,995:root:INFO: Epoch[5] Time cost=1.001 2019-02-27 14:51:52,069:root:INFO: Epoch[5] Validation-mse=3.035564 2019-02-27 14:51:53,068:root:INFO: Epoch[6] Train-mse=0.355614 2019-02-27 14:51:53,069:root:INFO: Epoch[6] Time cost=0.998 2019-02-27 14:51:53,134:root:INFO: Epoch[6] Validation-mse=3.111674 2019-02-27 14:51:54,152:root:INFO: Epoch[7] Train-mse=0.355230 2019-02-27 14:51:54,153:root:INFO: Epoch[7] Time cost=1.018 2019-02-27 14:51:54,230:root:INFO: Epoch[7] Validation-mse=3.327117 2019-02-27 14:51:55,202:root:INFO: Epoch[8] Train-mse=0.669648 2019-02-27 14:51:55,203:root:INFO: Epoch[8] Time cost=0.972 2019-02-27 14:51:55,273:root:INFO: Epoch[8] Validation-mse=4.119022 2019-02-27 14:51:56,318:root:INFO: Epoch[9] Train-mse=2.828806 2019-02-27 14:51:56,319:root:INFO: Epoch[9] Time cost=1.046 2019-02-27 14:51:56,380:root:INFO: Epoch[9] Validation-mse=5.946182 2019-02-27 14:51:57,537:root:INFO: Epoch[10] Train-mse=4.074934 2019-02-27 14:51:57,538:root:INFO: Epoch[10] Time cost=1.157 2019-02-27 14:51:57,601:root:INFO: Epoch[10] Validation-mse=3.578255 2019-02-27 14:51:58,611:root:INFO: Epoch[11] Train-mse=2.713380 2019-02-27 14:51:58,612:root:INFO: Epoch[11] Time cost=1.011 2019-02-27 14:51:58,681:root:INFO: Epoch[11] Validation-mse=2.832748 2019-02-27 14:51:59,680:root:INFO: Epoch[12] Train-mse=2.544748 2019-02-27 14:51:59,681:root:INFO: Epoch[12] Time cost=0.999 2019-02-27 14:51:59,740:root:INFO: Epoch[12] Validation-mse=3.092040 2019-02-27 14:52:00,819:root:INFO: Epoch[13] Train-mse=2.752240 2019-02-27 14:52:00,820:root:INFO: Epoch[13] Time cost=1.080 2019-02-27 14:52:00,894:root:INFO: Epoch[13] Validation-mse=2.507709 2019-02-27 14:52:01,868:root:INFO: Epoch[14] Train-mse=2.272493 2019-02-27 14:52:01,869:root:INFO: Epoch[14] Time cost=0.974 2019-02-27 14:52:01,939:root:INFO: Epoch[14] Validation-mse=2.545414 2019-02-27 14:52:02,992:root:INFO: Epoch[15] Train-mse=1.962754 2019-02-27 14:52:02,993:root:INFO: Epoch[15] Time cost=1.053 2019-02-27 14:52:03,052:root:INFO: Epoch[15] Validation-mse=3.936031 2019-02-27 14:52:04,065:root:INFO: Epoch[16] Train-mse=2.390086 2019-02-27 14:52:04,065:root:INFO: Epoch[16] Time cost=1.013 2019-02-27 14:52:04,142:root:INFO: Epoch[16] Validation-mse=4.206038 2019-02-27 14:52:05,148:root:INFO: Epoch[17] Train-mse=3.163517 2019-02-27 14:52:05,149:root:INFO: Epoch[17] Time cost=1.006 2019-02-27 14:52:05,209:root:INFO: Epoch[17] Validation-mse=4.414568 2019-02-27 14:52:06,207:root:INFO: Epoch[18] Train-mse=3.450250 2019-02-27 14:52:06,207:root:INFO: Epoch[18] Time cost=0.997 2019-02-27 14:52:06,273:root:INFO: Epoch[18] Validation-mse=5.286100 2019-02-27 14:52:07,255:root:INFO: Epoch[19] Train-mse=2.802456 2019-02-27 14:52:07,256:root:INFO: Epoch[19] Time cost=0.982 2019-02-27 14:52:07,329:root:INFO: Epoch[19] Validation-mse=3.315309
print(time2-time1)
21.75580096244812
y_sym_pred = sym_model_module.predict(sym_valid_iter)
s = sklearn.metrics.mean_squared_error(y_test, y_sym_pred.asnumpy())
s
3.3060569400329425
sklearn.metrics.explained_variance_score(y_test, y_sym_pred.asnumpy())
0.9966796086836532