import ensemble
import features
reload(ensemble)
reload(features)
from itertools import cycle
import numpy as np
import random
from IPython.parallel import Client
client = Client()
print len(client)

import cPickle
X, y = cPickle.load(open('data/blackbox.pkl', 'rb'))
print X.shape, y.shape
print np.unique(y)

## clear and create ensemble
!rm -fR tmp/blackbox_ensemble_iii/
ensemble_path = ensemble.new_ensemble('blackbox_ensemble_iii', 'tmp/')
print ensemble_path

## tri-kmeans features 
reload(features)
tri_kmeans = features.TriKmeansFeatures(n_clusters = 20, 
                        feat_patches = features.bootstrap_seqs(range(n_features), 
                                                                  n_iter = 500, subsize=20), 
                        client = client)
tri_X = tri_kmeans.fit_transform(X)
feature_patches = features.strided_seqs(range(tri_X.shape[1]), stride = 15, subsize=30)
data_name_prefix = 'tri_%d_%d'
for feats in feature_patches:
    train_X = features.patch(tri_X, train_index, feats)
    train_y = features.patch(y, train_index)
    test_X = features.patch(tri_X, test_index, feats)
    test_y = features.patch(y, test_index)
    train_name = 'train_' + data_name_prefix % (feats[0], feats[-1])
    test_name = 'test_' + data_name_prefix % (feats[0], feats[-1])
    data_records.append([train_name, (train_X, train_y), {}])
    data_records.append([test_name, (test_X, test_y), {}])
    data_names.append((train_name, test_name))

## Write data in batch
ensemble.batch_write_data(ensemble_path, data_records)
print len(data_names)

## different model configurations 
from sklearn import svm
from sklearn import linear_model
from sklearn import tree
from sklearn.grid_search import IterGrid
models = []
## tree models
tree_params = IterGrid({'criterion': ['gini', 'entropy'], 'max_depth': range(5, 16)})
for param in tree_params:
    model_name = 'tree_%s_%d' % (param['criterion'], param['max_depth'])
    models.append((model_name, tree.DecisionTreeClassifier(**param)))
"""
## svc models - slow to train and evaluate, no obvious accuracy improvement
svc_params = IterGrid({'C': np.logspace(1, 5, 5), 
                        'gamma': np.logspace(-5, 2, 8), 
                        'probability': [True]})
for param in svc_params:
    model_name = 'svc_%g_%g' % (param['C'], param['gamma'])
    models.append((model_name, svm.SVC(**param)))
"""
## linear models
"""
sgd_params = IterGrid({'penalty': ['l1', 'l2', 'elasticnet'], 
                       'alpha': np.logspace(-5, 2, 8)})
for param in sgd_params:
    model_name = 'sgd_%s_%g' % (param['penalty'], param['alpha'])
    models.append((model_name, linear_model.SGDClassifier(**param)))
"""

## write models into ensemble
print len(model_names)
selected_models = range(len(model_names))
from random import shuffle
shuffle(selected_models)
selected_models = selected_models[:10000]
model_names = np.array(model_names)[selected_models]
model_records = np.array(model_records)[selected_models]
print len(model_names)
ensemble.batch_write_model(ensemble_path, model_records)

## train models in parallel
model_data_pairs = zip(model_names, cycle(['train_data']))
ensemble.parallel_train_models(ensemble_path, model_data_pairs, client)

## construct the greedy ensemble
reload(ensemble)

## DISCRETE CLASSIFICATION
from sklearn.metrics import accuracy_score
ge = ensemble.GreedyEnsemble(ensemble_path, 
                             scorefn = ensemble.GreedyEnsemble.score_label_classification,
                             votefn = ensemble.GreedyEnsemble.vote_major_class,
                             client = client)
"""
## PROBALISTIC CLASSIFICATION
ge = ensemble.GreedyEnsemble(ensemble_path, 
                            scorefn = ensemble.GreedyEnsemble.score_label_prob_classification,
                            votefn = ensemble.GreedyEnsemble.vote_average,
                            client = client)
"""
ge.fit(model_names, verbose = True)

## performance checking on training data
print ge.score(data_type = 'train_data')

## performance checking on validation_data
print ge.score(data_type = 'validation_data')

!ls tmp/blackbox_ensemble_iii_optimal/

print 'current number of models in the optimal ensemble',len(ensemble.all_model_names(optimal_ensemble_path))
print ensemble.all_model_names(optimal_ensemble_path)
for model_name in ensemble.all_model_names(optimal_ensemble_path):
    ensemble.update_model_record(optimal_ensemble_path, model_name, {'is_probabilistic': False})

print gee.score('train_data')

print gee.score('validation_data')