#!/usr/bin/env python
# coding: utf-8

# <img src='https://raw.githubusercontent.com/autonomio/hyperio/master/logo.png' width=250px>

# In[ ]:


import talos
import pandas as pd

get_ipython().run_line_magic('matplotlib', 'inline')


# ## Table of Contents

# ##### 1. <a href=#one>Data Preparation</a>
# ##### 2. <a href=#two>Model Preparation</a>
# ##### 3. <a href=#three>Setting the Parameter Space Boundaries</a>
# ##### 4. <a href=#four>Run the Hyperparameter <code>Scan()</code></a>
# ##### 5. <a href=#five>Access the results through the <code>Scan</code> object</a>
# ##### 6. <a href=#six>Analysing the Scan results with <code>Reporting()</code></a>
# ##### 7. <a href=#seven>Evaluating Models with <code>Evaluate()</code></a>
# ##### 8. <a href=#eight>Deploying Models with <code>Deploy()</code></a>
# ##### 9. <a href=#nine>Restoring Models with <code>Restore()</code></a>

# ### 1. Data Preparation <a name="one"></a> 

# For this experiment, we're going to use the famous Iris dataset. 

# In[ ]:


x, y = talos.templates.datasets.iris()


# ### 2. Model Preparation  <a name="two"></a> 
# Talos works with any Keras model, without changing the structure of the model in anyway, or without introducing any new syntax. The below example shows clearly how this works. 
# 
# For this example, we have to import two helper functions from Talos, one for early stopping callout, and the other for using normalized learning rate values. Because we might want to work on trying out several optimizers in a single scan, without normalization, inputting of the values would become cumbersome.

# In[ ]:


from talos.utils import lr_normalizer


# Note that the only difference in the model below is how instead of using a label or value to define a given model parameter, we do it using a dictionary label. Also for optimizer we are using a learning rate parameter, which involves the use of two dictionary labels.

# In[ ]:


from keras.models import Sequential
from keras.layers import Dropout, Dense

def iris_model(x_train, y_train, x_val, y_val, params):
    
    model = Sequential()                            
    model.add(Dense(params['first_neuron'],
                    input_dim=x_train.shape[1],
                    activation='relu'))
    
    model.add(Dropout(params['dropout']))
    model.add(Dense(y_train.shape[1],
                    activation=params['last_activation']))

    model.compile(optimizer=params['optimizer'](lr=lr_normalizer(params['lr'], params['optimizer'])),
                  loss=params['loss'],
                  metrics=['acc'])

    out = model.fit(x_train, y_train,
                    batch_size=params['batch_size'],
                    epochs=params['epochs'],
                    verbose=0,
                    validation_data=[x_val, y_val])
    
    return out, model


# ### 3. Setting the Parameter Space Boundaries <a name="three"></a> 
# In the last and final step, we're going to create the dictionary, which will then be passed on to Talos together with the model above. Here we have three different ways to input values:
# 
# - as stepped ranges (min, max, steps)
# - as multiple values [in a list]
# - as a single value [in a list]
# 
# For values we don't want to use, it's ok to set it as None.
# 
# NOTE: at this point you have to import from Keras the optimizer, activations, and losses you want to scan for.

# In[ ]:


from keras.optimizers import Adam, Nadam
from keras.activations import softmax
from keras.losses import categorical_crossentropy, logcosh

p = {'lr': (0.1, 10, 10),
     'first_neuron':[4, 8, 16, 32, 64, 128],
     'batch_size': [2, 3, 4],
     'epochs': [200],
     'dropout': (0, 0.40, 10),
     'optimizer': [Adam, Nadam],
     'loss': ['categorical_crossentropy'],
     'last_activation': ['softmax'],
     'weight_regulizer': [None]}


# ### 4. Run the Hyperparameter <code>Scan()</code> <a name="four"></a> 
# Now we are ready to run the model based on the parameters and the layer configuration above. The exact same process would apply with any other model, just make sure to pass the model function name in the Scan() command as in the below example. To get started quickly, we're going to invoke the 'grid_downsample' parameter to 1/100 of the entire permutations.

# In[ ]:


scan_object = talos.Scan(x,
                         y, 
                         params=p,
                         model=iris_model,
                         experiment_name='iris',
                         fraction_limit=.001)


# ### 5. Access the results through the <code>Scan</code> object <a name="five"></a> 

# In[ ]:


# accessing the results data frame
scan_object.data.head()

# accessing epoch entropy values for each round
scan_object.learning_entropy

# access the summary details
scan_object.details


# In addition to statistics and meta-data related with the Scan, the used data (x and y) together with the saved model and model weights for each hyperparameter permutation is stored in the Scan object. 

# In[ ]:


# accessing the saved models
scan_object.saved_models

# accessing the saved weights for models
scan_object.saved_weights


# The Scan object can be further used, and is required, as input for Predict(), Evaluate(), and Deploy(). More about this in the corresponding sections below.

# ### 6. Analysing the Scan results with <code>Reporting()</code> <a name="six"></a> 

# In the Scan process, the results are stored round-by-round in the corresponding experiment log which is a .csv file stored in the present working directory. The Reporting() accepts as its source either a file name, or the Scan object. 

# In[ ]:


# use Scan object as input
analyze_object = talos.Analyze(scan_object)


# In[ ]:


# access the dataframe with the results
analyze_object.data


# In[ ]:


# get the number of rounds in the Scan
analyze_object.rounds()

# get the highest result for any metric
analyze_object.high('val_acc')

# get the round with the best result
analyze_object.rounds2high('val_acc')

# get the best paramaters
analyze_object.best_params('val_acc', ['acc', 'loss', 'val_loss'])

# get correlation for hyperparameters against a metric
analyze_object.correlate('val_loss', ['acc', 'loss', 'val_loss'])


# In addition to the key obsevations, several useful plots are available for analysis of the results.

# In[ ]:


# a regression plot for two dimensions 
analyze_object.plot_regs('val_acc', 'val_loss')

# line plot
analyze_object.plot_line('val_acc')

# up to two dimensional kernel density estimator
analyze_object.plot_kde('val_acc')

# a simple histogram
analyze_object.plot_hist('val_acc', bins=50)

# heatmap correlation
analyze_object.plot_corr('val_loss', ['acc', 'loss', 'val_loss'])

# a four dimensional bar grid
analyze_object.plot_bars('batch_size', 'val_acc', 'first_neuron', 'lr')


# ### 7. Evaluating Models with <code>Evaluate()</code> <a name="seven"></a> 

# Models can be evaluated with <code>Evaluate()</code> against a k-fold cross-validation. Ideally at least 50% of the data, or more if possible, is kept completely out of the <code>Scan</code> process and only exposed into Evaluate once one or more candidate models have been identified.

# In[ ]:


evaluate_object = talos.Evaluate(scan_object)
evaluate_object.evaluate(x, y, folds=10, metric='val_acc', task='multi_label')


# Once a sufficiently performing model have been found, a deployment package can be easily created.

# ### 8. Deploying Models with <code>Deploy()</code> <a name="eight"></a> 

# Once the right model or models have been found, you can create a deployment package with <code>Deploy()</code> which is then easy to transfer to a production or other environment, send via email, or upload to shared remote location. Best model is automatically chosen based on a given metric ('val_acc' by default).
# 
# The Deploy package is a zip file that consist of: 
# 
# - details of the scan
# - model weights
# - model json
# - results of the experiment
# - sample of x data
# - sample of y data
# 
# The <code>Deploy</code> package can be easily restored with <code>Restore()</code> which is covered in the next section.

# In[ ]:


talos.Deploy(scan_object=scan_object, model_name='iris_deploy', metric='val_acc');


# ### 9. Restoring Models with <code>Restore()</code> <a name="nine"></a> 

# Models can be evaluated with <code>Evaluate()</code> against a k-fold cross-validation. Ideally at least 50% of the data, or more if possible, is kept completely out of the <code>Scan</code> process and only exposed into Evaluate once one or more candidate models have been identified.

# In[ ]:


iris = talos.Restore('iris_deploy.zip')


# The <code>Restore</code> object now consists of the assets from the Scan object originally associated with the experiment, together with the model that had been picked as 'best'. The model can be immediately used for making prediction, or use in any other other way Keras model objects can be used.

# In[ ]:


# make predictions with the model
iris.model.predict(x)


# In addition, for book keeping purpose, and for simplicity of sharing models with team members and other stakeholders, various attributes are included in the <code>Restore</code> object:

# In[ ]:


# get the meta-data for the experiment
iris.details


# In[ ]:


# get the hyperparameter space boundary
iris.params


# In[ ]:


# sample of x and y data
iris.x
iris.y


# In[ ]:


# the results dataframe
iris.results


# 
# <img align=left src='https://img00.deviantart.net/920c/i/2014/137/a/2/pinkie_pie_thats_all_folks_by_dan232323-d7ipnd4.jpg' width=400>

# Back to the repository page >> http://github.com/autonomio/talos