#!/usr/bin/env python
# coding: utf-8

# <img src='https://raw.githubusercontent.com/autonomio/hyperio/master/logo.png' width=250px>

# This notebook is a compliment to the *Hyperparameter Optimization on Keras* article. 

# ## Overview
# 
# There are four steps to setting up an experiment with Talos:
# 
# 1) Imports and data
# 
# 2) Creating the Keras model
# 
# 3) Defining the Parameter Space Boundaries 
# 
# 4) Running the Experiment

# ## 1. The Required Inputs and Data

# In[ ]:


from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dropout, Dense

get_ipython().run_line_magic('matplotlib', 'inline')

import sys
sys.path.insert(0, '/Users/mikko/Documents/GitHub/talos')
import talos


# In[ ]:


# then we load the dataset
x, y = talos.templates.datasets.breast_cancer()

# and normalize every feature to mean 0, std 1
x = talos.utils.rescale_meanzero(x)


# ## 2. Creating the Keras Model

# In[ ]:


# first we have to make sure to input data and params into the function
def breast_cancer_model(x_train, y_train, x_val, y_val, params):
    
    from tensorflow.keras.layers import Dense, Dropout
    from tensorflow.keras.models import Sequential

    model = Sequential()
    model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1],
                    activation=params['activation'],
                    kernel_initializer=params['kernel_initializer']))
    
    model.add(Dropout(params['dropout']))

    model.add(Dense(1, activation=params['last_activation'],
                    kernel_initializer=params['kernel_initializer']))
    
    model.compile(loss=params['losses'],
                  optimizer=params['optimizer'],
                  metrics=['acc', talos.utils.metrics.f1score])
    
    history = model.fit(x_train, y_train, 
                        validation_data=[x_val, y_val],
                        batch_size=params['batch_size'],
                        callbacks=[talos.callbacks.TrainingPlot(metrics=['f1score'])],
                        epochs=params['epochs'],
                        verbose=0)

    return history, model


# ## 3. Defining the Parameter Space Boundary

# In[ ]:


# then we can go ahead and set the parameter space
p = {'first_neuron':[9, 10, 11],
     'hidden_layers':[0, 1, 2],
     'batch_size': [30],
     'epochs': [100],
     'dropout': [0],
     'kernel_initializer': ['uniform','normal'],
     'optimizer': ['Adagrad', 'Adam'],
     'losses': ['binary_crossentropy'],
     'activation':['relu', 'elu'],
     'last_activation': ['sigmoid']}


# ## 4. Starting the Experiment

# In[ ]:


# and run the experiment
t = talos.Scan(x=x,
               y=y,
               model=breast_cancer_model,
               params=p,
               experiment_name='breast_cancer',
               round_limit=50,
               disable_progress_bar=True)