#!/usr/bin/env python
# coding: utf-8

# # Genre recognition: experiment

# Goal: observe the effect of convergence on a small dataset (500 songs, 149 frames, $m=128$) with $\lambda_g=100$.
# 
# Conclusion: Relative stopping criterion works well on the outer loop. The stopping criterion `rtol=1e-3` is too restrictive and reduces performance over several metrics.
# 
# Observations:
# * Relative stopping criterion works well on the outer loop. Remove the need to set `N_outer`.
# * Objective and sparsity diminishes linearly with rtol.
# * The gap between the objective functions reduces to one order of magnitude.
# * Some iterations are needed to let some $Z$ coefficients grow while others vanish.
# * Running time is exponential to `1/rtol`.
# * `rtol=1e-5` is sufficient for `lg=100`.
# * The minimum `rtol` may depend on `lg`. Larger `lg` needs smaller `rtol`.
# * Ran for 2h.

# In[10]:


time_features = [63, 945, 2205, 3187]
objective = [1.853179e+05, 1.507532e+05, 1.433319e+05, 1.423020e+05]
inner_iterations = [34, 445, 1050, 1553]
outer_iterations = [2, 5, 6, 9]
sparsity = [78.9, 26.7, 19.9, 19.0]
objective_g = [1.500593e+05, 9.228578e+04, 7.472236e+04, 7.236831e+04]
objective_i = [3.444047e+04, 5.142558e+04, 5.758356e+04, 5.831160e+04]
objective_j = [8.181773e+02, 7.041881e+03, 1.102598e+04, 1.162210e+04]
accuracy = [55, 74, 74, 75]

import numpy as np
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')

def plot(*args, **kwargs):
    plt.figure(figsize=(8,5))
    x = range(len(Pvalues))
    log = 'log' in kwargs and kwargs['log'] is True
    pltfunc = plt.semilogy if log else plt.plot
    for var in args:
        pltfunc(x, globals()[var], '.-', label=var)
    plt.xlim(0, len(Pvalues)-1)
    plt.title('{} vs {}'.format(', '.join(args), Pname))
    plt.xlabel(Pname)
    plt.ylabel(' ,'.join(args))
    plt.xticks(x, Pvalues)
    plt.grid(True); plt.legend(loc='best'); plt.show()

plot('sparsity')
plot('objective')
plot('objective_g', 'objective_i', 'objective_j', log=True)
plot('accuracy')
plot('time_features')
plot('inner_iterations')
plot('outer_iterations')


# ## Hyper-parameters

# ### Parameter under test

# In[7]:


Pname = 'rtol'
Pvalues = [1e-3, 1e-4, 1e-5, 1e-6]

# Regenerate the graph or the features at each iteration.
regen_graph = False
regen_features = True


# ### Model parameters

# In[2]:


p = {}

# Preprocessing.

# Graph.
p['K'] = 10 + 1  # 5 to 10 + 1 for self-reference
p['dm'] = 'cosine'
p['Csigma'] = 1
p['diag'] = True
p['laplacian'] = 'normalized'

# Feature extraction.
p['m'] = 128  # 64, 128, 512
p['ld'] = 10
p['le'] = None
p['lg'] = 100

# Classification.
p['scale'] = None
p['Nvectors'] = 6
p['svm_type'] = 'C'
p['kernel'] = 'linear'
p['C'] = 1
p['nu'] = 0.5


# ### Numerical parameters

# In[3]:


# Dataset (10,100,644 | 5,100,149 | 2,10,644).
p['Ngenres'] = 5
p['Nclips'] = 100
p['Nframes'] = 149

# Graph.
p['tol'] = 1e-5

# Feature extraction.
p['rtol'] = 1e-3  # 1e-3, 1e-5, 1e-7
p['N_outer'] = 40  # 10, 15, 20

# Classification.
p['Ncv'] = 10


# ## Processing

# In[4]:


import numpy as np
import time

texperiment = time.time()

def separator():
    print('\n' + 50 * '-' + '\n')
    # Fair comparison when tuning parameters.
    np.random.seed(1)

#%run gtzan.ipynb
#%run audio_preprocessing.ipynb
if not regen_graph:
    get_ipython().run_line_magic('run', 'audio_graph.ipynb')
    separator()
if not regen_features:
    get_ipython().run_line_magic('run', 'audio_features.ipynb')
    separator()

# Hyper-parameter under test.
for p[Pname] in Pvalues:

    if regen_graph:
        get_ipython().run_line_magic('run', 'audio_graph.ipynb')
        separator()
    if regen_features:
        get_ipython().run_line_magic('run', 'audio_features.ipynb')
        separator()
    get_ipython().run_line_magic('run', 'audio_classification.ipynb')
    separator()

print('Experiment time: {:.0f} seconds'.format(time.time() - texperiment))