Comparative Evaluation and Combination of Audio Tempo Estimation Approaches (2011)¶

Results of the conference paper:¶

"Comparative Evaluation and Combination of Audio Tempo Estimation Approaches" , Zapata, Jose R.; Gómez, Emilia. AES 42nd International Conference: Semantic Audio, Ilmenau, Germany. pp. 198 - 207, Jul 2011.¶

http://www.aes.org/e-lib/browse.cfm?elib=15964 ¶

In [1]:

# Importing libraries
% matplotlib inline
import pandas as pd
import matplotlib.pyplot as plt
import plotly.offline as pyo # offline plotly plots
from plotly.offline import init_notebook_mode #Run plotly in jupyter
import plotly.graph_objs as go # Plotly graphs objects
init_notebook_mode() # Init plotly inside jupyter notebook

In [2]:

# Reading tempo estimations results
df = pd.read_excel("data/AES42_Tempo_2011.xlsx")
number_songs = len(df)
# only the first 5 rows
df.head()

Out[2]:

	SONG	GROUNDTRUTH	Essentia	SonicBasic	qmtempo	Aubio	Beatroot	Ellis	Tzanetakis	MIRTempo	...	Alo_spec	Dix_auco	Dix_indu	Dix_trac	Klapuri	Scheirer	Tzan_hist	Tzan_mmul	Tzan_msum	Uhle
0	Abba\Gold-GreatestHits\20sec\01-DancingQueen.wav	101.010084	201.0	100.3	103.36	101.332721	100.000000	135.135135	100.0	100.748146	...	80.2881	152	202.772	201.439	101	101.8190	198	100	100	67
1	Abba\Gold-GreatestHits\20sec\02-KnowingMe,Know...	105.755105	213.0	105.7	107.67	107.666016	109.090909	108.695652	106.5	107.416010	...	106.7432	216	106.400	212.720	108	108.0350	107	107	107	107
2	Abba\Gold-GreatestHits\20sec\03-TakeAChanceOnM...	107.142857	213.0	106.6	107.67	107.666016	105.263158	107.142857	107.5	107.171263	...	80.2881	214	106.751	214.826	108	220.0000	106	105	107	106
3	Abba\Gold-GreatestHits\20sec\04-MammaMia.wav	136.617092	136.0	136.9	139.67	135.999178	136.363636	136.363636	137.0	137.089537	...	68.3679	140	136.428	137.335	138	91.7879	135	135	135	135
4	Abba\Gold-GreatestHits\20sec\05-LayAllYourLove...	133.636421	132.0	132.5	136.00	132.512019	136.363636	66.371681	132.5	132.863812	...	133.1982	133	132.893	132.921	133	132.9380	132	132	132	132

5 rows × 25 columns

Tempo Ground truth histogram¶

In [3]:

df["GROUNDTRUTH"].hist(bins=30,ec="black",figsize=(12,6))
plt.suptitle("Songs Ground truth Histogram")
plt.xlabel("Tempo BPM")
plt.ylabel("Number of instances");

Defining evaluation functions¶

In [4]:

# Tempo evaluation measure 1 
def matchingFunction_1(computed,truth,accPerc):
    #normally accPerc = 4
    if ((computed >= truth*(1-accPerc/100))&(computed <= truth*(1+accPerc/100))):
        MF_1 = 1
    else:
        MF_1 = 0;
    return MF_1

In [5]:

# Tempo evaluation measure 2 
def matchingFunction_2(computed,truth,accPerc):
    # normally accuracy 4%
    if (    ((computed>=truth*(1-accPerc/100))&(computed<=truth*(1+accPerc/100))) |
        ((computed>=2*truth*(1-accPerc/100))&(computed<=2*truth*(1+accPerc/100))) |
        ((computed>=0.5*truth*(1-accPerc/100))&(computed<=0.5*truth*(1+accPerc/100))) |
        ((computed>=3*truth*(1-accPerc/100))&(computed<=3*truth*(1+accPerc/100))) |
        ((computed>=(1/3)*truth*(1-accPerc/100))&(computed<=(1/3)*truth*(1+accPerc/100)))):
        MF_2 = 1
    else:
        MF_2 = 0
    return MF_2    

In [6]:

cols_names = list(df)
del cols_names[0:2] # only Tempo estimator systems
evaluation_1 = df.loc[:, cols_names]
evaluation_2 = df.loc[:, cols_names]

In [7]:

for n in cols_names:
    evaluation_1[n] = [matchingFunction_1(df.loc[idx,n],df.loc[idx,"GROUNDTRUTH"],4) for idx in range(len(df))]
    evaluation_2[n] = [matchingFunction_2(df.loc[idx,n],df.loc[idx,"GROUNDTRUTH"],4) for idx in range(len(df))]
metric_1 = (evaluation_1.sum())*100/number_songs
metric_2 = (evaluation_2.sum())*100/number_songs    

General Evaluation Results¶

In [8]:

final_evaluation = pd.concat([metric_1,metric_2],axis=1)
final_evaluation.columns = ['Metric 1','Metric 2']
final_evaluation.style.highlight_max() # High lighting the maximun value per column

Out[8]:

	Metric 1	Metric 2
Essentia	24.5161	83.4409
SonicBasic	24.7312	50.7527
qmtempo	43.2258	80.4301
Aubio	39.3548	67.3118
Beatroot	23.2258	67.957
Ellis	45.5914	80.6452
Tzanetakis	25.5914	66.4516
MIRTempo	30.9677	65.5914
IBT	35.914	79.7849
BeatIt	60.4301	78.2796
Mpeg7-xm	48.3871	70.5376
jAudio	5.16129	32.2581
Alo_corr	23.4409	58.2796
Alo_spec	37.4194	68.6022
Dix_auco	16.9892	76.9892
Dix_indu	28.6022	62.5806
Dix_trac	19.3548	68.8172
Klapuri	58.4946	91.1828
Scheirer	37.8495	69.4624
Tzan_hist	21.2903	47.7419
Tzan_mmul	18.7097	41.0753
Tzan_msum	27.5269	52.4731
Uhle	41.9355	71.828

In [9]:

trace1 = go.Bar(
    x=final_evaluation.index,  # Algorithms names
    y=final_evaluation['Metric 1'],
    name = 'Metric 1'
)
trace2 = go.Bar(
    x=final_evaluation.index,  # Algorithms names
    y=final_evaluation['Metric 2'],
    name='Metric 2'
) 
data = [trace1, trace2]
layout = go.Layout(title='Tempo Evaluation results')
fig = go.Figure(data=data, layout=layout)
pyo.iplot(fig, filename='Bar-plot')

In [10]:

# final_evaluation.plot(kind='bar',title ="Evaluation Results", figsize=(16,8),grid = True,fontsize=12);
# Simple plot but not interactive

Sorted General Evaluation results (by Metric 1)¶

In [11]:

sorted_final_evaluation1 = final_evaluation.sort_values(by=['Metric 1'], ascending= False)
sorted_final_evaluation1.style.highlight_max() # High lighting the maximun value per column

Out[11]:

	Metric 1	Metric 2
BeatIt	60.4301	78.2796
Klapuri	58.4946	91.1828
Mpeg7-xm	48.3871	70.5376
Ellis	45.5914	80.6452
qmtempo	43.2258	80.4301
Uhle	41.9355	71.828
Aubio	39.3548	67.3118
Scheirer	37.8495	69.4624
Alo_spec	37.4194	68.6022
IBT	35.914	79.7849
MIRTempo	30.9677	65.5914
Dix_indu	28.6022	62.5806
Tzan_msum	27.5269	52.4731
Tzanetakis	25.5914	66.4516
SonicBasic	24.7312	50.7527
Essentia	24.5161	83.4409
Alo_corr	23.4409	58.2796
Beatroot	23.2258	67.957
Tzan_hist	21.2903	47.7419
Dix_trac	19.3548	68.8172
Tzan_mmul	18.7097	41.0753
Dix_auco	16.9892	76.9892
jAudio	5.16129	32.2581

In [12]:

trace1 = go.Bar(
    x=sorted_final_evaluation1.index,  # Algorithms names
    y=sorted_final_evaluation1['Metric 1'],
    name = 'Metric 1'
)
trace2 = go.Bar(
    x=sorted_final_evaluation1.index,  # Algorithms names
    y=sorted_final_evaluation1['Metric 2'],
    name='Metric 2'
) 
data = [trace1, trace2]
layout = go.Layout(
    title='Tempo Evaluation results (Sorted by Metric 1)'
)
fig = go.Figure(data=data, layout=layout)
pyo.iplot(fig, filename='Bar-plot')

Sorted General Evaluation results (by Metric 2)¶

In [13]:

sorted_final_evaluation2 = final_evaluation.sort_values(by=['Metric 2'], ascending= False)
sorted_final_evaluation2.style.highlight_max() # High lighting the maximun value per column

Out[13]:

	Metric 1	Metric 2
Klapuri	58.4946	91.1828
Essentia	24.5161	83.4409
Ellis	45.5914	80.6452
qmtempo	43.2258	80.4301
IBT	35.914	79.7849
BeatIt	60.4301	78.2796
Dix_auco	16.9892	76.9892
Uhle	41.9355	71.828
Mpeg7-xm	48.3871	70.5376
Scheirer	37.8495	69.4624
Dix_trac	19.3548	68.8172
Alo_spec	37.4194	68.6022
Beatroot	23.2258	67.957
Aubio	39.3548	67.3118
Tzanetakis	25.5914	66.4516
MIRTempo	30.9677	65.5914
Dix_indu	28.6022	62.5806
Alo_corr	23.4409	58.2796
Tzan_msum	27.5269	52.4731
SonicBasic	24.7312	50.7527
Tzan_hist	21.2903	47.7419
Tzan_mmul	18.7097	41.0753
jAudio	5.16129	32.2581

In [14]:

trace1 = go.Bar(
    x=sorted_final_evaluation2.index,  # Algorithms names
    y=sorted_final_evaluation2['Metric 1'],
    name = 'Metric 1'
)
trace2 = go.Bar(
    x=sorted_final_evaluation2.index,  # Algorithms names
    y=sorted_final_evaluation2['Metric 2'],
    name='Metric 2'
) 
data = [trace1, trace2]
layout = go.Layout(title='Tempo Evaluation results (Sorted by Metric 2)')
fig = go.Figure(data=data, layout=layout)
pyo.iplot(fig, filename='Bar-plot')

In [15]:

#final_evaluation.sort_values(by=['Metric 2']).plot(kind='bar',title ="Evaluation Results", figsize=(16,8),grid = True,fontsize=12);
# Simple plot but not interactive

Error relations¶

In [16]:

# Relation between estimated tempo and the ground truth
df2 = pd.read_excel("data/AES42_Tempo_Error_2011.xls")
df2.head(5)

Out[16]:

	SONG	GROUNDTRUTH	Essentia	Sonic Basic	qmtempo	Aubio	Beatroot	Ellis	Tzanetakis	MIRTempo	...	Anon	Dix_auco	Dix_indu	Dix_trac	Klapluri	Scheirer	Tzan_hist	Tzan_mmul	Tzan_msum	Uhle
0	Abba\Gold - Greatest Hits\20 sec\01-Dancing Qu...	101.010084	2	Good	Good	Good	Good	4/3	Good	Good	...	2	3/2	2	2	Good	Good	2	Good	Good	2/3
1	Abba\Gold - Greatest Hits\20 sec\02-Knowing Me...	105.755105	2	Good	Good	Good	Good	Good	Good	Good	...	Good	2	Good	2	Good	Good	Good	Good	Good	Good
2	Abba\Gold - Greatest Hits\20 sec\03-Take A Cha...	107.142857	2	Good	Good	Good	Good	Good	Good	Good	...	Good	2	Good	2	Good	2	Good	Good	Good	Good
3	Abba\Gold - Greatest Hits\20 sec\04-Mamma Mia.wa	136.617092	Good	Good	Good	Good	Good	Good	Good	Good	...	Good	Good	Good	Good	Good	2/3	Good	Good	Good	Good
4	Abba\Gold - Greatest Hits\20 sec\05-Lay All Yo...	133.636421	Good	Good	Good	Good	Good	1/2	Good	Good	...	Good	Good	Good	Good	Good	Good	Good	Good	Good	Good

5 rows × 26 columns

In [17]:

Allerrors = df2.iloc[:,2:].values.copy()
Allerrors.resize(Allerrors.size,1)
ErrorSeries = pd.DataFrame(Allerrors)
ErrorSeries[0]= ErrorSeries[0].astype(str)
#y.plot(kind='bar',title = 'Examen Final');# Simple histogram plot

In [18]:

hist = ErrorSeries[0].value_counts()
datah = [
  go.Bar(
    y = hist.values.astype(str),
    x = hist.index,
    name = "Errors"
    )
]
layout = go.Layout(title='Error Relations in all results')
fig = go.Figure(data=datah, layout=layout)
pyo.iplot(fig, filename='ErrorsHistogram')