Notebook

TriScale Interactive Plots¶

This notebook generates and displays all the plots from the TriScale paper. By default, large versions of the plots are shown. To exactly reproduce the plots from the paper, set PaperPlots=True in the first cell.

Figure 1
- Figure 1.a
- Figure 1.b
Figure 3
Figure 4
Figure 5
Figure 7
Figure 8
Figure 9
Figure 10

In [1]:

# Set to False to view larger versions of the plots
# Set to True  to reproduce the exact plots from the paper
PaperPlot = False

In [2]:

import copy
import json
import os
from pathlib import Path
import yaml
import zipfile

import numpy as np
import pandas as pd
import plotly.graph_objects as go
import plotly.io as pio
pio.renderers.default = "notebook"

import triscale
import triplots

Download Source Files and Data¶

[Back to top]

The entire dataset for this case study is available on Zenodo:

The wget commands below download the required files to reproduce this case study.

The .zip file is ~2.7 GB

In [3]:

# Set `download = True` to download (and extract) the data from this case study
# Eventually, adjust the record_id for the file version you are interested in.

# For reproducing the original TriScale paper, set `record_id = 3666724`

download = False
record_id = 3666724 # v3.0.1 (https://doi.org/10.5281/zenodo.3666724)

files= ['UseCase_Pantheon.zip',
       'UseCase_Glossy.zip'
       'UseCase_VideoStreaming.zip'
       'UseCase_FailureDetection.zip']
if download:
    for file in files:
        print(file)
        url = 'https://zenodo.org/record/'+str(record_id)+'/files/'+file 
        os.system('wget %s' %url)
        if file[-4:] == '.zip':    
            with zipfile.ZipFile(file,"r") as zip_file:
                zip_file.extractall()
        print('Done.')
else: 
    print('Nothing to download')

Nothing to download

In [4]:

import UseCase_Pantheon.pantheon as pantheon
import UseCase_Glossy.flocklab as flocklab
import UseCase_VideoStreaming.videostreaming as vs
import UseCase_FailureDetection.failuredetection as fd

In [5]:

# Select the different output format settings

if PaperPlot:
    output_format = 'CCR'
else:
    output_format = 'online'

if output_format == 'online':
    font_size_px = 14
    linewidth_px = 512
    landscapewidth_px = 654
    plot_path = None
    
    plot_path = Path('plots')

if output_format == 'CCR':
    font_size_pt = 7
    offset = 5 # to compensate for the rounding of unit conversions
    linewidth_pt = 241 - offset  
    landscapewidth_pt = 506 - offset
    
    # 1pt = 1.333px
    font_size_px = int(font_size_pt*1.333)+1
    linewidth_px = int(linewidth_pt*1.333)+1
    landscapewidth_px = int(landscapewidth_pt*1.333)+1

    plot_path = Path('plots_ccr')
    
# Create plot directory if don't exist
if (plot_path is not None) and not (os.path.exists(plot_path)):
    os.mkdir(plot_path)

Figure 1¶

Figure 1a¶

In [6]:

# Set to False to see the full range of the axes
# Set to True  to reproduce the figure shown in the paper
zoom = True

data_path = Path('UseCase_Pantheon/PantheonData/10runs_30s/2019-08-20T15:34:33:+0200')
perf_file = data_path / 'pantheon_perf.json'
meta_file = data_path / 'pantheon_metadata.json'
config_file = Path('UseCase_Pantheon/PantheonData/config.yml')

custom_layout = {
    "title":None,
    "showlegend":False,
    "font":{"size":font_size_px},
    "xaxis":{'range':[100,32],
             'autorange':False,
             'title':{'font':{'size':font_size_px}}},
    "yaxis":{'range':[-7,127],
             'title':{'font':{'size':font_size_px}}},
}

# Adjust the axis ranges and file name
if zoom:
    custom_layout["yaxis"]["range"]=[80,122]
    custom_layout["xaxis"]["range"]=[100,38]
    plot_filename = "plot_zoom_pantheon.pdf"
else: 
    plot_filename = "plot_summary_pantheon.pdf"

if PaperPlot:
    plot_filename = str(plot_path/plot_filename)
else:
    plot_filename = None
    
    
# Adjust the plot size    
if output_format == 'online':
    custom_layout['width'] = 0.8*landscapewidth_px
    custom_layout['height'] = 500
    custom_layout['margin'] = dict(l=65, r=0, t=70, b=65)


if output_format == 'CCR':
    custom_layout['width'] = 0.4*landscapewidth_px
    custom_layout['height'] = 185
    custom_layout['margin'] = dict(l=30, r=0, t=25, b=40)

# Produce the plot
pantheon.plot_pantheon(perf_file, 
                      meta_file, 
                      config_file, 
                      layout=custom_layout,
                      out_name=plot_filename,
                      show=True);

Figure 1b¶

In [7]:

# Set to False to see the full range of the axes
# Set to True  to reproduce the figure shown in the paper
zoom = True

result_dir_path = Path('UseCase_Pantheon/PantheonData/10runs_30s')
result_dir_list = [x for x in result_dir_path.iterdir() if x.is_dir()]
meta_data_file  = 'pantheon_metadata.json'
config_file = Path('UseCase_Pantheon/PantheonData/config.yml')
out_name = Path('UseCase_Pantheon/PantheonData/metrics_wo_convergence.csv')

# Definition of metrics and KPIs for TriScale analysis
convergence = {'expected': True,
               'confidence': 95,  # in %
               'tolerance': 5,    # in %
              }
metric_tput = {'name':'Mean throughput',
               'unit': 'Mbit/s',
               'measure':'mean',
               'bounds':[0,120],  # expected value range
               'tag':'throughput' # do not change the tag
              }
metric_delay = {'name':'95th perc. of one-way delay',
                'unit': 'ms',
                'measure':95,
                'bounds':[0,100], # expected value range
                'tag':'delay'     # do not change the tag
               }
KPI_tput  = {'percentile': 25,
             'confidence': 75,
             'name': 'KPI of throughput metric',
             'unit': 'Mbit/s',
             'bounds':[0,120],    # expected value range
             'tag':'throughput'   # do not change the tag
            }
KPI_delay = {'percentile': 75,
             'confidence': 75,
             'name': 'KPI of one-way delay metric',
             'unit': 'ms',
             'bounds':[0,100],    # expected value range
             'tag':'delay'        # do not change the tag
            }

metric_list = [metric_tput, metric_delay]
kpi_list = [ KPI_tput, KPI_delay ]
metrics, figure = pantheon.compute_metric(result_dir_list, 
                                  meta_data_file, 
                                  convergence,
                                  metric_list,
                                  out_name=out_name,
                                  plot=False,
                                  verbose=False)

custom_layout = {
    "title":None,
    "showlegend":False,
    "font":{"size":font_size_px},
    "xaxis":{'range':[100,32],
             'autorange':False,
             'title':{'font':{'size':font_size_px}}},
    "yaxis":{'range':[-7,127]},
}

# Adjust the Y-axis range and file name
if zoom:
    custom_layout["xaxis"]["range"]=[100,38]
    custom_layout["yaxis"]["range"]=[78,122]
    plot_filename = "plot_zoom_triscale_wo_convergence.pdf"
else: 
    plot_filename = "plot_summary_triscale.pdf"

if PaperPlot:
    plot_filename = str(plot_path/plot_filename)
else:
    plot_filename = None
    
    
# Adjust the plot size 
if output_format == 'online':
    custom_layout['width'] = 0.8*landscapewidth_px
    custom_layout['height'] = 500
    custom_layout['margin'] = dict(l=65, r=0, t=70, b=65)

if output_format == 'CCR':
    custom_layout['width'] = 0.4*landscapewidth_px
    custom_layout['height'] = 185
    custom_layout['margin'] = dict(l=30, r=0, t=25, b=40)

    
series_label = np.sort(metrics['datetime'].unique())
for series_ix in [series_label[0]]:

    # Get the metrics values for one series
    metric_series = metrics.loc[metrics['datetime'] == series_ix]

    # Plot them
    pantheon.plot_triscale_kpi(metric_series, 
                  result_dir_list[0] / meta_data_file, 
                  kpi_list,
                  config_file,
                  layout=custom_layout, 
                  out_name=plot_filename,
                  show=True)

Output retrieved from file. Skipping computation.

Figure 3¶

Figure 3a¶

In [8]:

# Construct the path to the different test results
result_dir_path = Path('UseCase_Pantheon/PantheonData/10runs_30s')
result_dir_list = [x for x in result_dir_path.iterdir() if x.is_dir()]
# Meta data file name
meta_data_file  = 'pantheon_metadata.json'
# Config file name and path
config_file     = Path('UseCase_Pantheon/PantheonData/config.yml')
out_name        = Path('UseCase_Pantheon/PantheonData/metrics.csv')

# Inputs
metric_delay = {'name':'One-way delay',
                'unit': 'ms',
                'measure':95,
                'bounds':[0,100], # expected value range
                'tag':'delay'     # do not change the tag
               }
convergence = {'expected': True,
               'confidence': 95,  # in %
               'tolerance': 5,    # in %
              }

# load the metrics
metrics = pantheon.compute_metric(result_dir_list, 
                                  meta_data_file, 
                                  convergence,
                                  [metric_delay],
                                  out_name = out_name)

data_file = str(result_dir_list[0] / 'cubic_datalink_delay_run1_flow1.csv')

# Plot file name
plot_out_name = "plot_example_metric.pdf"
if PaperPlot:
    plot_out_name = str(plot_path/plot_out_name)
else:
    plot_out_name = None

# Horizontal Y-axis title
ytitle = {
    "x":0,
    "y":1.10,
    "xref":"paper",
    "yref":"paper",
    "text":metric_delay['name'] +' [ '+ metric_delay['unit'] +' ]',
    "showarrow":False,
    "xanchor":'left'
}

custom_layout = {
    "title":None,
    "font":{"size":font_size_px},
    "xaxis":{'title':{'font':{'size':font_size_px},
                      'text':'Time [ s ]'}
            },
    "yaxis":{'title':{'text':''}},
    "annotations":[ytitle]   
}

# Adjust the plot size 
if output_format == 'online':
    custom_layout['width'] = 1.2*landscapewidth_px
    custom_layout['height'] = 400
    custom_layout['margin'] = dict(l=65, r=0, t=70, b=65)
#     custom_layout["legend"] = dict(x=0.1, y=0.1)

if output_format == 'CCR':
    custom_layout['width'] = 0.60*landscapewidth_px
    custom_layout['height'] = 200
    custom_layout['margin'] = dict(l=20, r=0, t=30, b=25)
    custom_layout['showlegend'] = False
    custom_layout["legend"] = dict(xanchor='center',
                                   x=0.5,
                                   yanchor='bottom',
                                   y=0.1, 
                                   font=dict(size=font_size_px-2),
                                   orientation='h')
    
# Produce the plot
triscale.analysis_metric(  data_file,
                           metric_delay,
                           plot=True,
                           plot_out_name=plot_out_name,
                           custom_layout = custom_layout,
                           convergence=convergence);

Output retrieved from file. Skipping computation.

Figure 3b¶

In [9]:

# Construct the path to the different test results
result_dir_path = Path('UseCase_Pantheon/PantheonData/10runs_30s')
result_dir_list = [x for x in result_dir_path.iterdir() if x.is_dir()]
# Meta data file name
meta_data_file  = 'pantheon_metadata.json'
# Config file name and path
config_file     = Path('UseCase_Pantheon/PantheonData/config.yml')
out_name        = Path('UseCase_Pantheon/PantheonData/metrics.csv')

# Inputs
metric_delay = {'name':'One-way delay',
                'unit': 'ms',
                'measure':95,
                'bounds':[0,100], # expected value range
                'tag':'delay'     # do not change the tag
               }
convergence = {'expected': True,
               'confidence': 95,  # in %
               'tolerance': 5,    # in %
              }
KPI_delay = {'percentile': 75,
             'confidence': 75,
             'name': 'One-way delay',
             'unit': 'ms',
             'bounds':[0,100],    # expected value range
             'tag':'delay'        # do not change the tag
            }

# Compute/load the metrics
metrics, figure = pantheon.compute_metric(result_dir_list, 
                                  meta_data_file, 
                                  convergence,
                                  [metric_delay],
                                  out_name = out_name
                                         )

# File name for saving plot
plot_out_name = "plot_example_KPI.pdf"
if PaperPlot:
    plot_out_name = str(plot_path/plot_out_name)
else:
    plot_out_name = None
    
metric_data = np.array(metrics.loc[(metrics['cc'] == 'cubic') &
                          (metrics['datetime'] == result_dir_list[1].stem)].delay_value)

# Compute the KPI
out = triscale.analysis_kpi(metric_data,
                    KPI_delay);
# KPI annotation
note = go.layout.Annotation(
        x=0.5,
        y=1,
        xref="paper",
        yref="paper",
        yanchor="bottom",
        text="KPI: %2.2f ms" % out[1],
        showarrow=False,
        font={'size':font_size_px+2 }
    )

custom_layout = {
    "title":None,
    "width":0.5*linewidth_px,
    "height":100,
    "margin":dict(l=0, r=0, t=5, b=40),
    "font":{"size":font_size_px},
    "showlegend": False,
    "yaxis":{'title':{'font':{'size':font_size_px}}},
    "xaxis":{'title':{'font':{'size':font_size_px},
                      'text':'One-way delay metric [ms]'}},
    "annotations":[note]
}

# Adjust the plot size 
if output_format == 'online':
    custom_layout['width'] = 0.5*linewidth_px
    custom_layout['height'] = 100
    custom_layout['margin'] = dict(l=0, r=0, t=5, b=40)

if output_format == 'CCR':
    custom_layout['width'] = 0.22*landscapewidth_px
    custom_layout['height'] = 60
    custom_layout['margin'] = dict(l=0, r=10, t=15, b=25)
    
# Produce the plot
out = triscale.analysis_kpi(metric_data,
                    KPI_delay,
                    to_plot=['horizontal'],
                    plot_out_name=plot_out_name,
                    custom_layout=custom_layout);

Output retrieved from file. Skipping computation.

Figure 3c¶

In [10]:

KPI_tput  = {'percentile': 25,
             'confidence': 75,
             'name': 'Average Throughput',
             'unit': 'Mbit/s',
             'bounds':[0,120],    # expected value range
             'tag':'throughput'   # do not change the tag
            }
KPI_delay = {'percentile': 75,
             'confidence': 75,
             'name': 'One-way delay',
             'unit': 'ms',
             'bounds':[0,100],    # expected value range
             'tag':'delay'        # do not change the tag
            }
score_delay = {'percentile': 75,
             'confidence': 75,
             'name': '95th perc. of One-way delay',
             'unit': 'ms',
             'bounds':[0,100],    # expected value range
             'tag':'delay'        # do not change the tag
            }
# Compute the KPIs
kpis = pantheon.compute_kpi(metrics,
                            [KPI_tput, KPI_delay],
                            'datetime',
                            plot=False,
                            verbose=False)

plot_out_name = "plot_example_var_score.pdf"
if PaperPlot:
    plot_out_name = str(plot_path/plot_out_name)
else:
    plot_out_name = None

kpi_data = np.array(kpis.loc[(kpis['cc'] == 'cubic')].delay_value)

# Compute the variability score
out = triscale.analysis_variability(kpi_data,
                                    score_delay);

note = go.layout.Annotation(
        x=0.5,
        y=1,
        xref="paper",
        yref="paper",
        yanchor="bottom",
        text="Var. Score: %2.2f ms" % out[3],
        showarrow=False,
        font={'size':font_size_px+2 }
    )
custom_layout = {
    "title":None,
    "showlegend": False,
    "font":{"size":font_size_px},
    "xaxis":{'title':{'font':{'size':font_size_px},
                      'text':'One-way delay KPI [ms]'}
            },
    "annotations":[note]

}
# Adjust the plot size 
if output_format == 'online':
    custom_layout['width'] = 0.5*linewidth_px
    custom_layout['height'] = 100
    custom_layout['margin'] = dict(l=0, r=0, t=5, b=40)

if output_format == 'CCR':
    custom_layout['width'] = 0.22*landscapewidth_px
    custom_layout['height'] = 60
    custom_layout['margin'] = dict(l=0, r=10, t=15, b=25)
    
# Produce the plot
triscale.analysis_variability(kpi_data,
                    score_delay,
                    to_plot=['horizontal'],
                    plot_out_name=plot_out_name,
                    custom_layout=custom_layout);

Invalid metric data (only one data point)
Invalid metric data (only one data point)
Invalid metric data (only one data point)
Invalid metric data (only one data point)

Figure 4¶

In [11]:

# TriScale metric definition
metric_tput = {'name':'Median throughput',
               'unit': 'Mbit/s',
               'measure':50,
               'bounds':[0,120],  # expected value range
               'tag':'throughput' # do not change the tag
              }
convergence = {'expected': True,
               'confidence': 95,  # in %
               'tolerance': 5,    # in %
              }

# Selection of the data to plot
result_dir_path = Path('UseCase_Pantheon/PantheonData/10_20_30_40_50_60s')
result_dir_list = [x for x in result_dir_path.iterdir() if x.is_dir()]
result_dir_list = sorted(result_dir_list)
meta_data_file  = 'pantheon_metadata.json'
sample = {'cc':'ledbat',
          'run':1}

# Horizontal Y axis label
ytitle = {
    "x":0,
    "y":1.15,
    "xref":"paper",
    "yref":"paper",
    "text":'Throughput [ '+ metric_tput['unit'] +' ]',
    "showarrow":False,
    "xanchor":'left'
}

# General layout
custom_layout = {
    'title':None,
    "font":{"size":font_size_px},
    "legend":{'x':.1, 'y':.9,'font':{'size':font_size_px-2}},
    "xaxis":{'title':{'font':{'size':font_size_px},
                      'text':'Time [ s ]',},
             'zeroline':False
            },
    "yaxis":{'title':{'text':''}}, 
    "annotations":[ytitle]
}

# Adjust the plot size 
if output_format == 'online':
    custom_layout['width'] = 0.6*linewidth_px
    custom_layout['height'] = 220
    custom_layout['margin'] = dict(l=40, r=0, t=35, b=40)
    custom_layout["legend"] = dict()

if output_format == 'CCR':
    custom_layout['width'] = 0.27*landscapewidth_px
    custom_layout['height'] = 125
    custom_layout['margin'] = dict(l=25, r=0, t=25, b=30)
    custom_layout["yaxis"]["range"]=[0,110]
    custom_layout["legend"] = dict(xanchor='right',
                                   x=1, 
                                   y=1, 
                                   font=dict(size=font_size_px-2))
    custom_layout['showlegend'] = False

# ordered of tests is all messed-up...
runtimes = [30,10,20,60,40,50]

# Do only the 30s and 60s plots
## 30s
i = 0
plot_out_name = "plot_ledbat_%s_runtime.pdf" % runtimes[i]
metrics_design, figure = pantheon.compute_metric( [result_dir_list[i]], 
                                      meta_data_file, 
                                      convergence,
                                      [metric_tput],
                                      plot=True,
                                      showplot=False,
                                      layout=custom_layout,
                                      verbose=False,
                                      sample=sample)

# Adjust the figure and save
figure.update_traces(marker=dict(size=font_size_px/4))
figure.write_image(str(plot_path/plot_out_name))
figure.show()

# Save plot
if output_format == 'CCR':
    figure.write_image(str(plot_path/plot_out_name))

## 60s
i = 3
custom_layout['width'] = custom_layout['width']*1.68 # make the x axis (roughly) the same scale
plot_out_name = "plot_ledbat_%s_runtime.pdf" % runtimes[i]
metrics_design, figure = pantheon.compute_metric( [result_dir_list[i]], 
                                      meta_data_file, 
                                      convergence,
                                      [metric_tput],
                                      plot=True,
                                      showplot=False,
                                      layout=custom_layout,
                                      verbose=False,
                                      sample=sample)

# Adjust the figure and save
figure.update_traces(marker=dict(size=font_size_px/4))
figure.write_image(str(plot_path/plot_out_name))
figure.show()

# Save plot
if output_format == 'CCR':
    figure.write_image(str(plot_path/plot_out_name))
        
if PaperPlot:
    # Create a figure with only the legend visible
    custom_layout['showlegend'] = True
    custom_layout['width'] = 0.133*landscapewidth_px
    custom_layout['height'] = 100
    custom_layout['margin'] = dict(l=0, r=0, t=0, b=0)
    custom_layout['xaxis'] = dict(visible=False, showline=False)
    custom_layout['yaxis'] = dict(visible=False, showline=False)
    custom_layout['annotations'][0]['visible']=False

    figure.update_layout(custom_layout)
    figure.show()
    figure.write_image(str(plot_path/'plot_ledbat_legend.pdf'))

Figure 5¶

In [12]:

data_file = Path('UseCase_Glossy/Data_FlockLab/2019-08_FlockLab_sky.csv')
df = flocklab.parse_data_file(str(data_file), active_link_threshold=50)
plot_name = 'plot_flocklab_autocorr.pdf'

link_quality_bounds = [0,100]
link_quality_name = 'PRR [%]'

# Produce the plot
fig_theil, fig_autocorr = triscale.network_profiling(df, link_quality_bounds, link_quality_name)

# Adjust layout
custom_layout = {
    'title':None,
    "font":{"size":font_size_px},
    "legend":{'x':.4, 'y':0.9,'font':{'size':font_size_px}},
    "xaxis":{'title':{'font':{'size':font_size_px},
                      'text':'Lag'}
            },
}

# Adjust the plot size 
if output_format == 'online':
    custom_layout['width'] = 1.5*linewidth_px
    custom_layout['height'] = 400
    custom_layout['margin'] = dict(l=40, r=0, t=0, b=35)
    custom_layout["legend"] = dict(xanchor='right',
                                   x=1, 
                                   y=1, 
                                   font=dict(size=font_size_px))
    # convert the displayed values from lag to days
    fig_autocorr.update_xaxes(
        tickvals=[12,84,168,252],
        ticktext=[1,7,14,21])

if output_format == 'CCR':
    custom_layout['width'] = linewidth_px
    custom_layout['height'] = 180
    custom_layout['margin'] = dict(l=20, r=0, t=0, b=35)
    custom_layout["legend"] = dict(xanchor='right',
                                   x=1, 
                                   y=1, 
                                   font=dict(size=font_size_px))
    # convert the displayed values from lag to days
    fig_autocorr.update_xaxes(
        tickvals=[12,84,168,252],
        ticktext=[1,7,14,21])

# Show plot
fig_autocorr.update_layout(custom_layout)
fig_autocorr.update_xaxes(title_text='Lag (days)')
fig_autocorr.show()

# Save plot
if output_format == 'CCR':
    fig_autocorr.write_image(str(plot_path/plot_name))

Figure 7¶

In [13]:

# Path to test results
result_dir_path = Path('UseCase_VideoStreaming/FCC/linear')

# Metadata
protocol_list = [x.stem for x in result_dir_path.iterdir()]
protocol_list = list(set(protocol_list)) # make list elements unique
config_file = Path('UseCase_VideoStreaming/config.yml')

# Define the KPIs
KPI_percentiles = np.arange(2,100,2) # percentiles
KPI_confidence = 95                  # confidence level
KPI_base  = {'confidence': KPI_confidence,
             'bound': 'lower',
             'bounds': [-3,3],
             'unit': '',
            }
KPI_list = []
for p in KPI_percentiles:
    kpi = copy.deepcopy(KPI_base)
    kpi['percentile'] = p
    kpi['name'] = 'P%d'%p
    KPI_list.append(kpi)
    
# Compute/load KPIs values
out_name = Path('UseCase_VideoStreaming') / 'kpis.csv'
QoE = vs.compute_kpi(
    protocol_list,
    KPI_list,
    result_dir_path,
    out_name=out_name
)

plot_out_name = "plot_vs_pensieve.pdf"
sample = dict(
    sample_cdf=True,
    protocol=['pensieve']
)
figure = vs.plot_cdf(
    QoE,
    config_file,
    result_dir_path,
    sample=sample
)

# Set layout
custom_layout = {
    "font":{"size":font_size_px},
    "xaxis":{'title':{'font':{'size':font_size_px}},
             'rangemode':'nonnegative'},
    "yaxis":{'title':{'font':{'size':font_size_px}}},
    "legend": dict(x=0.97, 
                   y=0.1, 
                   orientation='v',
                   xanchor='right',
                   font=dict(size=font_size_px))
}

# Adjust the plot size 
if output_format == 'online':
    custom_layout['width'] = 1.5*linewidth_px
    custom_layout['height'] = 300
    custom_layout['margin'] = dict(l=40, r=0, t=35, b=40)
    custom_layout["legend"] = dict()
if output_format == 'CCR':
    custom_layout["width"] = 0.8*linewidth_px
    custom_layout["height"] = 120
    custom_layout["margin"] = dict(l=35, r=0, t=0, b=25)
    
figure.update_layout(custom_layout)
figure.show()

# Save plot
if output_format == 'CCR':
    figure.write_image(str(plot_path/plot_out_name))

Output retrieved from file. Skipping computation.

Figure 8¶

In [14]:

result_dir_path = Path('UseCase_Pantheon/PantheonData/10runs_30s')
result_dir_list = [x for x in result_dir_path.iterdir() if x.is_dir()]
meta_data_file  = 'pantheon_metadata.json'
config_file     = Path('UseCase_Pantheon/PantheonData/config.yml')
convergence = {'expected': False,
               'confidence': 95,  # in %
               'tolerance': 5,    # in %
              }
metric_tput = {'name':'Median throughput',
               'unit': 'MBit/s',
               'measure':'mean',
               'bounds':[0,120],  # expected value range
               'tag':'throughput' # do not change the tag
              }
metric_delay = {'name':'95th perc. of one-way delay',
                'unit': 'ms',
                'measure':95,
                'bounds':[0,100], # expected value range
                'tag':'delay'     # do not change the tag
               }
KPI_tput  = {'percentile': 25,
             'confidence': 75,
             'name': 'KPI throughput',
             'unit': 'MBit/s',
             'bounds':[0,120],    # expected value range
             'tag':'throughput'   # do not change the tag
            }
KPI_delay = {'percentile': 75,
             'confidence': 75,
             'name': 'KPI One-way delay',
             'unit': 'ms',
             'bounds':[0,100],    # expected value range
             'tag':'delay'        # do not change the tag
            }
score_tput  = {'percentile': 75,
             'confidence': 75,
             'name': 'Throughput',
             'unit': 'MBit/s',
             'bounds':[0,120],    # expected value range
             'tag':'throughput'   # do not change the tag
            }
score_delay = {'percentile': 75,
             'confidence': 75,
             'name': 'One-way delay',
             'unit': 'ms',
             'bounds':[0,100],    # expected value range
             'tag':'delay'        # do not change the tag
            }

out_name = Path('UseCase_Pantheon/PantheonData/metrics_wo_convergence.csv')

metric_list = [metric_tput, metric_delay]
kpi_list = [ KPI_tput, KPI_delay ]
score_list = [ score_tput, score_delay ]

metrics, figure = pantheon.compute_metric(result_dir_list, 
                                  meta_data_file, 
                                  convergence,
                                  metric_list,
                                  out_name=out_name,
                                  force_computation=False,
                                  plot=False,
                                  verbose=False)
KPIs = pantheon.compute_kpi(metrics,
                            kpi_list,
                            series='datetime',
                            plot=False,
                            verbose=False)
scores = pantheon.compute_score(KPIs,
                            score_list,
                            plot=False,
                            verbose=False)
custom_layout = {
    'title':None,
    'font':{"size":font_size_px},
    'xaxis':{'tickangle':-45,
             'title':{'font':{'size':font_size_px-2}}},
    'xaxis2':{'tickangle':-45,
             'title':{'font':{'size':font_size_px-2}}}
}

# Adjust the plot size 
if output_format == 'online':
    custom_layout['width'] = linewidth_px
    custom_layout['height'] = 650
    custom_layout['margin'] = dict()
    
    # Plot two graphs in column
    in_column = True

if output_format == 'CCR':
    custom_layout['width'] = linewidth_px
    custom_layout['height'] = 350
    custom_layout['margin'] = dict(l=40, r=10, t=0, b=55)
    
    # Plot two graphs in column
    in_column = True
    

# Produce the figure
plot_out_name = "plot_score_matrix.pdf"
figure = pantheon.plot_triscale_scores_matrix(    scores,
                                                  score_list,
                                                  config_file,
                                                  layout = custom_layout,
                                                  in_column=in_column,
                                                  show=False
                                                )
# Adjust the column title font size
for i in figure['layout']['annotations']:
    i['font'] = dict(size=font_size_px)

# Save and show
figure.write_image(str(plot_path/plot_out_name))
figure.show()

Output retrieved from file. Skipping computation.

Figure 9¶

In [15]:

# === TPR ===

plot_out_name = "plot_fd_tpr.pdf"

# Construct the path to the different test results
result_dir  = Path('UseCase_FailureDetection')
config_file = Path('UseCase_FailureDetection/config.yml')
out_file = result_dir / 'kpis.csv'

# KPI definition
KPI  = {    'percentile' : 50,
            'confidence' : 95,
            'bounds': [0,1],
            'bound': 'lower'
       }

# Retrieve KPIs
metric_df = None
kpis = fd.compute_kpis(metric_df,KPI,config_file,out_name=out_file)

# Generate the figure
figure = fd.plot_TPR(kpis,config_file)

# Set layout
custom_layout = {
    "font":{"size":font_size_px},
    "xaxis":{'title':{'font':{'size':font_size_px}}},
    "yaxis":{'title':{'font':{'size':font_size_px}}},
    "legend": dict(x=0.5, 
                   y=0.1, 
                   orientation='h',
                   xanchor='center',
                   font=dict(size=font_size_px-2))
}
if output_format == 'online':
    custom_layout['width'] = 1.5*linewidth_px
    custom_layout['height'] = 300
    custom_layout['margin'] = dict(l=40, r=0, t=35, b=40)
    custom_layout["legend"] = dict()
    
if output_format == 'CCR':
    custom_layout["width"] = linewidth_px
    custom_layout["height"] = 125
    custom_layout["margin"] = dict(l=35, r=0, t=0, b=30)
    custom_layout["legend"]['yanchor'] = 'bottom'
    custom_layout["legend"]['y'] = 0.97

figure.update_layout(custom_layout)
figure.show()

# Save plot
if output_format == 'CCR':
    figure.write_image(str(plot_path/plot_out_name))

    
# === Speed ===
    
plot_out_name = "plot_fd_speed.pdf"

# Generate the figure
figure = fd.plot_speed(kpis,config_file)

# Ajust layout
if output_format == 'online':
    custom_layout['width'] = 1.5*linewidth_px
    custom_layout['height'] = 300
    custom_layout['margin'] = dict(l=40, r=0, t=35, b=40)
    custom_layout["legend"] = dict()
if output_format == 'CCR':
    custom_layout['showlegend'] = False
    custom_layout["height"]-= 20
    
figure.update_layout(custom_layout)
figure.show()

# Save plot
if output_format == 'CCR':
    figure.write_image(str(plot_path/plot_out_name))

Output retrieved from file. Skipping computation.

Figure 10¶

In [16]:

# Path to test results
result_dir_path = Path('UseCase_VideoStreaming/FCC/linear')

# Metadata
protocol_list = [x.stem for x in result_dir_path.iterdir()]
protocol_list = list(set(protocol_list)) # make list elements unique
config_file = Path('UseCase_VideoStreaming/config.yml')

# Define the KPIs
KPI_percentiles = np.arange(2,100,2) # percentiles
KPI_confidence = 95                  # confidence level
KPI_base  = {'confidence': KPI_confidence,
             'bound': 'lower',
             'bounds': [-3,3],
             'unit': '',
            }
KPI_list = []
for p in KPI_percentiles:
    kpi = copy.deepcopy(KPI_base)
    kpi['percentile'] = p
    kpi['name'] = 'P%d'%p
    KPI_list.append(kpi)
    
# Compute/load KPIs values
out_name = Path('UseCase_VideoStreaming') / 'kpis.csv'
QoE = vs.compute_kpi(
    protocol_list,
    KPI_list,
    result_dir_path,
    out_name=out_name
)

plot_out_name = "plot_vs_all.pdf"
sample = dict(
    sample_cdf=False,
)
figure = vs.plot_cdf(
    QoE,
    config_file,
    result_dir_path,
    sample=sample
)

# Set layout
custom_layout = {
    "font":{"size":font_size_px},
    "xaxis":{'title':{'font':{'size':font_size_px}},
             'rangemode':'nonnegative'},
    "yaxis":{'title':{'font':{'size':font_size_px}}},
    "legend": dict(x=0.97, 
                   y=0.1, 
                   orientation='v',
                   xanchor='right',
                   font=dict(size=font_size_px))
}


# Ajust layout
if output_format == 'online':
    custom_layout['width'] = 1.5*linewidth_px
    custom_layout['height'] = 300
    custom_layout['margin'] = dict(l=40, r=0, t=35, b=40)
    custom_layout["legend"] = dict()
    custom_layout["xaxis"]["rangemode"] = 'normal'
    
if output_format == 'CCR':
    custom_layout["width"]  = linewidth_px
    custom_layout["height"] = 225
    custom_layout["legend"]['x'] = 1
    custom_layout["legend"]['y'] = 0.1
    custom_layout["legend"]['yanchor'] = 'bottom'
    custom_layout["legend"]['orientation'] = 'v'
    custom_layout['margin'] = dict(l=40, r=0, t=0, b=30)

figure.update_layout(custom_layout)
figure.show()

# Save plot
if output_format == 'CCR':
    figure.write_image(str(plot_path/plot_out_name))

Output retrieved from file. Skipping computation.