nbviewer
to read as a set of slidesFile -> Open...
Upload
.ipynb
file you just downloadedUpload
next to the notebook name in the file listing and wait for the button to disappearMicrosoft -- specifically the Python team in the data & analytics group in Azure -- paid to get me to the conference.
And of course there are job openings at Microsoft:
Interpreters! You might call them a virtual machine (VM) or a Python implementation.
All 4 of them ...
... if people stay motivated and funded.
Because you can't always change which interpreter you use.
The results are more like "guidelines" than something you can consider rigously measured.
2.0
means it took 2x/200% as long as CPython 2.7.100.5
means the benchmark to 0.5x/50% as long1.0
is equivalent to CPython 2.7.10benchmark_names = ['2to3', 'call_method', 'call_method_slots', 'call_method_unknown', 'call_simple', 'chameleon_v2', 'chaos', 'django_v2', 'etree_generate', 'etree_iterparse', 'etree_parse', 'etree_process', 'fannkuch', 'fastpickle', 'fastunpickle', 'float', 'formatted_logging', 'go', 'hexiom2', 'json_dump_v2', 'json_load', 'mako_v2', 'meteor_contest', 'nbody', 'nqueens', 'pathlib', 'pickle_dict', 'pickle_list', 'pidigits', 'raytrace', 'regex_compile', 'regex_effbot', 'regex_v8', 'richards', 'silent_logging', 'simple_logging', 'spectral_norm', 'telco', 'tornado_http', 'unpickle_list']
pandas_data = [('Pyjion', [None, 1.09, 1.11, 1.28, 1.25, None, 1.47, None, 0.87, 2.29, 1.51, 1.02, 1.13, 0.64, 1.04, 1.01, 1.66, 1.21, 1.22, 1.32, 0.57, 0.88, 1.0, 0.96, 1.24, 1.21, 0.65, 0.56, 0.96, 1.12, 1.09, 1.0, 0.8, 0.95, 1.27, 1.67, 1.51, 0.02, None, 0.71]), ('CPython 3.5.0 PGO', [0.85, 0.87, 0.87, 0.89, 0.99, 0.87, 1.11, 0.81, 0.75, 1.9, 1.3, 0.88, 1.11, 0.69, 1.04, 0.94, 1.12, 0.9, 0.95, 1.23, 0.5, 0.85, 0.97, 0.93, 1.01, 1.08, 0.81, 0.61, 0.94, 0.96, 0.8, 1.02, 0.89, 0.76, 1.16, 1.05, 1.47, 0.01, 1.01, 0.75]), ('Jython 2.7.0', [2.57, 0.83, 0.78, 0.76, 0.63, None, 0.84, 1.11, 3.45, 16.58, 40.3, 3.19, 0.66, 1.12, 2.35, 0.96, 1.09, 0.53, 0.93, 1.55, 2.64, 1.42, 0.88, 1.24, 1.18, None, 2.7, 1.65, 0.93, 0.74, 0.96, 1.26, 1.64, 0.75, 1.48, 1.24, 0.64, 1.15, None, 1.64]), ('IronPython 2.7.5', [None, 0.47, 0.44, 0.91, 0.18, None, 0.64, None, 9.51, None, 64.26, 8.9, 0.86, 1.16, 0.94, 0.9, 1.39, 1.58, 0.83, 20.98, 8.84, 1.98, 1.05, 0.58, 1.03, 4.99, 1.34, 0.93, 1.34, 0.61, None, 1.0, 1.06, 1.08, 1.35, 1.6, 0.91, 1.27, None, 0.63]), ('PyPy 4.0.0', [0.92, 0.02, 0.02, 0.02, 0.02, 0.43, 0.02, 0.07, 0.3, 1.22, 2.75, 0.3, 0.18, 2.1, 1.4, 0.1, 0.18, 0.35, 0.1, 0.36, 0.28, 0.46, 0.44, 0.14, 0.2, 0.55, 4.85, 3.31, 2.87, 0.01, 0.18, 0.58, 2.04, 0.01, None, 0.16, 0.06, 0.03, 0.91, 0.88]), ('CPython 3.5.0', [0.99, 1.0, 0.98, 0.95, 1.09, 0.95, 1.28, 0.94, 0.84, 2.27, 1.44, 1.11, 1.22, 0.62, 1.2, 1.1, 1.23, 1.12, 1.12, 1.27, 0.52, 0.85, 0.98, 1.06, 1.16, 1.16, 0.64, 0.55, 0.95, 1.1, 0.89, 1.0, 0.77, 0.99, 1.25, 1.27, 1.64, 0.02, 1.06, 0.67])]
import pandas
df = pandas.DataFrame.from_items(pandas_data, orient='index', columns=benchmark_names).sort()
import pprint
pprint.pprint(benchmark_names)
['2to3', 'call_method', 'call_method_slots', 'call_method_unknown', 'call_simple', 'chameleon_v2', 'chaos', 'django_v2', 'etree_generate', 'etree_iterparse', 'etree_parse', 'etree_process', 'fannkuch', 'fastpickle', 'fastunpickle', 'float', 'formatted_logging', 'go', 'hexiom2', 'json_dump_v2', 'json_load', 'mako_v2', 'meteor_contest', 'nbody', 'nqueens', 'pathlib', 'pickle_dict', 'pickle_list', 'pidigits', 'raytrace', 'regex_compile', 'regex_effbot', 'regex_v8', 'richards', 'silent_logging', 'simple_logging', 'spectral_norm', 'telco', 'tornado_http', 'unpickle_list']
%matplotlib inline
import seaborn
seaborn.set_style("whitegrid") # Make baseline stand out.
seaborn.mpl.rc("figure", figsize=(16, 5), dpi=80) # Make everything big at 1280x720.
outlier_cutoff = 2.0 # When locking down scale, cap at 2x slower than CPython 2.7.10.
y_label = 'Time taken relative to CPython 2.7.10 (smaller is better)'
def add_baseline(plot):
"""Add a black line at 1.0 to represent parity of performance."""
plot.axhline(1, color="black")
def benchplot(results, ylim=None):
"""Plot all the benchmark results individually as a boxplot."""
p = results.plot(kind='bar')
add_baseline(p)
p.set_ylabel(y_label)
if ylim:
p.set_ylim(0.0, outlier_cutoff)
return p
def overviewplot(results):
"""Plot all the benchmark results in an overview style using a boxplot and stripplot.
The y axis representing performance compared to CPython 2.7.10 is capped at 2x slower
to present a consistent scale across plots.
"""
p = seaborn.boxplot(data=results, orient='v', color="seagreen", fliersize=0)
p2 = seaborn.stripplot(data=results, orient='v', color="purple", edgecolor="gray", jitter=True)
add_baseline(p)
p.set_ylim(0.0, outlier_cutoff) # For consistent scaling.
return p, p2
ax = df.plot(kind='bar', legend=None)
ax.set_ylabel(y_label)
<matplotlib.text.Text at 0x7fb96d9f01d0>
Sliced and diced ...
perf.py -b 2n3,-startup,-chameleon_v2,-pathlib,-tornado_http,-unpack_sequence
jython = df.loc['Jython 2.7.0']
Jython has two REALLY bad benchmarks.
benchplot(jython)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96d936048>
Going to crop the data to 2.0
on the y-axis make it readable.
benchplot(jython, ylim=outlier_cutoff)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96d56de10>
A visual overview of performance, with a crop of 2.0
to keep a consistent scale across all interpreters.
overviewplot(jython)
(<matplotlib.axes._subplots.AxesSubplot at 0x7fb96d417160>, <matplotlib.axes._subplots.AxesSubplot at 0x7fb96d417160>)
perf.py -a -b 2n3,-startup,-2to3,-django_v2,-etree_iterparse,-regex_compile,-tornado_http
ironpython = df.loc['IronPython 2.7.5']
benchplot(ironpython)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96d3fa278>
Cropping the REALLY bad benchmarks to make the data readable
benchplot(ironpython, ylim=outlier_cutoff)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96d258b38>
overviewplot(ironpython)
(<matplotlib.axes._subplots.AxesSubplot at 0x7fb96d10e630>, <matplotlib.axes._subplots.AxesSubplot at 0x7fb96d10e630>)
perf.py -b 2n3,-startup
pypy = df.loc['PyPy 4.0.0']
benchplot(pypy)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96d0ea5f8>
benchplot(pypy, ylim=outlier_cutoff)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96cf4dbe0>
overviewplot(pypy)
(<matplotlib.axes._subplots.AxesSubplot at 0x7fb96ce05ba8>, <matplotlib.axes._subplots.AxesSubplot at 0x7fb96ce05ba8>)
perf.py -b 2n3,-startup
cpython = df.loc['CPython 3.5.0']
benchplot(cpython)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96cde2978>
benchplot(cpython, ylim=outlier_cutoff)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96cc3ccf8>
overviewplot(cpython)
(<matplotlib.axes._subplots.AxesSubplot at 0x7fb96caf76d8>, <matplotlib.axes._subplots.AxesSubplot at 0x7fb96caf76d8>)
perf.py -b 2n3,-startup
Trained on Python's test suite.
cpython_pgo = df.loc['CPython 3.5.0 PGO']
benchplot(cpython_pgo)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96cf1b8d0>
overviewplot(cpython_pgo)
(<matplotlib.axes._subplots.AxesSubplot at 0x7fb96c994048>, <matplotlib.axes._subplots.AxesSubplot at 0x7fb96c994048>)
# Which benchmarks are **faster** without PGO?
cpython_faster = cpython < cpython_pgo
cpython_faster[cpython_faster.isin([True])]
fastpickle True pickle_dict True pickle_list True regex_effbot True regex_v8 True unpickle_list True dtype: bool
(cpython_pgo - cpython)[cpython_faster]
fastpickle 0.07 pickle_dict 0.17 pickle_list 0.06 regex_effbot 0.02 regex_v8 0.12 unpickle_list 0.08 dtype: float64
# All benchmarks +/- 10% speed difference.
threshold = 0.10
cpython_diff = cpython_pgo - cpython
cpython_diff_faster = cpython_diff[cpython_diff < -threshold]
cpython_diff_slower = cpython_diff[cpython_diff > threshold]
pandas.concat([cpython_diff_faster, cpython_diff_slower])
2to3 -0.14 call_method -0.13 call_method_slots -0.11 call_simple -0.10 chaos -0.17 django_v2 -0.13 etree_iterparse -0.37 etree_parse -0.14 etree_process -0.23 fannkuch -0.11 fastunpickle -0.16 float -0.16 formatted_logging -0.11 go -0.22 hexiom2 -0.17 nbody -0.13 nqueens -0.15 raytrace -0.14 richards -0.23 simple_logging -0.22 spectral_norm -0.17 pickle_dict 0.17 regex_v8 0.12 dtype: float64
perf.py -b 2n3,-startup,-2to3,-chameleon_v2,-tornado_http
pyjion = df.loc['Pyjion']
benchplot(pyjion)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96c935438>
benchplot(pyjion, outlier_cutoff)
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96c7c9e48>
overviewplot(pyjion)
(<matplotlib.axes._subplots.AxesSubplot at 0x7fb96c68a400>, <matplotlib.axes._subplots.AxesSubplot at 0x7fb96c68a400>)
Remember, scale is capped at 2x slower than CPython 2.7.10.
overviewplot(df.transpose())
(<matplotlib.axes._subplots.AxesSubplot at 0x7fb96c65aa20>, <matplotlib.axes._subplots.AxesSubplot at 0x7fb96c65aa20>)
Gives a simplistic number to classify overall performance.
def geometric_mean(series):
return series.prod(skipna=True) ** (1/series.count())
def geometric_mean_series(dataframe):
interpreters = dataframe.index
gmean = []
for interpreter in interpreters:
gmean.append(geometric_mean(dataframe.loc[interpreter]))
return pandas.Series(data=gmean, index=interpreters)
benchplot(geometric_mean_series(df))
<matplotlib.axes._subplots.AxesSubplot at 0x7fb96c49a588>