In [1]:
%load_ext watermark
%watermark -a 'cs224' -u -d -v -p numpy,pandas,matplotlib,sklearn,h5py,zipline
cs224 
last updated: 2020-12-29 

CPython 3.6.12
IPython 7.16.1

numpy 1.19.1
pandas 0.22.0
matplotlib 3.3.1
sklearn 0.23.2
h5py 2.10.0
zipline 1.4.1
In [2]:
%matplotlib inline
import numpy as np, scipy, scipy.stats as stats, pandas as pd, matplotlib.pyplot as plt, seaborn as sns
import sklearn, sklearn.pipeline, sklearn.model_selection, sklearn.preprocessing, sklearn.linear_model
import re, fnmatch

pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
# pd.set_option('display.float_format', lambda x: '%.2f' % x)
np.set_printoptions(edgeitems=10)
np.set_printoptions(linewidth=1000)
np.set_printoptions(suppress=True)
np.core.arrayprint._line_width = 180

SEED = 42
np.random.seed(SEED)

sns.set()
In [3]:
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:60% !important; }</style>"))
In [4]:
import os,sys
path = os.path.realpath(os.path.abspath('') + '/csi_futures_data')
sys.path.append(path)
In [5]:
%load_ext autoreload
%autoreload 1
%aimport csi_futures_data
In [6]:
from IPython.display import display, HTML

from IPython.display import display_html
def display_side_by_side(*args):
    html_str=''
    for df in args:
        if type(df) == np.ndarray:
            df = pd.DataFrame(df)
        html_str+=df.to_html()
    html_str = html_str.replace('table','table style="display:inline"')
    # print(html_str)
    display_html(html_str,raw=True)

CSS = """
.output {
    flex-direction: row;
}
"""

def display_graphs_side_by_side(*args):
    html_str='<table><tr>'
    for g in args:
        html_str += '<td>'
        html_str += g._repr_svg_()
        html_str += '</td>'
    html_str += '</tr></table>'
    display_html(html_str,raw=True)
    

display(HTML("<style>.container { width:70% !important; }</style>"))
In [7]:
# the following list is a hand curated list of futures as a combination of data from:
# - Following the Trend
# - Trading Evolved
# - zipline.finance.constants.ROOT_SYMBOL_TO_ETA
# - zipline.finance.constants.FUTURE_EXCHANGE_FEES_BY_SYMBOL
csi_futures_data_df = csi_futures_data.futures_lookup_.copy()
csi_futures_data_df
Out[7]:
csi_symbol root_symbol multiplier minor_fx_adj description exchange sector
0 KC2 KC 37500.0 0.01 Coffee CSCE Agricultural
1 C2 _C 5000.0 0.01 Corn CBT Agricultural
2 C2 CN 5000.0 0.01 Corn CBT Agricultural
3 CT2 CT 50000.0 0.01 Cotton #2 NYCE Agricultural
4 LH LH 40000.0 0.01 Lean Hogs CME Agricultural
5 LC LC 40000.0 0.01 Live Cattle CME Agricultural
6 LB LB 110.0 1.00 Lumber CME Agricultural
7 O2 _O 5000.0 0.01 Oats-CBT CBT Agricultural
8 O2 OA 5000.0 0.01 Oats-CBT CBT Agricultural
9 OJ2 OJ 15000.0 0.01 Orange Juice NYCE Agricultural
10 RR2 RR 2000.0 1.00 Rice(Rough) CBT Agricultural
11 S2 _S 5000.0 0.01 Soybean CBT Agricultural
12 S2 SY 5000.0 0.01 Soybean CBT Agricultural
13 LSU LS 50.0 1.00 Sugar #5(White) EURONEXT Agricultural
14 W2 _W 5000.0 0.01 Wheat CBT Agricultural
15 W2 WC 5000.0 0.01 Wheat CBT Agricultural
16 KW2 KW 5000.0 0.01 Wheat-Kansas City KCBT Agricultural
17 MW2 MW 5000.0 0.01 Wheat-Spring MGE Agricultural
18 DA DA 200000.0 0.01 Class III Milk CME Agricultural
19 FC FC 50000.0 0.01 Cattle-Feeder CME Agricultural
20 LRC LR 10.0 1.00 Robusta Coffee New (LCE) EURONEXT Agricultural
21 BO2 BO 60000.0 0.01 Soybean Oil CBT Agricultural
22 SM2 SM 100.0 1.00 Soybean Meal CBT Agricultural
23 CC2 CC 10.0 1.00 Cocoa NYCE Agricultural
24 SB2 SB 112000.0 0.01 Sugar #11 NYCE Agricultural
25 XC2 CM 1000.0 0.01 Corn E-Mini CBOT Agricultural
26 XS2 MS 1000.0 0.01 Soybeans E-Mini CBOT Agricultural
27 CL2 CL 1000.0 1.00 Crude Oil-Light NYMEX Non-Agricultural
28 LCO LO 1000.0 1.00 Brent Crude ICE Non-Agricultural
29 HO2 HO 42000.0 1.00 Heating Oil NYMEX Non-Agricultural
... ... ... ... ... ... ... ...
53 NE NZ 100000.0 1.00 NZD/USD CME Currency
54 SF SF 125000.0 1.00 CHF/USD CME Currency
55 DX2 DX 1000.0 1.00 USD Index FINEX Currency
56 MP MP 500000.0 1.00 MEP/USD CME Currency
57 MP ME 500000.0 1.00 MEP/USD CME Currency
58 EX EE 62500.0 1.00 Euro FX E-mini CME Currency
59 M6E EU 12500.0 1.00 E-micro EUR/USD Futures CME Currency
60 JT JE 62500.0 1.00 Japanese Yen E-mini CME Currency
61 ES ES 50.0 1.00 S&P 500 (E-mini) CME Equities
62 NQ NQ 20.0 1.00 Nasdaq (E-mini) CME Equities
63 ER2 TF 50.0 1.00 Russel 2000 (E-mini) CME Equities
64 ER2 ER 50.0 1.00 Russel 2000 (E-mini) CME Equities
65 VX VX 1000.0 1.00 Volatility Index CFE Equities
66 YM YM 5.0 1.00 Dow e mini CBT Equities
67 NK NK 5.0 1.00 Nikkei 225 CME Equities
68 STW TW 100.0 1.00 MSCI Taiwan SGX Equities
69 MEM EI 50.0 1.00 MSCI Emerging Markets Mini ICE Equities
70 MFS MG 50.0 1.00 MSCI EAFE Mini (Europe, Australasia and Far East) ICE Equities
71 EMD MI 100.0 1.00 S&P 400 MidCap E-Mini CME Equities
72 RS1 RM 50.0 1.00 Russell 1000 Mini CME Equities
73 SP2 SP 250.0 1.00 S&P 500 Futures CME Equities
74 FV FV 1000.0 1.00 US Treasury Note 5yr CBT Rates
75 ED ED 2500.0 1.00 Eurodollar 3m CME Rates
76 US US 1000.0 1.00 US Treasury Long Bond 30yr CBT Rates
77 TU TU 2000.0 1.00 US Treasury Note 2yr CBT Rates
78 TY TY 1000.0 1.00 US Treasury Note 10y CBT Rates
79 FF FF 4167.0 1.00 30-Day Federal Funds CBOT Rates
80 F1U FI 1000.0 1.00 5-Year Deliverable Interest Rate Swap Futures CBOT Rates
81 N1U TN 1000.0 1.00 10-Year Deliverable Interest Rate Swap Futures CBOT Rates
82 UL2 UB 1000.0 1.00 Ultra Tbond CBOT Rates

83 rows × 7 columns

In [8]:
# the meta.csv is from the package that Andreas Clenow provides for his random_futures_data: https://www.dropbox.com/s/etocgt9zgeedo22/data.zip?dl=0
futures_meta_df = pd.read_csv('meta.csv', index_col=0)
futures_meta_df
Out[8]:
root_symbol multiplier minor_fx_adj description exchange sector
0 AD 100000 1.00 AUD/USD CME Currency
1 BO 600 0.01 Soybean Oil CBT Agricultural
2 BP 62500 1.00 GBP/USD CME Currency
3 _C 5000 0.01 Corn CBT Agricultural
4 CC 10 1.00 Cocoa NYCE Agricultural
5 CD 100000 1.00 CAD/USD CME Currency
6 CL 1000 1.00 Crude Oil NYMEX Non-Agricultural
7 CT 50000 0.01 Cotton #2 NYCE Agricultural
8 CU 125000 1.00 EUR/USD CME Currency
9 DA 200000 1.00 Class III Milk CME Agricultural
10 DX 1000 1.00 USD Index FINEX Currency
11 ED 2500 1.00 Eurodollar 3m CME Rates
13 ES 50 1.00 S&P 500 (E-mini) CME Equities
14 FC 50000 0.01 Cattle-Feeder CME Agricultural
15 FV 1000 1.00 US Treasury Note 5yr CBT Rates
16 GC 100 1.00 Gold COMEX Non-Agricultural
17 HG 25000 0.01 Copper COMEX Non-Agricultural
18 HO 42000 1.00 Heating Oil NYMEX Non-Agricultural
20 JY 125000 1.00 JPY/USD CME Currency
21 KC 37500 0.01 Coffee Arabica NYCE Agricultural
22 KW 5000 0.01 Wheat-Kansas City KCBT Agricultural
23 LB 110 1.00 Lumber CME Agricultural
24 LC 40000 0.01 Live Cattle CME Agricultural
25 LO 1000 1.00 Brent Crude ICE Non-Agricultural
26 LG 100 1.00 Petroleum Gas Oil ICE Non-Agricultural
27 LH 40000 0.01 Lean Hogs CME Agricultural
29 LR 10 1.00 Coffee Robusta EURONEXT Agricultural
30 LS 50 1.00 Sugar #5(White) EURONEXT Agricultural
31 MP 500000 0.01 MEP/USD CME Currency
32 MW 5000 0.01 Wheat-Spring MGE Agricultural
33 NE 100000 1.00 NZD/USD CME Currency
34 NG 10000 1.00 Natural Gas NYMEX Non-Agricultural
35 NK 5 1.00 Nikkei 225 CME Equities
36 NQ 20 1.00 Nasdaq (E-mini) CME Equities
37 _O 5000 0.01 Oats-CBT CBT Agricultural
38 OJ 15000 0.01 Orange Juice NYCE Agricultural
39 PA 100 1.00 Palladium NYMEX Non-Agricultural
40 PL 50 1.00 Platinum NYMEX Non-Agricultural
41 RB 42000 1.00 Petroleum Gasoline NYMEX Non-Agricultural
42 RR 2000 1.00 Rice(Rough) CBT Agricultural
43 _S 5000 0.01 Soybean CBT Agricultural
44 SB 120000 0.01 Sugar #11 NYCE Agricultural
45 SF 125000 1.00 CHF/USD CME Currency
46 SI 5000 0.01 Silver COMEX Non-Agricultural
48 SM 100 1.00 Soybean Meal CBT Agricultural
50 TW 100 1.00 MSCI Taiwan SGX Equities
51 TF 100 1.00 Russel 2000 (E-mini) NYFE Equities
53 TU 2000 1.00 US Treasury Note 2yr CBT Rates
54 TY 1000 1.00 US Treasury Note 10y CBT Rates
55 US 1000 1.00 US Treasury Long Bond 30yr CBT Rates
56 VX 1000 1.00 Volatility Index CFE Equities
57 _W 5000 0.01 Wheat CBT Agricultural
59 YM 5 1.00 Dow e mini CBT Equities
In [9]:
# outdated: http://www.csidata.com/factsheets/factsheet-futures.html
# csidata_futures_fact_sheet_df = pd.read_excel('~/2020-10-13-csidata-futures-fact-sheet.xlsx')
# csidata_futures_fact_sheet_df.drop(csidata_futures_fact_sheet_df.columns[1], axis=1, inplace=True)
# csidata_futures_fact_sheet_df.head()
In [10]:
# New, up to date: http://www.csidata.com/factsheets.php?type=commodity&format=htmltable&exchangeid=
csidata_futures_fact_sheet_df = pd.read_csv('./2020-10-29-commodityfactsheet.csv', parse_dates=['StartDate', 'EndDate'])
csidata_futures_fact_sheet_df = csidata_futures_fact_sheet_df[['UACsiNumber', 'SymbolUA', 'ExchangeSymbol', 'Exchange', 'Name', 'SessionType', 'IsActive', 'TerminalPointValue', 'FullPointValue', 'Currency', 'ContractSize', 'Units', 'MinimumTick', 'TickValue', 'StartDate', 'EndDate',\
                                                               'LinkSymbol', 'ConversionFactorCode', 'HasCurrentDayVolume', 'HasCurrentDayOpenInterest', 'HasKnownExpirationDates', 'LastTotalVolume']]
csidata_futures_fact_sheet_df.head()
Out[10]:
UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume
0 1 IB NaN CBT Broilers-Iced NaN 0 3.0 300.0 USD 30000 lbs cents/lb 1.0 3.0 1969-08-19 1981-01-22 NaN 2 0 0 0 NaN
1 2 LC LC CME Live Cattle Combined 1 4.0 400.0 USD 40000 lbs cents/lb 2.5 10.0 1964-11-30 2020-10-28 NaN 2 0 0 1 57781.0
2 3 CC CC ICEUS Cocoa RTH 1 10.0 10.0 USD 10 tonnes USD/tonne 1.0 10.0 1965-12-30 2020-10-28 CC2 starts at 1965-12-30 0 0 0 1 31624.0
3 4 LH LH CME Lean Hogs Combined 1 4.0 400.0 USD 40000 lbs cents/lb 2.5 10.0 1966-02-28 2020-10-28 NaN 2 0 0 1 30958.0
4 5 PB PB CME Pork Bellies RTH 0 4.0 400.0 USD 40000 lbs cents/lb 2.5 10.0 1961-09-18 2011-07-15 NaN 2 0 0 0 NaN
In [11]:
s1 = set(csi_futures_data_df.root_symbol)
s2 = set(futures_meta_df.root_symbol)
In [12]:
# verify that all symbols that are present in the meta.csv are present in csi_futures_data_df
s2 - s1
Out[12]:
set()

Verify the multipliers and minor_fx_adj between meta.csv and what the CSI data factsheet says

My understanding is that the values in csi_futures_data_df are correct and the values in meta.csv are incorrect.

In [13]:
ldf = pd.merge(csi_futures_data_df, futures_meta_df, how='left', on=['root_symbol'])
ldf.head()
Out[13]:
csi_symbol root_symbol multiplier_x minor_fx_adj_x description_x exchange_x sector_x multiplier_y minor_fx_adj_y description_y exchange_y sector_y
0 KC2 KC 37500.0 0.01 Coffee CSCE Agricultural 37500.0 0.01 Coffee Arabica NYCE Agricultural
1 C2 _C 5000.0 0.01 Corn CBT Agricultural 5000.0 0.01 Corn CBT Agricultural
2 C2 CN 5000.0 0.01 Corn CBT Agricultural NaN NaN NaN NaN NaN
3 CT2 CT 50000.0 0.01 Cotton #2 NYCE Agricultural 50000.0 0.01 Cotton #2 NYCE Agricultural
4 LH LH 40000.0 0.01 Lean Hogs CME Agricultural 40000.0 0.01 Lean Hogs CME Agricultural
In [14]:
ldf[(ldf['multiplier_x'] != ldf['multiplier_y']) & ldf['root_symbol'].isin(futures_meta_df['root_symbol'])]
Out[14]:
csi_symbol root_symbol multiplier_x minor_fx_adj_x description_x exchange_x sector_x multiplier_y minor_fx_adj_y description_y exchange_y sector_y
21 BO2 BO 60000.0 0.01 Soybean Oil CBT Agricultural 600.0 0.01 Soybean Oil CBT Agricultural
24 SB2 SB 112000.0 0.01 Sugar #11 NYCE Agricultural 120000.0 0.01 Sugar #11 NYCE Agricultural
63 ER2 TF 50.0 1.00 Russel 2000 (E-mini) CME Equities 100.0 1.00 Russel 2000 (E-mini) NYFE Equities
In [15]:
ldf[(ldf['minor_fx_adj_x'] != ldf['minor_fx_adj_y']) & ldf['root_symbol'].isin(futures_meta_df['root_symbol'])]
Out[15]:
csi_symbol root_symbol multiplier_x minor_fx_adj_x description_x exchange_x sector_x multiplier_y minor_fx_adj_y description_y exchange_y sector_y
18 DA DA 200000.0 0.01 Class III Milk CME Agricultural 200000.0 1.00 Class III Milk CME Agricultural
56 MP MP 500000.0 1.00 MEP/USD CME Currency 500000.0 0.01 MEP/USD CME Currency
In [16]:
ldf[(ldf['exchange_x'] != ldf['exchange_y']) & ldf['root_symbol'].isin(futures_meta_df['root_symbol'])]
Out[16]:
csi_symbol root_symbol multiplier_x minor_fx_adj_x description_x exchange_x sector_x multiplier_y minor_fx_adj_y description_y exchange_y sector_y
0 KC2 KC 37500.0 0.01 Coffee CSCE Agricultural 37500.0 0.01 Coffee Arabica NYCE Agricultural
63 ER2 TF 50.0 1.00 Russel 2000 (E-mini) CME Equities 100.0 1.00 Russel 2000 (E-mini) NYFE Equities

Verify the FullPointValue as given by CSI data's factsheet with the value in csi_futures_data_df

In [17]:
ldf_ = pd.merge(csi_futures_data_df, csidata_futures_fact_sheet_df, how='left', left_on=['csi_symbol'], right_on=['SymbolUA'])
#ldf_ = ldf_[['csi_symbol', 'root_symbol', 'multiplier', 'minor_fx_adj', 'description', 'exchange', 'sector', 'EXCHANGE', 'EXCHANGE SYMBOL', 'NAME', 'ACTIVE', 'CONTRACT SIZE', 'UNIT OF MEASURE', 'CURRENCY', 'FULL POINT VALUE']]
#ldf_['numeric_full_point_value'] = ldf_['FULL POINT VALUE'].str.extract(r'^(\d+).*$').astype(np.int)
ldf_['calc_poin_value'] = (ldf_['multiplier'] * ldf_['minor_fx_adj'])#.astype(np.int)
ldf_.head()
Out[17]:
csi_symbol root_symbol multiplier minor_fx_adj description exchange sector UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume calc_poin_value
0 KC2 KC 37500.0 0.01 Coffee CSCE Agricultural 1146 KC2 KC ICEUS Coffee Combined 1 3.75 375.0 USD 37500 lbs cents/pound 5.0 18.75 1972-08-16 2020-10-28 KC starts at 1972-08-16 2 0 0 1 31867.0 375.0
1 C2 _C 5000.0 0.01 Corn CBT Agricultural 412 C2 C CBT Corn Combined 1 6.25 50.0 USD 5000 bu cents/bu 2.0 12.50 1949-01-03 2020-10-28 C starts at 1946-01-02 -1 0 0 1 426691.0 50.0
2 C2 CN 5000.0 0.01 Corn CBT Agricultural 412 C2 C CBT Corn Combined 1 6.25 50.0 USD 5000 bu cents/bu 2.0 12.50 1949-01-03 2020-10-28 C starts at 1946-01-02 -1 0 0 1 426691.0 50.0
3 CT2 CT 50000.0 0.01 Cotton #2 NYCE Agricultural 1144 CT2 CT ICEUS Cotton #2 Combined 1 5.00 500.0 USD 50000 lbs cents/pound 1.0 5.00 1967-03-22 2020-10-28 CT starts at 1967-03-22 2 0 0 1 35749.0 500.0
4 LH LH 40000.0 0.01 Lean Hogs CME Agricultural 4 LH LH CME Lean Hogs Combined 1 4.00 400.0 USD 40000 lbs cents/lb 2.5 10.00 1966-02-28 2020-10-28 NaN 2 0 0 1 30958.0 400.0
In [18]:
# This here is the most important as this makes sure that we agree on the point value
# The expected result is that the dataframe has 0 entries
ldf_[ldf_['FullPointValue'] != ldf_['calc_poin_value']]
Out[18]:
csi_symbol root_symbol multiplier minor_fx_adj description exchange sector UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume calc_poin_value
In [19]:
# In general I tried to use the symbols that represent the combined floor and electronic session
ldf_[~ldf_['SessionType'].str.contains(r'Combined').astype(np.bool)]
Out[19]:
csi_symbol root_symbol multiplier minor_fx_adj description exchange sector UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume calc_poin_value
In [20]:
list(ldf_[~ldf_['SessionType'].str.contains(r'Combined').astype(np.bool)].loc[:,'csi_symbol'].values)
Out[20]:
[]
In [21]:
# The name of the exchange is irrelevant, but just for the sake of showing the differences
ldf_[ldf_['exchange'] != ldf_['Exchange']].loc[:,['csi_symbol', 'root_symbol', 'multiplier', 'minor_fx_adj', 'description', 'sector', 'exchange', 'Exchange']]
Out[21]:
csi_symbol root_symbol multiplier minor_fx_adj description sector exchange Exchange
0 KC2 KC 37500.00 0.01 Coffee Agricultural CSCE ICEUS
3 CT2 CT 50000.00 0.01 Cotton #2 Agricultural NYCE ICEUS
9 OJ2 OJ 15000.00 0.01 Orange Juice Agricultural NYCE ICEUS
13 LSU LS 50.00 1.00 Sugar #5(White) Agricultural EURONEXT LCE
20 LRC LR 10.00 1.00 Robusta Coffee New (LCE) Agricultural EURONEXT LCE
23 CC2 CC 10.00 1.00 Cocoa Agricultural NYCE ICEUS
24 SB2 SB 112000.00 0.01 Sugar #11 Agricultural NYCE ICEUS
25 XC2 CM 1000.00 0.01 Corn E-Mini Agricultural CBOT CBT
26 XS2 MS 1000.00 0.01 Soybeans E-Mini Agricultural CBOT CBT
28 LCO LO 1000.00 1.00 Brent Crude Non-Agricultural ICE ICEEUROPE
38 LGO LG 100.00 1.00 Petroleum Gas Oil Non-Agricultural ICE ICEEUROPE
39 ER AI 100.00 1.00 Bloomberg Commodity Index Futures Non-Agricultural CME CBT
43 IRB XB 42000.00 1.00 RBOB Gasoline Futures Non-Agricultural NYMEX ICEEUROPE
44 YG XG 32.15 1.00 Gold mini-sized Non-Agricultural ICE ICEUS
45 YI YS 1000.00 0.01 Silver mini-sized Non-Agricultural ICE ICEUS
55 DX2 DX 1000.00 1.00 USD Index Currency FINEX ICEUS
69 MEM EI 50.00 1.00 MSCI Emerging Markets Mini Equities ICE ICEUS
70 MFS MG 50.00 1.00 MSCI EAFE Mini (Europe, Australasia and Far East) Equities ICE ICEUS
79 FF FF 4167.00 1.00 30-Day Federal Funds Rates CBOT CBT
80 F1U FI 1000.00 1.00 5-Year Deliverable Interest Rate Swap Futures Rates CBOT CBT
81 N1U TN 1000.00 1.00 10-Year Deliverable Interest Rate Swap Futures Rates CBOT CBT
82 UL2 UB 1000.00 1.00 Ultra Tbond Rates CBOT CBT
In [22]:
# All futures trade in USD
ldf_[ldf_['Currency'] != 'USD']
Out[22]:
csi_symbol root_symbol multiplier minor_fx_adj description exchange sector UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume calc_poin_value
In [23]:
# XXX This here is also very important as this makes sure that we agree on the minor_fx_adj
#     Only for "Class III Milk" the pattern is a bit different, but this USD/CWT should also mean that minor_fx_adj is 0.01, e.g. this should be correct.
ldf_['factsheet_minor_fx_adj'] = -1.0
selector = ldf_['Units'].str.startswith('cents').astype(np.bool)
ldf_.loc[selector, 'factsheet_minor_fx_adj'] = 0.01
ldf_.loc[~selector, 'factsheet_minor_fx_adj'] = 1.0
ldf_[ldf_['factsheet_minor_fx_adj'] != ldf_['minor_fx_adj']]
Out[23]:
csi_symbol root_symbol multiplier minor_fx_adj description exchange sector UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume calc_poin_value factsheet_minor_fx_adj
18 DA DA 200000.0 0.01 Class III Milk CME Agricultural 404 DA DA CME Class III Milk Combined 1 20.0 2000.0 USD 200000 lbs. USD/CWT 1.0 20.0 1996-01-11 2020-10-28 NaN 2 0 0 1 2095.0 2000.0 1.0

Verify that all contracts are still traded

In [24]:
# make sure that all contracts are still trading
ldf_[ldf_['LastTotalVolume'] < 1.0]
Out[24]:
csi_symbol root_symbol multiplier minor_fx_adj description exchange sector UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume calc_poin_value factsheet_minor_fx_adj
In [25]:
# make sure that all contracts are still trading
ldf_[ldf_['EndDate'] != pd.Timestamp('2020-10-28')]
Out[25]:
csi_symbol root_symbol multiplier minor_fx_adj description exchange sector UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume calc_poin_value factsheet_minor_fx_adj
In [26]:
# not all contracts existed at the beginning of the time winow
ldf_[ldf_['StartDate'] > pd.Timestamp('2000-12-31')]
Out[26]:
csi_symbol root_symbol multiplier minor_fx_adj description exchange sector UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume calc_poin_value factsheet_minor_fx_adj
20 LRC LR 10.0 1.00 Robusta Coffee New (LCE) EURONEXT Agricultural 816 LRC RC LCE Robusta Coffee NaN 1 10.000 10.0 USD 10 tonne USD/tonne 1.0 10.000 2008-01-11 2020-10-28 LKD starts at 1991-03-01 0 1 0 1 14463.0 10.0 1.00
25 XC2 CM 1000.0 0.01 Corn E-Mini CBOT Agricultural 1192 XC2 YC CBT mini Corn Combined 1 1.250 10.0 USD 1000 bushels cents/bushel 1.0 1.250 2007-05-15 2020-10-28 NaN -1 0 0 1 2589.0 10.0 0.01
26 XS2 MS 1000.0 0.01 Soybeans E-Mini CBOT Agricultural 1194 XS2 YK CBT mini Soybeans Combined 1 1.250 10.0 USD 1000 bushels cents/bushel 1.0 1.250 2007-05-15 2020-10-28 NaN -1 0 0 1 3510.0 10.0 0.01
31 RB2 RB 42000.0 1.00 Gasoline-Reformulated Blendstock NYMEX Non-Agricultural 976 RB2 RB NYMEX Gasoline-Reformulated Blendstock Combined 1 4.200 42000.0 USD 42000 gallons USD/gallon 1.0 4.200 2005-10-03 2020-10-28 HU2 starts at 1984-12-03 4 0 0 1 126660.0 42000.0 1.00
39 ER AI 100.0 1.00 Bloomberg Commodity Index Futures CME Non-Agricultural 1081 ER AW CBT Bloomberg Commodity Index (DJ UBS Commodity In... NaN 1 10.000 100.0 USD USD 100 x Index points 1.0 10.000 2006-10-13 2020-10-28 NaN 1 0 0 1 755.0 100.0 1.00
40 AC ET 29000.0 1.00 Ethanol CBT Non-Agricultural 962 AC ACF CBT Ethanol NaN 1 29.000 29000.0 USD 29000 gallons USD/gallon 1.0 29.000 2005-03-23 2020-10-28 NaN 3 0 0 1 14.0 29000.0 1.00
41 QG QG 2500.0 1.00 Natural Gas E-mini NYMEX Non-Agricultural 724 QG QG NYMEX e-mini Henry Hub Natural Gas NaN 1 2.500 2500.0 USD 2500 mmBtu $/mmBtu 5.0 12.500 2002-06-17 2020-10-28 NaN 3 0 0 1 8516.0 2500.0 1.00
42 QM QM 500.0 1.00 Crude Oil E-Mini NYMEX Non-Agricultural 725 QM QM NYMEX e-mini Crude Oil NaN 1 0.500 500.0 USD 500 barrels $/barrels 25.0 12.500 2002-06-17 2020-10-28 NaN 3 0 0 1 14565.0 500.0 1.00
43 IRB XB 42000.0 1.00 RBOB Gasoline Futures NYMEX Non-Agricultural 772 IRB N ICEEUROPE RBOB Gasoline NaN 1 4.200 42000.0 USD 42000 gallons USD/gallons 1.0 4.200 2006-04-24 2020-10-28 NaN 4 0 0 1 64543.0 42000.0 1.00
59 M6E EU 12500.0 1.00 E-micro EUR/USD Futures CME Currency 1415 M6E M6E CME e-micro EUR/USD NaN 1 0.125 12500.0 USD EUR 12500 USD / EUR 10.0 1.250 2009-03-23 2020-10-28 NaN 5 0 0 1 15432.0 12500.0 1.00
63 ER2 TF 50.0 1.00 Russel 2000 (E-mini) CME Equities 677 ER2 RTY CME e-mini Russell 2000 Index Combined 1 0.500 50.0 USD $50 x Index points 10.0 5.000 2001-10-23 2020-10-28 NaN 2 0 0 1 143923.0 50.0 1.00
64 ER2 ER 50.0 1.00 Russel 2000 (E-mini) CME Equities 677 ER2 RTY CME e-mini Russell 2000 Index Combined 1 0.500 50.0 USD $50 x Index points 10.0 5.000 2001-10-23 2020-10-28 NaN 2 0 0 1 143923.0 50.0 1.00
65 VX VX 1000.0 1.00 Volatility Index CFE Equities 919 VX VX CFE CBOE Volatility Index (VIX) Combined 1 1.000 1000.0 USD $1000 x Index points 10.0 10.000 2004-03-26 2020-10-28 NaN 3 0 0 1 155378.0 1000.0 1.00
66 YM YM 5.0 1.00 Dow e mini CBT Equities 699 YM YM CBT e-mini Dow ($5) Combined 1 5.000 5.0 USD $5 x Index points 1.0 5.000 2002-04-05 2020-10-28 DJ starts at 1997-10-06 0 0 0 1 177242.0 5.0 1.00
69 MEM EI 50.0 1.00 MSCI Emerging Markets Mini ICE Equities 1552 MEM MME ICEUS mini MSCI Emerging Markets Index NaN 1 0.050 50.0 USD USD 50 x Index points 100.0 5.000 2010-09-24 2020-10-28 EMI starts at 2008-05-27 3 0 0 1 70951.0 50.0 1.00
70 MFS MG 50.0 1.00 MSCI EAFE Mini (Europe, Australasia and Far East) ICE Equities 1570 MFS MFS ICEUS mini MSCI EAFE Index NaN 1 0.050 50.0 USD USD 50 x Index points 100.0 5.000 2010-09-24 2020-10-28 NaN 3 0 0 1 25781.0 50.0 1.00
71 EMD MI 100.0 1.00 S&P 400 MidCap E-Mini CME Equities 684 EMD EMD CME e-mini S&P Midcap 400 Index NaN 1 1.000 100.0 USD $100 x index points 10.0 10.000 2002-01-28 2020-10-28 MD starts at 1992-02-13 2 0 0 1 9728.0 100.0 1.00
72 RS1 RM 50.0 1.00 Russell 1000 Mini CME Equities 1032 RS1 RS1 CME e-mini Russell 1000 Index NaN 1 0.500 50.0 USD $50 x index points 10.0 5.000 2006-06-15 2020-10-28 NaN 2 0 0 1 76.0 50.0 1.00
80 F1U FI 1000.0 1.00 5-Year Deliverable Interest Rate Swap Futures CBOT Rates 1839 F1U F1U CBT 5yr Deliverable Interest Rate Swap Combined 1 3.125 1000.0 USD $100000 points 5.0 15.625 2012-12-03 2020-10-28 NaN -8 0 0 1 264.0 1000.0 1.00
81 N1U TN 1000.0 1.00 10-Year Deliverable Interest Rate Swap Futures CBOT Rates 1836 N1U N1U CBT 10yr Deliverable Interest Rate Swap Combined 1 3.125 1000.0 USD $100000 points 5.0 15.625 2012-12-03 2020-10-28 NaN -7 0 0 1 1046.0 1000.0 1.00
82 UL2 UB 1000.0 1.00 Ultra Tbond CBOT Rates 1507 UL2 UL CBT Ultra T-Bonds Combined 1 31.250 1000.0 USD USD 100000 points 1.0 31.250 2010-01-11 2020-10-28 NaN -3 0 0 1 125777.0 1000.0 1.00
In [27]:
# Some indices are the continuation of others
csidata_futures_fact_sheet_df[csidata_futures_fact_sheet_df['SymbolUA'].isin(['LRC', 'LKD', 'LKR'])]
Out[27]:
UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume
97 50 LKR NaN LCE Coffee-Robusta NaN 0 0.5 5.0 GBP 5 tonnes GBP/tonne 1.0 0.5 1968-01-02 1992-03-31 NaN 1 0 0 1 NaN
217 148 LKD NaN LCE Coffee-Robusta NaN 0 5.0 5.0 USD 5 tonnes USD/tonne 1.0 5.0 1991-03-01 2009-01-30 LKR starts at 1968-01-02 0 1 0 1 NaN
901 816 LRC RC LCE Robusta Coffee NaN 1 10.0 10.0 USD 10 tonne USD/tonne 1.0 10.0 2008-01-11 2020-10-28 LKD starts at 1991-03-01 0 1 0 1 14463.0
In [28]:
# Some indices are the continuation of others
csidata_futures_fact_sheet_df[csidata_futures_fact_sheet_df['SymbolUA'].isin(['RB2', 'HU2', 'HU'])]
Out[28]:
UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume
282 224 HU RBOB NYMEX Gasoline Unleaded RTH 0 4.2 42000.0 USD 42000 gallons USD/gal 1.0 4.2 1984-12-03 2006-12-29 NaN 4 0 0 1 NaN
926 858 HU2 YQ NYMEX Unleaded Gasoline Combined 0 4.2 42000.0 USD 42000 gallons USD/gallon 1.0 4.2 1984-12-03 2006-12-29 HU starts at 1984-12-03 4 0 0 0 NaN
1043 976 RB2 RB NYMEX Gasoline-Reformulated Blendstock Combined 1 4.2 42000.0 USD 42000 gallons USD/gallon 1.0 4.2 2005-10-03 2020-10-28 HU2 starts at 1984-12-03 4 0 0 1 126660.0
In [29]:
# Some indices are the continuation of others
csidata_futures_fact_sheet_df[csidata_futures_fact_sheet_df['SymbolUA'].isin(['YM', 'DJ'])]
Out[29]:
UACsiNumber SymbolUA ExchangeSymbol Exchange Name SessionType IsActive TerminalPointValue FullPointValue Currency ContractSize Units MinimumTick TickValue StartDate EndDate LinkSymbol ConversionFactorCode HasCurrentDayVolume HasCurrentDayOpenInterest HasKnownExpirationDates LastTotalVolume
543 496 DJ ZD CBT Dow Jones Industrial Index Combined 0 10.0 10.0 USD $10 x Index points 1.0 10.0 1997-10-06 2015-06-19 NaN 0 0 0 1 NaN
748 699 YM YM CBT e-mini Dow ($5) Combined 1 5.0 5.0 USD $5 x Index points 1.0 5.0 2002-04-05 2020-10-28 DJ starts at 1997-10-06 0 0 0 1 177242.0

Verify that for all mentioned contracts the files are downloaded and available

In [30]:
# Verify that all the data files are available
ldf['cash_file_available'] = False
ldf['first_future_year'] = ''
ldf['first_future_month'] = ''
ldf['last_future_year'] = ''
ldf['last_future_month'] = ''

for idx, row in ldf_.iterrows():
    csi_symbol = row['csi_symbol']
    csi_file_symbol = csi_symbol.ljust(3, '_')
    dir_path = csi_futures_data.data_path3 + '/' + csi_symbol
    file_name_list = sorted(fnmatch.filter(os.listdir(dir_path), '*.CSV'))
    cash_file_name = file_name_list[0]
    
    r = re.match(r'^{}(\d\d\d\d)(.).*$'.format(csi_file_symbol), cash_file_name)
    if not r:
        raise RuntimeError('cash_file_name does not match file pattern')
    cash_file_available = r.group(2) == '$'
    ldf_.loc[idx, 'cash_file_available'] = cash_file_available
    if cash_file_available:
        futures_file_name_list_start = 1
    else:
        futures_file_name_list_start = 0
    
    futures_file_name_list = file_name_list[futures_file_name_list_start:]
    
    first_future = futures_file_name_list[0]
    r = re.match(r'^{}(\d\d\d\d)(.).*$'.format(csi_file_symbol), first_future)
    if not r:
        raise RuntimeError('first_future does not match file pattern')
    first_future_year  = r.group(1)
    first_future_month = r.group(2)
    
    last_future = futures_file_name_list[-1]
    r = re.match(r'^{}(\d\d\d\d)(.).*$'.format(csi_file_symbol), last_future)
    if not r:
        raise RuntimeError('last_future does not match file pattern')
    last_future_year  = r.group(1)
    last_future_month = r.group(2)
    ldf_.loc[idx, 'first_future_year'] = first_future_year
    ldf_.loc[idx, 'first_future_month'] = first_future_month
    ldf_.loc[idx, 'last_future_year'] = last_future_year
    ldf_.loc[idx, 'last_future_month'] = last_future_month
In [31]:
ldf2 = ldf_[['csi_symbol','root_symbol','multiplier','minor_fx_adj','description','sector', 'ExchangeSymbol','Exchange','Name','SessionType','FullPointValue','Currency','ContractSize','Units','StartDate','EndDate','LastTotalVolume','calc_poin_value',
 'factsheet_minor_fx_adj', 'cash_file_available','first_future_year','first_future_month','last_future_year','last_future_month']]
ldf2
Out[31]:
csi_symbol root_symbol multiplier minor_fx_adj description sector ExchangeSymbol Exchange Name SessionType FullPointValue Currency ContractSize Units StartDate EndDate LastTotalVolume calc_poin_value factsheet_minor_fx_adj cash_file_available first_future_year first_future_month last_future_year last_future_month
0 KC2 KC 37500.0 0.01 Coffee Agricultural KC ICEUS Coffee Combined 375.0 USD 37500 lbs cents/pound 1972-08-16 2020-10-28 31867.0 375.0 0.01 True 2000 Z 2023 U
1 C2 _C 5000.0 0.01 Corn Agricultural C CBT Corn Combined 50.0 USD 5000 bu cents/bu 1949-01-03 2020-10-28 426691.0 50.0 0.01 True 2000 X 2023 Z
2 C2 CN 5000.0 0.01 Corn Agricultural C CBT Corn Combined 50.0 USD 5000 bu cents/bu 1949-01-03 2020-10-28 426691.0 50.0 0.01 True 2000 X 2023 Z
3 CT2 CT 50000.0 0.01 Cotton #2 Agricultural CT ICEUS Cotton #2 Combined 500.0 USD 50000 lbs cents/pound 1967-03-22 2020-10-28 35749.0 500.0 0.01 True 2000 V 2023 N
4 LH LH 40000.0 0.01 Lean Hogs Agricultural LH CME Lean Hogs Combined 400.0 USD 40000 lbs cents/lb 1966-02-28 2020-10-28 30958.0 400.0 0.01 True 2000 V 2022 J
5 LC LC 40000.0 0.01 Live Cattle Agricultural LC CME Live Cattle Combined 400.0 USD 40000 lbs cents/lb 1964-11-30 2020-10-28 57781.0 400.0 0.01 True 2000 V 2022 G
6 LB LB 110.0 1.00 Lumber Agricultural LB CME Lumber Combined 110.0 USD 110000 board ft USD/1000 board ft 1969-10-01 2020-10-28 465.0 110.0 1.00 True 2000 X 2021 X
7 O2 _O 5000.0 0.01 Oats-CBT Agricultural O CBT Oats Combined 50.0 USD 5000 bu cents/bu 1949-01-03 2020-10-28 534.0 50.0 0.01 True 2000 Z 2023 U
8 O2 OA 5000.0 0.01 Oats-CBT Agricultural O CBT Oats Combined 50.0 USD 5000 bu cents/bu 1949-01-03 2020-10-28 534.0 50.0 0.01 True 2000 Z 2023 U
9 OJ2 OJ 15000.0 0.01 Orange Juice Agricultural OJ ICEUS Frozen Orange Juice Combined 150.0 USD 15000 lbs cents/pound 1966-10-26 2020-10-28 768.0 150.0 0.01 False 2000 X 2023 U
10 RR2 RR 2000.0 1.00 Rice(Rough) Agricultural RRF CBT Rough Rice Combined 2000.0 USD 2000CWT USD/CWT 1986-08-20 2020-10-28 1722.0 2000.0 1.00 True 2000 X 2021 X
11 S2 _S 5000.0 0.01 Soybean Agricultural S CBT Soybeans Combined 50.0 USD 5000bu cents/bu 1949-01-03 2020-10-28 342446.0 50.0 0.01 True 2000 X 2023 X
12 S2 SY 5000.0 0.01 Soybean Agricultural S CBT Soybeans Combined 50.0 USD 5000bu cents/bu 1949-01-03 2020-10-28 342446.0 50.0 0.01 True 2000 X 2023 X
13 LSU LS 50.0 1.00 Sugar #5(White) Agricultural W LCE White Sugar (#5) NaN 50.0 USD 50 tonnes USD/tonne 1987-07-30 2020-10-28 11840.0 50.0 1.00 True 2000 Z 2022 K
14 W2 _W 5000.0 0.01 Wheat Agricultural W CBT Wheat Combined 50.0 USD 5000 bu cents/bu 1996-03-01 2020-10-28 105855.0 50.0 0.01 True 2000 Z 2023 N
15 W2 WC 5000.0 0.01 Wheat Agricultural W CBT Wheat Combined 50.0 USD 5000 bu cents/bu 1996-03-01 2020-10-28 105855.0 50.0 0.01 True 2000 Z 2023 N
16 KW2 KW 5000.0 0.01 Wheat-Kansas City Agricultural KE KCBT Kansas City HRW Wheat Combined 50.0 USD 5000 bushels cents/bushel 1966-05-16 2020-10-28 45300.0 50.0 0.01 True 2000 Z 2023 N
17 MW2 MW 5000.0 0.01 Wheat-Spring Agricultural MWE MGE Spring Wheat Combined 50.0 USD 5000 bushels cents/bushel 1967-01-03 2020-10-28 8613.0 50.0 0.01 True 2000 Z 2022 K
18 DA DA 200000.0 0.01 Class III Milk Agricultural DA CME Class III Milk Combined 2000.0 USD 200000 lbs. USD/CWT 1996-01-11 2020-10-28 2095.0 2000.0 1.00 True 2000 V 2022 U
19 FC FC 50000.0 0.01 Cattle-Feeder Agricultural FC CME Feeder Cattle Combined 500.0 USD 50000 lbs cents/lb 1971-11-30 2020-10-28 10759.0 500.0 0.01 True 2000 V 2021 U
20 LRC LR 10.0 1.00 Robusta Coffee New (LCE) Agricultural RC LCE Robusta Coffee NaN 10.0 USD 10 tonne USD/tonne 2008-01-11 2020-10-28 14463.0 10.0 1.00 True 2000 X 2022 K
21 BO2 BO 60000.0 0.01 Soybean Oil Agricultural BO CBT Soybean Oil Combined 600.0 USD 60000lb cents/lb 1996-03-01 2020-10-28 109035.0 600.0 0.01 True 2000 V 2023 Z
22 SM2 SM 100.0 1.00 Soybean Meal Agricultural SM CBT Soybean Meal Combined 100.0 USD 100 short tons USD/short ton 1951-08-29 2020-10-28 108217.0 100.0 1.00 True 2000 V 2023 Z
23 CC2 CC 10.0 1.00 Cocoa Agricultural CC ICEUS Cocoa Combined 10.0 USD 10 tonnes USD/tonne 1965-12-30 2020-10-28 31624.0 10.0 1.00 True 2000 Z 2022 U
24 SB2 SB 112000.0 0.01 Sugar #11 Agricultural SB ICEUS Sugar #11 Combined 1120.0 USD 112000 lbs cents/pound 1965-12-30 2020-10-28 124605.0 1120.0 0.01 True 2001 H 2023 N
25 XC2 CM 1000.0 0.01 Corn E-Mini Agricultural YC CBT mini Corn Combined 10.0 USD 1000 bushels cents/bushel 2007-05-15 2020-10-28 2589.0 10.0 0.01 True 2007 N 2023 Z
26 XS2 MS 1000.0 0.01 Soybeans E-Mini Agricultural YK CBT mini Soybeans Combined 10.0 USD 1000 bushels cents/bushel 2007-05-15 2020-10-28 3510.0 10.0 0.01 False 2007 N 2023 X
27 CL2 CL 1000.0 1.00 Crude Oil-Light Non-Agricultural CL NYMEX Crude Oil Combined 1000.0 USD 1000 barrels USD/barrel 1983-03-30 2020-10-28 567900.0 1000.0 1.00 True 2000 X 2031 G
28 LCO LO 1000.0 1.00 Brent Crude Non-Agricultural B ICEEUROPE Brent Crude Combined 1000.0 USD 1000 barrels USD/barrel 1988-06-23 2020-10-28 781133.0 1000.0 1.00 True 2000 X 2029 H
29 HO2 HO 42000.0 1.00 Heating Oil Non-Agricultural HO NYMEX NY Harbor ULSD Combined 42000.0 USD 42000 gallons USD/gallon 1978-11-15 2020-10-28 184195.0 42000.0 1.00 True 2000 X 2024 F
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
53 NE NZ 100000.0 1.00 NZD/USD Currency NE CME New Zealand Dollar Combined 100000.0 USD 100000 NZD $/NZD 1997-05-07 2020-10-28 17026.0 100000.0 1.00 True 2000 Z 2022 H
54 SF SF 125000.0 1.00 CHF/USD Currency SF CME Swiss Franc Combined 125000.0 USD CHF 125000 USD/CHF 1972-05-16 2020-10-28 16511.0 125000.0 1.00 True 2000 Z 2025 U
55 DX2 DX 1000.0 1.00 USD Index Currency DX ICEUS U.S. Dollar Index Combined 1000.0 USD $1000 x Index points 1985-11-20 2020-10-28 14066.0 1000.0 1.00 True 2000 Z 2021 U
56 MP MP 500000.0 1.00 MEP/USD Currency MP CME Mexican Peso Combined 500000.0 USD MXN 500000 USD/MXN 1995-04-25 2020-10-28 41927.0 500000.0 1.00 True 2000 X 2022 H
57 MP ME 500000.0 1.00 MEP/USD Currency MP CME Mexican Peso Combined 500000.0 USD MXN 500000 USD/MXN 1995-04-25 2020-10-28 41927.0 500000.0 1.00 True 2000 X 2022 H
58 EX EE 62500.0 1.00 Euro FX E-mini Currency E7 CME e-mini Euro NaN 62500.0 USD EUR 62500 USD/EUR 1999-10-08 2020-10-28 4125.0 62500.0 1.00 False 2000 Z 2021 H
59 M6E EU 12500.0 1.00 E-micro EUR/USD Futures Currency M6E CME e-micro EUR/USD NaN 12500.0 USD EUR 12500 USD / EUR 2009-03-23 2020-10-28 15432.0 12500.0 1.00 True 2009 M 2021 H
60 JT JE 62500.0 1.00 Japanese Yen E-mini Currency J7 CME e-mini Japanese Yen NaN 62500.0 USD JPY 6250000 USD/JPY 100 1999-10-08 2020-10-28 427.0 62500.0 1.00 False 2000 Z 2021 H
61 ES ES 50.0 1.00 S&P 500 (E-mini) Equities ES CME e-mini S&P 500 Index Combined 50.0 USD $50 x Index points 1997-09-09 2020-10-28 1567010.0 50.0 1.00 True 2000 Z 2021 Z
62 NQ NQ 20.0 1.00 Nasdaq (E-mini) Equities NQ CME e-mini Nasdaq 100 Index NaN 20.0 USD $20 x Index points 1999-06-21 2020-10-28 552143.0 20.0 1.00 True 2000 Z 2021 Z
63 ER2 TF 50.0 1.00 Russel 2000 (E-mini) Equities RTY CME e-mini Russell 2000 Index Combined 50.0 USD $50 x Index points 2001-10-23 2020-10-28 143923.0 50.0 1.00 True 2001 Z 2021 Z
64 ER2 ER 50.0 1.00 Russel 2000 (E-mini) Equities RTY CME e-mini Russell 2000 Index Combined 50.0 USD $50 x Index points 2001-10-23 2020-10-28 143923.0 50.0 1.00 True 2001 Z 2021 Z
65 VX VX 1000.0 1.00 Volatility Index Equities VX CFE CBOE Volatility Index (VIX) Combined 1000.0 USD $1000 x Index points 2004-03-26 2020-10-28 155378.0 1000.0 1.00 True 2004 K 2021 N
66 YM YM 5.0 1.00 Dow e mini Equities YM CBT e-mini Dow ($5) Combined 5.0 USD $5 x Index points 2002-04-05 2020-10-28 177242.0 5.0 1.00 True 2000 Z 2021 U
67 NK NK 5.0 1.00 Nikkei 225 Equities NK CME Nikkei 225 Index Combined 5.0 USD $5.00 x Index points 1990-09-25 2020-10-28 3969.0 5.0 1.00 True 2000 Z 2025 Z
68 STW TW 100.0 1.00 MSCI Taiwan Equities TW SGX MSCI Taiwan Index Combined 100.0 USD $100 x Index points 1997-01-09 2020-10-28 64596.0 100.0 1.00 True 2000 V 2023 U
69 MEM EI 50.0 1.00 MSCI Emerging Markets Mini Equities MME ICEUS mini MSCI Emerging Markets Index NaN 50.0 USD USD 50 x Index points 2010-09-24 2020-10-28 70951.0 50.0 1.00 True 2010 Z 2023 U
70 MFS MG 50.0 1.00 MSCI EAFE Mini (Europe, Australasia and Far East) Equities MFS ICEUS mini MSCI EAFE Index NaN 50.0 USD USD 50 x Index points 2010-09-24 2020-10-28 25781.0 50.0 1.00 True 2010 Z 2023 U
71 EMD MI 100.0 1.00 S&P 400 MidCap E-Mini Equities EMD CME e-mini S&P Midcap 400 Index NaN 100.0 USD $100 x index points 2002-01-28 2020-10-28 9728.0 100.0 1.00 False 2000 Z 2021 Z
72 RS1 RM 50.0 1.00 Russell 1000 Mini Equities RS1 CME e-mini Russell 1000 Index NaN 50.0 USD $50 x index points 2006-06-15 2020-10-28 76.0 50.0 1.00 False 2006 M 2021 Z
73 SP2 SP 250.0 1.00 S&P 500 Futures Equities SP CME S&P 500 Index Combined 250.0 USD $250 x Index points 1982-04-21 2020-10-28 1237.0 250.0 1.00 False 2000 Z 2024 Z
74 FV FV 1000.0 1.00 US Treasury Note 5yr Rates FV CBT 5-Year U.S. T-Note Combined 1000.0 USD $100000 points 1988-05-20 2020-10-28 475624.0 1000.0 1.00 True 2000 Z 2021 M
75 ED ED 2500.0 1.00 Eurodollar 3m Rates ED CME Eurodollar Combined 2500.0 USD $1000000 USD/bas pt 1981-12-09 2020-10-28 964628.0 2500.0 1.00 True 2000 V 2030 U
76 US US 1000.0 1.00 US Treasury Long Bond 30yr Rates US CBT U.S. T-Bond Combined 1000.0 USD $100000 points 1977-08-22 2020-10-28 275204.0 1000.0 1.00 True 2000 Z 2021 M
77 TU TU 2000.0 1.00 US Treasury Note 2yr Rates TU CBT 2-Year U.S. T-Note Combined 2000.0 USD $200000 points 1990-06-22 2020-10-28 152282.0 2000.0 1.00 True 2000 Z 2021 M
78 TY TY 1000.0 1.00 US Treasury Note 10y Rates TYF CBT 10-Year U.S. T-Note Combined 1000.0 USD $100000 points 1982-05-03 2020-10-28 970079.0 1000.0 1.00 True 2000 Z 2021 M
79 FF FF 4167.0 1.00 30-Day Federal Funds Rates FF CBT 30 Day Federal Funds Combined 4167.0 USD $5000000 USD/bas pt 1988-10-03 2020-10-28 35066.0 4167.0 1.00 False 2000 V 2025 U
80 F1U FI 1000.0 1.00 5-Year Deliverable Interest Rate Swap Futures Rates F1U CBT 5yr Deliverable Interest Rate Swap Combined 1000.0 USD $100000 points 2012-12-03 2020-10-28 264.0 1000.0 1.00 True 2013 H 2021 H
81 N1U TN 1000.0 1.00 10-Year Deliverable Interest Rate Swap Futures Rates N1U CBT 10yr Deliverable Interest Rate Swap Combined 1000.0 USD $100000 points 2012-12-03 2020-10-28 1046.0 1000.0 1.00 True 2013 H 2021 H
82 UL2 UB 1000.0 1.00 Ultra Tbond Rates UL CBT Ultra T-Bonds Combined 1000.0 USD USD 100000 points 2010-01-11 2020-10-28 125777.0 1000.0 1.00 False 2010 H 2021 M

83 rows × 24 columns

In [32]:
# See above: not all contracts started at the beginning of the time window at 2000
# ldf2[ldf2['first_future_year'] > '2000']
In [33]:
# ldf2[ldf2['last_future_year'] < '2020']
In [34]:
# ldf2[ldf2['StartDate'] > pd.Timestamp('2000-12-31')]
In [35]:
#ldf_.to_excel('futures_portfolio_overview.xlsx')

Get an overview of all contracts plus information about their usage in the different trading strategies described by Andreas

In [36]:
ldf = csi_futures_data.futures_markets_lookup.copy()
ldf['any_market'] = ldf['trend_following_markets'] | ldf['time_return_markets'] | ldf['counter_trend_markets'] | ldf['curve_trading_markets']
ldf.head()
Out[36]:
index csi_symbol root_symbol multiplier minor_fx_adj description exchange sector is_in_bundle ROOT_SYMBOL_TO_ETA FUTURE_EXCHANGE_FEES_BY_SYMBOL trend_following_markets time_return_markets counter_trend_markets curve_trading_markets any_market
root_symbol
KC 0.0 KC2 KC 37500.0 0.01 Coffee CSCE agricultural True False False True True True True True
_C 1.0 C2 _C 5000.0 0.01 Corn CBT agricultural True False False False False False False False
CN 2.0 C2 CN 5000.0 0.01 Corn CBT agricultural True True True True True False True True
CT 3.0 CT2 CT 50000.0 0.01 Cotton #2 NYCE agricultural True False False True True True True True
LH 4.0 LH LH 40000.0 0.01 Lean Hogs CME agricultural True True True False False False False False
In [37]:
# ldf.to_excel('futures_markets_lookup.xlsx')
In [38]:
# I checked manually: the following are all not traded any more. They were part of the Quantopian futures offering in 2018.
ldf[~ldf['is_in_bundle']]
Out[38]:
index csi_symbol root_symbol multiplier minor_fx_adj description exchange sector is_in_bundle ROOT_SYMBOL_TO_ETA FUTURE_EXCHANGE_FEES_BY_SYMBOL trend_following_markets time_return_markets counter_trend_markets curve_trading_markets any_market
root_symbol
BD NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
DJ NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
EL NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
FS NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
HU NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
MB NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
MD NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
ND NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
PB NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
TB NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
TS NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
BL NaN NaN NaN NaN NaN NaN NaN NaN False False False True True True False True
In [39]:
# The zipline.finance.constants.ROOT_SYMBOL_TO_ETA and zipline.finance.constants.FUTURE_EXCHANGE_FEES_BY_SYMBOL variables agree with each other
ldf[ldf['ROOT_SYMBOL_TO_ETA'] != ldf['FUTURE_EXCHANGE_FEES_BY_SYMBOL']]
Out[39]:
index csi_symbol root_symbol multiplier minor_fx_adj description exchange sector is_in_bundle ROOT_SYMBOL_TO_ETA FUTURE_EXCHANGE_FEES_BY_SYMBOL trend_following_markets time_return_markets counter_trend_markets curve_trading_markets any_market
root_symbol
In [40]:
# See above: these are the contracts that are not traded any longer
ldf[pd.isnull(ldf['root_symbol'])]
Out[40]:
index csi_symbol root_symbol multiplier minor_fx_adj description exchange sector is_in_bundle ROOT_SYMBOL_TO_ETA FUTURE_EXCHANGE_FEES_BY_SYMBOL trend_following_markets time_return_markets counter_trend_markets curve_trading_markets any_market
root_symbol
BD NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
DJ NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
EL NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
FS NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
HU NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
MB NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
MD NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
ND NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
PB NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
TB NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
TS NaN NaN NaN NaN NaN NaN NaN NaN False True True False False False False False
BL NaN NaN NaN NaN NaN NaN NaN NaN False False False True True True False True
In [41]:
# The following are the contracts that are mentioned in Andreas' books, but were not part of the Quantopian futures offering
ldf[~pd.isnull(ldf['root_symbol']) & ~ldf['ROOT_SYMBOL_TO_ETA']]
Out[41]:
index csi_symbol root_symbol multiplier minor_fx_adj description exchange sector is_in_bundle ROOT_SYMBOL_TO_ETA FUTURE_EXCHANGE_FEES_BY_SYMBOL trend_following_markets time_return_markets counter_trend_markets curve_trading_markets any_market
root_symbol
KC 0.0 KC2 KC 37500.0 0.01 Coffee CSCE agricultural True False False True True True True True
_C 1.0 C2 _C 5000.0 0.01 Corn CBT agricultural True False False False False False False False
CT 3.0 CT2 CT 50000.0 0.01 Cotton #2 NYCE agricultural True False False True True True True True
_O 7.0 O2 _O 5000.0 0.01 Oats-CBT CBT agricultural True False False False False False False False
OJ 9.0 OJ2 OJ 15000.0 0.01 Orange Juice NYCE agricultural True False False False False True False True
_S 11.0 S2 _S 5000.0 0.01 Soybean CBT agricultural True False False False False False False False
LS 13.0 LSU LS 50.0 1.00 Sugar #5(White) EURONEXT agricultural True False False True True True False True
_W 14.0 W2 _W 5000.0 0.01 Wheat CBT agricultural True False False False False False False False
KW 16.0 KW2 KW 5000.0 0.01 Wheat-Kansas City KCBT agricultural True False False False False False False False
DA 18.0 DA DA 200000.0 0.01 Class III Milk CME agricultural True False False False False False False False
LR 20.0 LRC LR 10.0 1.00 Robusta Coffee New (LCE) EURONEXT agricultural True False False True True True False True
CC 23.0 CC2 CC 10.0 1.00 Cocoa NYCE agricultural True False False False True True False True
LO 28.0 LCO LO 1000.0 1.00 Brent Crude ICE nonagricultural True False False False False True False True
RB 31.0 RB2 RB 42000.0 1.00 Gasoline-Reformulated Blendstock NYMEX nonagricultural True False False True True True True True
SI 36.0 SI2 SI 5000.0 0.01 Silver COMEX nonagricultural True False False False False False False False
LG 38.0 LGO LG 100.0 1.00 Petroleum Gas Oil ICE nonagricultural True False False True True True False True
CU 48.0 CU CU 125000.0 1.00 EUR/USD CME currencies True False False False False False False False
NE 52.0 NE NE 100000.0 1.00 NZD/USD CME currencies True False False False False False False False
DX 55.0 DX2 DX 1000.0 1.00 USD Index FINEX currencies True False False True True True False True
MP 56.0 MP MP 500000.0 1.00 MEP/USD CME currencies True False False False False False False False
TF 63.0 ER2 TF 50.0 1.00 Russel 2000 (E-mini) CME equities True False False False False False False False
TW 68.0 STW TW 100.0 1.00 MSCI Taiwan SGX equities True False False True True False False True
In [42]:
# Get all the symbols that Andreas used in his trend_following_markets strategy
tfm1 = csi_futures_data.get_bundle_market_symbols('trend_following_markets')
tfm1
Out[42]:
['KC',
 'CN',
 'CT',
 'OA',
 'SY',
 'LS',
 'WC',
 'FC',
 'LR',
 'SM',
 'SB',
 'CL',
 'HO',
 'NG',
 'RB',
 'GC',
 'HG',
 'PA',
 'PL',
 'SV',
 'LG',
 'AD',
 'BP',
 'EC',
 'CD',
 'JY',
 'NZ',
 'SF',
 'DX',
 'ES',
 'NQ',
 'VX',
 'YM',
 'NK',
 'TW',
 'FV',
 'ED',
 'US',
 'TU',
 'TY']
In [43]:
tfm2 = csi_futures_data.trend_following_markets
tfm2
Out[43]:
['AD',
 'BP',
 'CD',
 'EC',
 'DX',
 'JY',
 'NZ',
 'SF',
 'CN',
 'CT',
 'FC',
 'KC',
 'LR',
 'LS',
 'OA',
 'SY',
 'SB',
 'SM',
 'WC',
 'BL',
 'CL',
 'GC',
 'HG',
 'HO',
 'LG',
 'NG',
 'PA',
 'PL',
 'RB',
 'SV',
 'ES',
 'NK',
 'NQ',
 'TW',
 'VX',
 'YM',
 'ED',
 'FV',
 'TU',
 'TY',
 'US']
In [44]:
set(tfm1) - set(tfm2)
Out[44]:
set()
In [45]:
set(tfm2) - set(tfm1)
Out[45]:
{'BL'}