# Supervised Machine Learning basics: Titanic example¶

Florent Leclercq,
Imperial Centre for Inference and Cosmology, Imperial College London,
[email protected]

In :
import numpy as np
from sklearn.ensemble import RandomForestClassifier


In :
# data set available at this address: https://www.kaggle.com/c/titanic/data
# (version slightly modified to be conveniently loaded with numpy)
dtype = {'names':('PassengerId','Survived','Pclass','Name','Sex',
'Age','SibSp','Parch','Ticket','Fare','Cabin','Embarked'),
'formats': ('i4','i4','i4','S20','S6','S20','i4','i4','S20','f8','S20','S20')}


Data dictionary

Variable Definition Key
Survived Survival 0 = No, 1 = Yes
Pclass Ticket class 1 = 1st, 2 = 2nd, 3 = 3rd
Sex Sex
Age Age in years
SibSp # of siblings / spouses aboard the Titanic
Parch # of parents / children aboard the Titanic
Ticket Ticket number
Fare Passenger fare
Cabin Cabin number
Embarked Port of Embarkation C = Cherbourg, Q = Queenstown, S = Southampton
In :
data

Out:
(501, 0, 3, b'"Calic, Mr. Petar"', b'male', b'17', 0, 0, b'315086', 8.6625, b'', b'S')
In :
data

Out:
(601, 1, 2, b'"Jacobsohn, Mrs. Sid', b'female', b'24', 2, 1, b'243847', 27., b'', b'S')
In :
def perform_splitting(condition):
selected_passengers = data[np.where(condition)]
Nsplit = selected_passengers.size
Nsurvived_split = np.sum(selected_passengers['Survived']==1)
Ndied_split = np.sum(selected_passengers['Survived']==0)
return Nsurvived_split, Ndied_split, Nsplit

def entropy(Nsurvived, Ndied, Ntot):
assert(Nsurvived + Ndied == Ntot)
return -Nsurvived/Ntot*np.log2(Nsurvived/Ntot) -Ndied/Ntot*np.log2(Ndied/Ntot)


## Parent Entropy¶

In :
Ntot = data['Survived'].size
Nsurvived = np.sum(data['Survived']==1)
Ndied = np.sum(data['Survived']==0)
H_parent = entropy(Nsurvived, Ndied, Ntot)
print("Ndied={}, Nsurvived={}, Ntot={}".format(Ndied,Nsurvived,Ntot))
print("H_parent={}".format(H_parent))

Ndied=549, Nsurvived=342, Ntot=891
H_parent=0.9607079018756469


## Information gain: ticket class¶

In :
Nsurvived_first, Ndied_first, Nfirst = perform_splitting(data['Pclass']==1)
H_first = entropy(Nsurvived_first, Ndied_first, Nfirst)
print("Ndied_first={}, Nsurvived_first={}, Nfirst={}".format(Ndied_first,Nsurvived_first,Nfirst))
print("H_first={}".format(H_first))

Ndied_first=80, Nsurvived_first=136, Nfirst=216
H_first=0.9509560484549725

In :
Nsurvived_second, Ndied_second, Nsecond = perform_splitting(data['Pclass']==2)
H_second = entropy(Nsurvived_second, Ndied_second, Nsecond)
print("Ndied_second={}, Nsurvived_second={}, Nsecond={}".format(Ndied_second,Nsurvived_second,Nsecond))
print("H_second={}".format(H_second))

Ndied_second=97, Nsurvived_second=87, Nsecond=184
H_second=0.9978683156711936

In :
Nsurvived_third, Ndied_third, Nthird = perform_splitting(data['Pclass']==3)
H_third = entropy(Nsurvived_third, Ndied_third, Nthird)
print("Ndied_third={}, Nsurvived_third={}, Nthird={}".format(Ndied_third,Nsurvived_third,Nthird))
print("H_third={}".format(H_third))

Ndied_third=372, Nsurvived_third=119, Nthird=491
H_third=0.7989470522661535

In :
H_class = Nfirst/Ntot*H_first + Nsecond/Ntot*H_second + Nthird/Ntot*H_third
IG_class = H_parent - H_class
print("IG_class={}".format(IG_class))

IG_class=0.0838310452960116


## Information gain: sex¶

In :
Nsurvived_male, Ndied_male, Nmale = perform_splitting(data['Sex']==b'male')
H_male = entropy(Nsurvived_male, Ndied_male, Nmale)
print("Ndied_male={}, Nsurvived_male={}, Nmale={}".format(Ndied_male,Nsurvived_male,Nmale))
print("H_male={}".format(H_male))

Ndied_male=468, Nsurvived_male=109, Nmale=577
H_male=0.6991817891208407

In :
Nsurvived_female, Ndied_female, Nfemale = perform_splitting(data['Sex']==b'female')
H_female = entropy(Nsurvived_female, Ndied_female, Nfemale)
print("Ndied_female={}, Nsurvived_female={}, Nfemale={}".format(Ndied_female,Nsurvived_female,Nfemale))
print("H_female={}".format(H_female))

Ndied_female=81, Nsurvived_female=233, Nfemale=314
H_female=0.8236550739295191

In :
H_sex = Nmale/Ntot*H_male + Nfemale/Ntot*H_female
IG_sex = H_parent - H_sex
print("IG_sex={}".format(IG_sex))

IG_sex=0.2176601066606142


## Information gain: port of embarkation¶

In :
Nsurvived_cherbourg, Ndied_cherbourg, Ncherbourg = perform_splitting(data['Embarked']==b'C')
H_cherbourg = entropy(Nsurvived_cherbourg, Ndied_cherbourg, Ncherbourg)
print("Ndied_cherbourg={}, Nsurvived_cherbourg={}, Ncherbourg={}"
.format(Ndied_cherbourg,Nsurvived_cherbourg,Ncherbourg))
print("H_cherbourg={}".format(H_cherbourg))

Ndied_cherbourg=75, Nsurvived_cherbourg=93, Ncherbourg=168
H_cherbourg=0.9917033083725818

In :
Nsurvived_queenstown, Ndied_queenstown, Nqueenstown = perform_splitting(data['Embarked']==b'Q')
H_queenstown = entropy(Nsurvived_queenstown, Ndied_queenstown, Nqueenstown)
print("Ndied_queenstown={}, Nsurvived_queenstown={}, Nqueenstown={}"
.format(Ndied_queenstown,Nsurvived_queenstown,Nqueenstown))
print("H_queenstown={}".format(H_queenstown))

Ndied_queenstown=47, Nsurvived_queenstown=30, Nqueenstown=77
H_queenstown=0.9645476589143234

In :
Nsurvived_southampton, Ndied_southampton, Nsouthampton = perform_splitting(data['Embarked']==b'S')
H_southampton = entropy(Nsurvived_southampton, Ndied_southampton, Nsouthampton)
print("Ndied_southampton={}, Nsurvived_southampton={}, Nsouthampton={}"
.format(Ndied_southampton,Nsurvived_southampton,Nsouthampton))
print("H_southampton={}".format(H_southampton))

Ndied_southampton=427, Nsurvived_southampton=217, Nsouthampton=644
H_southampton=0.921876486346913

In :
H_embarked = Ncherbourg/Ntot*H_cherbourg + Nqueenstown/Ntot*H_queenstown + Nsouthampton/Ntot*H_southampton
IG_embarked = H_parent - H_embarked
print("IG_embarked={}".format(IG_embarked))

IG_embarked=0.024047090707960517


## Off-the-shelf machine learning algorithm¶

In :
data_Sex=np.zeros(len(data),dtype=np.int)
data_Sex[np.where(data['Sex']==b'male')]=1
data_Sex[np.where(data['Sex']==b'female')]=2
data_Embarked=3*np.ones(len(data),dtype=np.int)
data_Embarked[np.where(data['Embarked']==b'C')]=0
data_Embarked[np.where(data['Embarked']==b'Q')]=1
data_Embarked[np.where(data['Embarked']==b'S')]=2
data_Embarked[np.where((data_Embarked!=0)*(data_Embarked!=1)*(data_Embarked!=2))]=3

In :
features = np.array((data['Pclass'],data_Sex,data_Embarked),dtype=np.int).T
label = data['Survived']

In :
model = RandomForestClassifier(n_estimators=10)
model.fit(features, label)

Out:
RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
max_depth=None, max_features='auto', max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=None,
oob_score=False, random_state=None, verbose=0,
warm_start=False)
In :
# prediction for a woman in first class, embarked in Southampton
ans = model.predict([[1, 2, 0]])
survival = "survived" if ans==1 else "died"
print(survival)

survived

In :
# prediction for a man in third class, embarked in Cherbourg
ans = model.predict([[3, 1, 0]])
survival = "survived" if ans==1 else "died"
print(survival)

died