In [1]:
%pylab inline
In [2]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
Import titanic data using pandas¶
Modified version of the "Titanic" data can be found at: http://http://facweb.cs.depaul.edu/mobasher/classes/csc478/Data/titanic-trimmed.csv. Original unmodified Titanic data is available at CMU StatLib.
In [3]:
url = "http://facweb.cs.depaul.edu/mobasher/classes/csc478/Data/titanic-trimmed.csv"
titanic = pd.read_csv(url)
titanic.head(10)
Out[3]:
In [5]:
titanic.describe(include="all")
Out[5]:
Handling missing variables¶
In [6]:
titanic[titanic.age.isnull()].shape
Out[6]:
In [7]:
age_mean = titanic.age.mean()
titanic.age.fillna(age_mean, axis=0, inplace=True)
titanic.dropna(axis=0, inplace=True)
In [8]:
titanic.shape
Out[8]:
In [9]:
titanic.set_index('pid', drop=True, inplace=True)
titanic.head()
Out[9]:
Creating dummy variables for categorical features¶
In [10]:
titanic_ssf = pd.get_dummies(titanic)
titanic_ssf.head(10)
Out[10]:
In [11]:
titanic_names = titanic_ssf.columns.values
titanic_names
Out[11]:
In [12]:
y = titanic_ssf['survived']
X = titanic_ssf[titanic_names[1:]]
X.head()
Out[12]:
In [13]:
titanic_ssf.describe().T
Out[13]:
Build the training and testing dataset¶
In [14]:
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=33)
A versatile function to measure performance of a model¶
In [15]:
from sklearn import metrics
def measure_performance(X, y, clf, show_accuracy=True, show_classification_report=True, show_confussion_matrix=True):
y_pred = clf.predict(X)
if show_accuracy:
print "Accuracy:{0:.3f}".format(metrics.accuracy_score(y, y_pred)),"\n"
if show_classification_report:
print "Classification report"
print metrics.classification_report(y, y_pred),"\n"
if show_confussion_matrix:
print "Confussion matrix"
print metrics.confusion_matrix(y, y_pred),"\n"
Let's now compare performnace of standard decision tree classifier against some ensemble methods: Random Forest Classifier and Ada Boost.¶
In [16]:
from sklearn import tree
dt = tree.DecisionTreeClassifier(criterion='entropy')
dt = dt.fit(X_train, y_train)
In [17]:
from sklearn import metrics
measure_performance(X_test, y_test, dt, show_confussion_matrix=False, show_classification_report=False)
In [24]:
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier()
rf = rf.fit(X_train, y_train)
In [25]:
measure_performance(X_test, y_test, rf, show_confussion_matrix=False, show_classification_report=False)
Exploring and comparing model parameters¶
In [26]:
print rf.get_params()
The following is a general function that performs cross-validation using a range of values for a specified parameter of a model¶
In [27]:
from sklearn.cross_validation import KFold
def calc_params(X, y, clf, param_values, param_name, K):
# Convert input to Numpy arrays
X = np.array(X)
y = np.array(y)
# initialize training and testing scores with zeros
train_scores = np.zeros(len(param_values))
test_scores = np.zeros(len(param_values))
# iterate over the different parameter values
for i, param_value in enumerate(param_values):
print param_name, ' = ', param_value
# set classifier parameters
clf.set_params(**{param_name:param_value})
# initialize the K scores obtained for each fold
k_train_scores = np.zeros(K)
k_test_scores = np.zeros(K)
# create KFold cross validation
cv = KFold(len(X), K, shuffle=True, random_state=0)
# iterate over the K folds
for j, (train, test) in enumerate(cv):
# fit the classifier in the corresponding fold
# and obtain the corresponding accuracy scores on train and test sets
clf.fit([X[k] for k in train], y[train])
k_train_scores[j] = clf.score([X[k] for k in train], y[train])
k_test_scores[j] = clf.score([X[k] for k in test], y[test])
# store the mean of the K fold scores
train_scores[i] = np.mean(k_train_scores)
test_scores[i] = np.mean(k_test_scores)
# plot the training and testing scores in a log scale
plt.plot(param_values, train_scores, label='Train', alpha=0.4, lw=2, c='b')
plt.plot(param_values, test_scores, label='X-Val', alpha=0.4, lw=2, c='g')
plt.legend(loc=7)
plt.xlabel(param_name + " values")
plt.ylabel("Mean cross validation accuracy")
# return the training and testing scores on each parameter value
return train_scores, test_scores
Now we can explore the impact of min_samples_leaf more systematically¶
In [29]:
msl = range(1,6)
print msl
In [30]:
train_scores, test_scores = calc_params(X_train, y_train, rf, msl, 'min_samples_leaf', 5)
In [31]:
nest = range(5, 101, 5)
print nest
In [32]:
train_scores, test_scores = calc_params(X_train, y_train, rf, nest, 'n_estimators', 5)
In [33]:
rf = RandomForestClassifier(n_estimators=25, min_samples_leaf=3)
rf = rf.fit(X_train, y_train)
measure_performance(X_test, y_test, rf, show_confussion_matrix=False, show_classification_report=False)
In [34]:
rf.feature_importances_
Out[34]:
In [36]:
rf.estimators_
Out[36]:
In [37]:
from sklearn.ensemble import AdaBoostClassifier
In [38]:
ab = AdaBoostClassifier()
ab = ab.fit(X_train, y_train)
In [39]:
measure_performance(X_test, y_test, ab, show_confussion_matrix=False, show_classification_report=False)
In [40]:
train_scores, test_scores = calc_params(X_train, y_train, ab, nest, 'n_estimators', 5)
In [43]:
ab = AdaBoostClassifier(n_estimators=20)
ab = ab.fit(X_train, y_train)
measure_performance(X_test, y_test, ab, show_confussion_matrix=False, show_classification_report=False)