Import necessary dependencies¶
In [1]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import model_evaluation_utils as meu
from sklearn.model_selection import train_test_split
from collections import Counter
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder
%matplotlib inline
Load and Merge datasets¶
In [2]:
white_wine = pd.read_csv('winequality-white.csv', sep=';')
red_wine = pd.read_csv('winequality-red.csv', sep=';')
red_wine['wine_type'] = 'red' # add a column for the type
white_wine['wine_type'] = 'white'
wines = pd.concat([red_wine, white_wine])
wines['quality_label'] = wines['quality'].apply(lambda value: 'low' if value <= 5 else 'medium' if value <= 7 else 'high')
wines = wines.sample(frac=1, random_state=42).reset_index(drop=True)
In [3]:
wines.head()
Out[3]:
Predicting Wine Types¶
Prepare Training and Testing datasets¶
In [58]:
wtp_features = wines.iloc[:,:-3]
wtp_feature_names = wtp_features.columns
wtp_class_labels = np.array(wines['wine_type'])
wtp_train_X, wtp_test_X, wtp_train_y, wtp_test_y = train_test_split(wtp_features, wtp_class_labels,
test_size=0.3, random_state=42)
print(Counter(wtp_train_y), Counter(wtp_test_y))
print('Features:', list(wtp_feature_names))
Feature Scaling¶
In [5]:
# Define the scaler
wtp_ss = StandardScaler().fit(wtp_train_X)
# Scale the train set
wtp_train_SX = wtp_ss.transform(wtp_train_X)
# Scale the test set
wtp_test_SX = wtp_ss.transform(wtp_test_X)
Train a Model using Logistic Regression¶
In [6]:
from sklearn.linear_model import LogisticRegression
wtp_lr = LogisticRegression()
wtp_lr.fit(wtp_train_SX, wtp_train_y)
Out[6]:
Predict and Evaluate Model Performance¶
In [7]:
wtp_lr_predictions = wtp_lr.predict(wtp_test_SX)
meu.display_model_performance_metrics(true_labels=wtp_test_y, predicted_labels=wtp_lr_predictions,
classes=['red', 'white'])
Train a Model using Deep Learning (MLP)¶
Encode Response class labels¶
In [8]:
le = LabelEncoder()
le.fit(wtp_train_y)
# encode wine type labels
wtp_train_ey = le.transform(wtp_train_y)
wtp_test_ey = le.transform(wtp_test_y)
Build & Compile DNN Model Architecture¶
In [41]:
from keras.models import Sequential
from keras.layers import Dense
wtp_dnn_model = Sequential()
wtp_dnn_model.add(Dense(16, activation='relu', input_shape=(11,)))
wtp_dnn_model.add(Dense(16, activation='relu'))
wtp_dnn_model.add(Dense(16, activation='relu'))
wtp_dnn_model.add(Dense(1, activation='sigmoid'))
wtp_dnn_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
Train the Model¶
In [42]:
history = wtp_dnn_model.fit(wtp_train_SX, wtp_train_ey, epochs=10, batch_size=5,
shuffle=True, validation_split=0.1, verbose=1)
Predict on Test dataset¶
In [43]:
wtp_dnn_ypred = wtp_dnn_model.predict_classes(wtp_test_SX)
wtp_dnn_predictions = le.inverse_transform(wtp_dnn_ypred)
Evaluate Model Performance¶
In [44]:
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 4))
t = f.suptitle('Deep Neural Net Performance', fontsize=12)
f.subplots_adjust(top=0.85, wspace=0.3)
epochs = list(range(1,11))
ax1.plot(epochs, history.history['acc'], label='Train Accuracy')
ax1.plot(epochs, history.history['val_acc'], label='Validation Accuracy')
ax1.set_xticks(epochs)
ax1.set_ylabel('Accuracy Value')
ax1.set_xlabel('Epoch')
ax1.set_title('Accuracy')
l1 = ax1.legend(loc="best")
ax2.plot(epochs, history.history['loss'], label='Train Loss')
ax2.plot(epochs, history.history['val_loss'], label='Validation Loss')
ax2.set_xticks(epochs)
ax2.set_ylabel('Loss Value')
ax2.set_xlabel('Epoch')
ax2.set_title('Loss')
l2 = ax2.legend(loc="best")
In [45]:
meu.display_model_performance_metrics(true_labels=wtp_test_y, predicted_labels=wtp_dnn_predictions,
classes=['red', 'white'])
Model Interpretation¶
View Feature importances¶
In [14]:
from skater.core.explanations import Interpretation
from skater.model import InMemoryModel
wtp_interpreter = Interpretation(wtp_test_SX, feature_names=wtp_features.columns)
wtp_im_model = InMemoryModel(wtp_lr.predict_proba, examples=wtp_train_SX, target_names=wtp_lr.classes_)
plots = wtp_interpreter.feature_importance.plot_feature_importance(wtp_im_model, ascending=False)
View model ROC curve¶
In [15]:
meu.plot_model_roc_curve(wtp_lr, wtp_test_SX, wtp_test_y)
Visualize Model Decision Surface¶
In [59]:
feature_indices = [i for i, feature in enumerate(wtp_feature_names)
if feature in ['density', 'total sulfur dioxide']]
meu.plot_model_decision_surface(clf=wtp_lr, train_features=wtp_train_SX[:, feature_indices],
train_labels=wtp_train_y, plot_step=0.02, cmap=plt.cm.Wistia_r,
markers=[',', 'o'], alphas=[0.9, 0.6], colors=['r', 'y'])
Predicting Wine Quality¶
Prepare Training and Testing datasets¶
In [17]:
wqp_features = wines.iloc[:,:-3]
wqp_class_labels = np.array(wines['quality_label'])
wqp_label_names = ['low', 'medium', 'high']
wqp_feature_names = list(wqp_features.columns)
wqp_train_X, wqp_test_X, wqp_train_y, wqp_test_y = train_test_split(wqp_features, wqp_class_labels,
test_size=0.3, random_state=42)
print(Counter(wqp_train_y), Counter(wqp_test_y))
print('Features:', wqp_feature_names)
Feature Scaling¶
In [18]:
# Define the scaler
wqp_ss = StandardScaler().fit(wqp_train_X)
# Scale the train set
wqp_train_SX = wqp_ss.transform(wqp_train_X)
# Scale the test set
wqp_test_SX = wqp_ss.transform(wqp_test_X)
Train, Predict & Evaluate Model using Decision Tree¶
In [19]:
from sklearn.tree import DecisionTreeClassifier
wqp_dt = DecisionTreeClassifier()
wqp_dt.fit(wqp_train_SX, wqp_train_y)
wqp_dt_predictions = wqp_dt.predict(wqp_test_SX)
meu.display_model_performance_metrics(true_labels=wqp_test_y, predicted_labels=wqp_dt_predictions,
classes=wqp_label_names)
View Feature Importances from Decision Tree Model¶
In [20]:
wqp_dt_feature_importances = wqp_dt.feature_importances_
wqp_dt_feature_names, wqp_dt_feature_scores = zip(*sorted(zip(wqp_feature_names, wqp_dt_feature_importances),
key=lambda x: x[1]))
y_position = list(range(len(wqp_dt_feature_names)))
plt.barh(y_position, wqp_dt_feature_scores, height=0.6, align='center')
plt.yticks(y_position , wqp_dt_feature_names)
plt.xlabel('Relative Importance Score')
plt.ylabel('Feature')
t = plt.title('Feature Importances for Decision Tree')
Visualize the Decision Tree¶
In [21]:
from graphviz import Source
from sklearn import tree
from IPython.display import Image
graph = Source(tree.export_graphviz(wqp_dt, out_file=None, class_names=wqp_label_names,
filled=True, rounded=True, special_characters=False,
feature_names=wqp_feature_names, max_depth=3))
png_data = graph.pipe(format='png')
with open('dtree_structure.png','wb') as f:
f.write(png_data)
Image(png_data)
Out[21]:
Train, Predict & Evaluate Model using Random Forests¶
In [22]:
from sklearn.ensemble import RandomForestClassifier
# train the model
wqp_rf = RandomForestClassifier()
wqp_rf.fit(wqp_train_SX, wqp_train_y)
# predict and evaluate performance
wqp_rf_predictions = wqp_rf.predict(wqp_test_SX)
meu.display_model_performance_metrics(true_labels=wqp_test_y, predicted_labels=wqp_rf_predictions,
classes=wqp_label_names)
Hyperparameter tuning with Grid Search & Cross Validation¶
In [23]:
print(wqp_rf.get_params())
Get the best hyperparameter values¶
In [24]:
from sklearn.model_selection import GridSearchCV
param_grid = {
'n_estimators': [100, 200, 300, 500],
'max_features': ['auto', None, 'log2']
}
wqp_clf = GridSearchCV(RandomForestClassifier(random_state=42), param_grid, cv=5,
scoring='accuracy')
wqp_clf.fit(wqp_train_SX, wqp_train_y)
print(wqp_clf.best_params_)
View grid search results¶
In [25]:
results = wqp_clf.cv_results_
for param, score_mean, score_sd in zip(results['params'], results['mean_test_score'], results['std_test_score']):
print(param, round(score_mean, 4), round(score_sd, 4))
Train, Predict & Evaluate Random Forest Model with tuned hyperparameters¶
In [26]:
wqp_rf = RandomForestClassifier(n_estimators=200, max_features='auto', random_state=42)
wqp_rf.fit(wqp_train_SX, wqp_train_y)
wqp_rf_predictions = wqp_rf.predict(wqp_test_SX)
meu.display_model_performance_metrics(true_labels=wqp_test_y, predicted_labels=wqp_rf_predictions,
classes=wqp_label_names)
Train, Predict & Evaluate Model using Extreme Gradient Boosting¶
Load and set dependencies¶
In [27]:
import os
mingw_path = r'C:\mingw-w64\mingw64\bin'
os.environ['PATH'] = mingw_path + ';' + os.environ['PATH']
import xgboost as xgb
Train the model¶
In [28]:
wqp_xgb_model = xgb.XGBClassifier(seed=42)
wqp_xgb_model.fit(wqp_train_SX, wqp_train_y)
Out[28]:
Predict and Evaluate Model¶
In [29]:
wqp_xgb_predictions = wqp_xgb_model.predict(wqp_test_SX)
meu.display_model_performance_metrics(true_labels=wqp_test_y, predicted_labels=wqp_xgb_predictions,
classes=wqp_label_names)
Get the best hyperparameter values¶
In [30]:
param_grid = {
'n_estimators': [100, 200, 300],
'max_depth': [5, 10, 15],
'learning_rate': [0.3, 0.5]
}
wqp_clf = GridSearchCV(xgb.XGBClassifier(tree_method='exact', seed=42), param_grid,
cv=5, scoring='accuracy')
wqp_clf.fit(wqp_train_SX, wqp_train_y)
print(wqp_clf.best_params_)
View grid search results¶
In [31]:
results = wqp_clf.cv_results_
for param, score_mean, score_sd in zip(results['params'], results['mean_test_score'], results['std_test_score']):
print(param, round(score_mean, 4), round(score_sd, 4))
Train, Predict & Evaluate Extreme Gradient Boosted Model with tuned hyperparameters¶
In [32]:
wqp_xgb_model = xgb.XGBClassifier(seed=42, max_depth=10, learning_rate=0.3, n_estimators=100)
wqp_xgb_model.fit(wqp_train_SX, wqp_train_y)
wqp_xgb_predictions = wqp_xgb_model.predict(wqp_test_SX)
meu.display_model_performance_metrics(true_labels=wqp_test_y, predicted_labels=wqp_xgb_predictions,
classes=wqp_label_names)
Model Interpretation¶
Comparative analysis of Model Feature importances¶
In [33]:
from skater.core.explanations import Interpretation
from skater.model import InMemoryModel
# leveraging skater for feature importances
interpreter = Interpretation(wqp_test_SX, feature_names=wqp_feature_names)
wqp_im_model = InMemoryModel(wqp_rf.predict_proba, examples=wqp_train_SX, target_names=wqp_rf.classes_)
# retrieving feature importances from the scikit-learn estimator
wqp_rf_feature_importances = wqp_rf.feature_importances_
wqp_rf_feature_names, wqp_rf_feature_scores = zip(*sorted(zip(wqp_feature_names, wqp_rf_feature_importances),
key=lambda x: x[1]))
# plot the feature importance plots
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 4))
t = f.suptitle('Feature Importances for Random Forest', fontsize=12)
f.subplots_adjust(top=0.85, wspace=0.6)
y_position = list(range(len(wqp_rf_feature_names)))
ax1.barh(y_position, wqp_rf_feature_scores, height=0.6, align='center', tick_label=wqp_rf_feature_names)
ax1.set_title("Scikit-Learn")
ax1.set_xlabel('Relative Importance Score')
ax1.set_ylabel('Feature')
plots = interpreter.feature_importance.plot_feature_importance(wqp_im_model, ascending=False, ax=ax2)
ax2.set_title("Skater")
ax2.set_xlabel('Relative Importance Score')
ax2.set_ylabel('Feature')
Out[33]:
View Model ROC Curve¶
In [34]:
meu.plot_model_roc_curve(wqp_rf, wqp_test_SX, wqp_test_y)
Visualize Model decision surface¶
In [35]:
feature_indices = [i for i, feature in enumerate(wqp_feature_names)
if feature in ['alcohol', 'volatile acidity']]
meu.plot_model_decision_surface(clf=wqp_rf, train_features=wqp_train_SX[:, feature_indices],
train_labels=wqp_train_y, plot_step=0.02, cmap=plt.cm.RdYlBu,
markers=[',', 'd', '+'], alphas=[1.0, 0.8, 0.5], colors=['r', 'b', 'y'])
Interpreting Model Predictions¶
In [36]:
from skater.core.local_interpretation.lime.lime_tabular import LimeTabularExplainer
exp = LimeTabularExplainer(wqp_train_SX, feature_names=wqp_feature_names,
discretize_continuous=True,
class_names=wqp_rf.classes_)
In [80]:
exp.explain_instance(wqp_test_SX[10], wqp_rf.predict_proba, top_labels=1).show_in_notebook()