House Prices: Advanced Regression Techniques¶

# Imports

# Necessary imports
import os
import time
import timeit
import numpy as np
import pandas as pd
import scipy
import warnings
warnings.filterwarnings('ignore')

# Visualization packages
import sklearn
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

# Modelling packages
from sklearn import ensemble, linear_model
from sklearn.feature_selection import chi2, f_classif, SelectKBest 
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import adjusted_rand_score, classification_report, confusion_matrix, silhouette_score
from sklearn.model_selection import cross_val_score, GridSearchCV, train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import normalize
from sklearn.utils import resample

# Plotly packages
import cufflinks as cf
import ipywidgets as widgets
import plotly as py
import plotly.figure_factory as ff
import plotly.graph_objs as go
from plotly import tools
from scipy import special
py.offline.init_notebook_mode(connected=True)

# Import the data
housing_train = pd.read_csv("train.csv")
housing_test = pd.read_csv("test.csv")

# Drop the 'id' column since it's not a predictor
housing_train.drop(['Id'], axis=1, inplace=True)

# Preview the dataset
housing_train.head()

# Print the size of the dataframe
print('Size of the dataframe: {} listings x {} features'.format(housing_train.shape[0],housing_train.shape[1]))

# View all 80 data columns
print('Column names: \n\n', housing_train.columns)

# Check for any missing values
print('Missing values: \n\n', housing_train.isna().sum().sort_values(ascending=False).head(20))

# Visualize the distribution of housing prices
plt.rcParams['figure.figsize'] = [10,5]

plt.xlim(0,500000)
plt.title('Sale Price Distribution')
sns.distplot(housing_train['SalePrice'])

np.log(8)

# Apply log transformation to the sale prices
housing_train['Sale_norm'] = np.log(housing_train['SalePrice'])

# Visualize the distribution of housing prices
plt.rcParams['figure.figsize'] = [10,5]

plt.title('Sale Price Distribution')
sns.distplot(housing_train['Sale_norm'])

# View descriptive statistics for all numerical categories
housing_train.describe()

# Describe unique occurences for each categorical variable
print('Unique values for each feature: \n\n', housing_train.nunique().sort_values(ascending = False).head(20))

# Creates a correlation matrix among the predictor variables
plt.rcParams['figure.figsize'] = [10, 10]

plt.title('Correlation matrix for numeric features')
correlation_martix = housing_train.corr()
sns.heatmap(correlation_martix, vmax = 1, square = True)
plt.show()

# Impute all remaining missing values with 0s
housing_train.fillna(0, inplace=True)
housing_train.head()

# Create a linear regression model
lr = linear_model.LinearRegression()

# Fit the model to the data, and predict values
start_time = timeit.default_timer()
lr.fit(X, y)
y_pred = lr.predict(X)
elapsed_time = timeit.default_timer() - start_time

# Cross validate our model
lr_cv = cross_val_score(lr, X, y, cv=5, n_jobs=-1)

# Print the results of the model
print('\nLinear Regression score: {:.5f}(+/- {:.2f})\n'.format(lr_cv.mean(), lr_cv.std()*2))
print('Runtime: ', elapsed_time, 'seconds\n')

# Create a random forest model
rfr = ensemble.RandomForestRegressor()

# Fit the model to the data, and predict values
start_time = timeit.default_timer()
rfr.fit(X, y)
y_pred_rfr = rfr.predict(X)
elapsed_time = timeit.default_timer() - start_time

# Cross validate our model
rfr_cv = cross_val_score(rfr, X, y, cv=5, n_jobs=-1)

# Print the results of our cross validation matrix
print('Random Forest Regression score: {:.5f}(+/- {:.2f})\n'.format(rfr_cv.mean(), rfr_cv.std()*2))
print('Runtime: ', elapsed_time, 'seconds\n')

plt.rcParams['figure.figsize'] = [15,5]

# Plot feature importances for random forest model
rfr_features = pd.Series(rfr.feature_importances_, X_train.columns).sort_values(ascending=False).head(15)
rfr_features.plot(kind='bar', title = 'Feature importance')
plt.ylabel('Feature Importance Score')
plt.show()

plt.rcParams['figure.figsize'] = [10,10]

# Comparing predicted results to actual results
ax = sns.scatterplot(x = 'True Values', 
                     y = 'Predicted Values', 
                     data = pd.DataFrame({'True Values': y, 'Predicted Values': y_pred}))\
                    .set_title('Predicted vs Actual Results (normalized)')

# Revert to original values
y = np.e**y
y_pred = np.e**y_pred

plt.rcParams['figure.figsize'] = [10,10]

# Comparing predicted results to actual results
ax = sns.scatterplot(x = 'True Values', 
                     y = 'Predicted Values', 
                     data = pd.DataFrame({'True Values': y, 'Predicted Values': y_pred}))\
                    .set_title('Predicted vs Actual Results (True)')

get_ipython().run_cell_magic('html', '', '<script src="https://cdn.rawgit.com/parente/4c3e6936d0d7a46fd071/raw/65b816fb9bdd3c28b4ddf3af602bfd6015486383/code_toggle.js"></script>')