PREDICTING MEDICAL COSTS WITH REGRESSION MODELS¶

!pip install seaborn
!pip install chart_studio

# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)


from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import Pipeline
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.linear_model import Ridge
from sklearn.model_selection import GridSearchCV
import math  


# Plotly Packages
from plotly import tools
import chart_studio.plotly as py
import plotly.figure_factory as ff
import plotly.graph_objects as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
import plotly.express as px
from plotly.subplots import make_subplots
init_notebook_mode(connected=True)



import seaborn as sns
import matplotlib.pyplot as plt
import missingno as msno


# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session

df = pd.read_csv('/kaggle/input/insurance/insurance.csv')
df.head()

df.info()

df.describe()

# VIEW CORRELATION BETWEEN EACH FEATURE TO THE TARGET VARIABLE (CHARGES)

df.corr()['charges'].sort_values().to_frame()

# CREATE HEATMAP TO VIEW CORRELATIONS

f, ax = plt.subplots(figsize=(10, 8))
corr = df.corr()
sns.heatmap(corr, mask=np.zeros_like(corr, dtype=np.bool), cmap='YlGnBu', square=True, ax=ax)

df['age'] = df['age'].astype('int') #Convert data type of 'age' to int
df['children'] = df['children'].astype('int') #Convert data type of 'children' to int

#Create new feature 'age_cat' to categorize values from 'age' colummn

df['age_cat'] = np.nan 

for col in [df]:
    col.loc[(col['age'] >= 18) & (col['age'] <= 35), 'age_cat'] = 'young adult'
    col.loc[(col['age'] > 35) & (col['age'] <= 55), 'age_cat'] = 'middle age'
    col.loc[col['age'] > 55, 'age_cat'] = 'senior adult'

    
#Create new feature 'weight_status' to categorize values from 'bmi' colummn

df["weight_status"] = np.nan

for col in [df]:
    col.loc[col["bmi"] < 18.5, "weight_status"] = "underweight"
    col.loc[(col["bmi"] >= 18.5) & (col["bmi"] < 24.986), "weight_status"] = "normal weight"
    col.loc[(col["bmi"] >= 25) & (col["bmi"] < 29.926), "weight_status"] = "overweight"
    col.loc[col["bmi"] >= 30, "weight_status"] = "obese"
    
df.head()

# Determine the distribution of charges

charge_dist = df["charges"]
logcharge = np.log(df["charges"])

trace0 = go.Histogram(x=charge_dist,
                      histnorm='probability',
                      name="Charges Distribution",
                      marker = dict(color = '#02919e',)
                     )

trace1 = go.Histogram(x=logcharge,
                      histnorm='probability',
                      name="Charges Distribution using Log",
                      marker = dict(color = '#9e2602')
                     )

fig = make_subplots(rows=2, cols=1,
                    subplot_titles=('Charge Distribution','Log Charge Distribution'),
                    print_grid=False)

fig.append_trace(trace0, 1, 1)
fig.append_trace(trace1, 2, 1)

fig['layout'].update(showlegend=True, title='Charge Distribution', bargap=0.05)

iplot(fig, filename='custom-sized-subplot-with-subplot-titles')

df['sex'].shape

fig = px.histogram(df, x="sex")
fig.show()

fig = px.histogram(df, x="smoker", title='SMOKER DISTRIBUTION')
fig.show()

print('Percentage of smokers:\n ', df.smoker.value_counts() / len(df))

fig = ff.create_distplot([df['age']], ['age'], bin_size=1, colors=['blue'])
fig.update_layout(title_text="Distribution of Age", title_x=0.5)
fig.show()

labels = df['age_cat'].unique()
values = df['age_cat'].value_counts()

fig = go.Figure(data=[go.Pie(labels=labels, 
                             values=values, 
                             hoverinfo='label+percent', 
                             textinfo='value+percent', 
                             textfont=dict(size=14))])
fig.update_layout(title_text="Distribution of Age Categories", title_x=0.5)
fig.show()

#DISTRIBUTION OF BODY MASS INDEX

fig = ff.create_distplot([df['bmi']], ['bmi'], bin_size=1, colors=['#c97318'])
fig.update_layout(title_text="Distribution of BMI", title_x=0.5)
fig.show()

labels = df['weight_status'].unique()
values = df['weight_status'].value_counts()

fig = go.Figure(data=[go.Pie(labels=labels, 
                             values=values, 
                             hoverinfo='label+percent', 
                             textinfo='value+percent', 
                             textfont=dict(size=14))])

fig.update_layout(title_text="Distribution of Weight Status", title_x=0.5)
fig.update_traces(marker=dict(line=dict(color='#000000', width=1)))

fig.show()

fig = px.histogram(df, x="region")
fig.show()

labels = df['children'].sort_values().unique()
values = df['children'].value_counts()

fig = go.Figure(data=[go.Pie(labels=labels, 
                             values=values, 
                             hoverinfo='label+percent', 
                             textinfo='value+percent', 
                             textfont=dict(size=14),
                             pull=[0.00, 0.00, 0.00, 0.00, 0.2, 0.3, 0.1])])

fig.update_layout(title_text="Distribution of Children", title_x=0.5)
fig.show()

s_f = df[(df['smoker'] == 'yes') & (df['sex'] == 'female')]
s_m = df[(df['smoker'] == 'yes') & (df['sex'] == 'male')]

ns_f = df[(df['smoker'] == 'no') & (df['sex'] == 'female')]
ns_m = df[(df['smoker'] == 'no') & (df['sex'] == 'male')]

fig = go.Figure(data=[go.Bar(name='Female', x=df['smoker'].unique(), y=[len(s_f), len(ns_f)]),
                      go.Bar(name='Male', x=df['smoker'].unique(), y=[len(s_m), len(ns_m)]),
                     ]
               )

# Change the bar mode
fig.update_layout(barmode='group', title_text = "Smoker Status by Sex")
fig.show()

labels = df['smoker'].sort_values().unique()
values_f = df['smoker'].loc[df['sex'] == 'female'].value_counts()
values_m = df['smoker'].loc[df['sex'] == 'male'].value_counts()

fig = make_subplots(rows=1, cols=2, specs=[[{'type':'domain'}, {'type':'domain'}]])

fig.add_trace(go.Pie(labels=labels, values=values_f, name='Female Smokers'), 1,1)
fig.add_trace(go.Pie(labels=labels, values=values_m, name='Male Smokers'), 1,2)

fig.update_traces(hole=.4, hoverinfo="label+percent+name", textinfo='value+percent')

fig.update_layout(title_text="Smokers by Sex")
fig.show()

ages = df['age'].loc[df['smoker'] == 'yes']

fig = ff.create_distplot([ages], ['age'], bin_size=1, colors=['#c97318'])
fig.update_layout(title_text="Distribution of smoker ages", title_x=0.5)
fig.show()

smoker_age = df.groupby(['age_cat','smoker'])['smoker'].count().to_frame()
smoker_age.rename(columns={smoker_age.columns[0]: 'count'}, inplace=True)
smoker_age = smoker_age.reset_index()

plt.figure(figsize=(14,8))
sns.barplot(x='age_cat', y='count', hue='smoker', palette='mako', data=smoker_age)

labels = df['smoker'].sort_values().unique()
values_ya = df['smoker'].loc[df['age_cat'] == 'young adult'].value_counts()
values_ma = df['smoker'].loc[df['age_cat'] == 'middle age'].value_counts()
values_sa = df['smoker'].loc[df['age_cat'] == 'senior adult'].value_counts()

fig = make_subplots(rows=1, cols=3, specs=[[{'type':'domain'}, {'type':'domain'}, {'type':'domain'}]],
                   subplot_titles=['Young Adult', 'Middle Age', 'Senior Adult'])

fig.add_trace(go.Pie(labels=labels, values=values_ya, name='Young Adult Smokers'), 1,1)
fig.add_trace(go.Pie(labels=labels, values=values_ma, name='Middle Age Smokers'), 1,2)
fig.add_trace(go.Pie(labels=labels, values=values_sa, name='Senior Adult Smokers'), 1,3)

fig.update_traces(hole=.4, hoverinfo="label+percent+name", textinfo='value+percent')

fig.update_layout(title_text="Smoker Percentages by Age Categories")
fig.show()