In [132]:
import numpy as np
import pandas as pd
import seaborn as sns
import requests
import string
import io
from pandas.io.json import json_normalize
from datascience import *
# These lines do some fancy plotting magic.
import matplotlib
%matplotlib inline
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
from matplotlib.ticker import AutoMinorLocator, MultipleLocator, FuncFormatter
import warnings
warnings.simplefilter('ignore', FutureWarning)
Load URL, read CSV, and display as panda frame¶
In [3]:
moma_artworks_url = "https://media.githubusercontent.com/media/MuseumofModernArt/collection/master/Artworks.csv"
artworks = pd.read_csv(moma_artworks_url)
In [4]:
moma_artists_url = "https://media.githubusercontent.com/media/MuseumofModernArt/collection/master/Artists.csv"
artists = pd.read_csv(moma_artists_url)
Explore data¶
In [5]:
#get column, row to understand size of data
artists.shape
Out[5]:
In [6]:
artists.columns
Out[6]:
In [7]:
artists.info()
In [8]:
#get column, row to understand size of data
artworks.shape
Out[8]:
In [9]:
artworks.columns
Out[9]:
In [10]:
artworks.info()
Create a master dataset (artists and artworks)¶
In [11]:
# rearrange artworks with Constituent ID at first
# use Consitituent ID as unique identifier to join two datasets
artworks = artworks[['ConstituentID', 'Title', 'Artist', 'ArtistBio', 'Nationality',
'BeginDate', 'EndDate', 'Gender', 'Date', 'Medium', 'Dimensions',
'CreditLine', 'AccessionNumber', 'Classification', 'Department',
'DateAcquired', 'Cataloged', 'ObjectID', 'URL', 'ThumbnailURL',
'Circumference (cm)', 'Depth (cm)', 'Diameter (cm)', 'Height (cm)',
'Length (cm)', 'Weight (kg)', 'Width (cm)', 'Seat Height (cm)',
'Duration (sec.)']]
artworks.head()
Out[11]:
In [12]:
# change ConstituentID to string
artists['ConstituentID'] = artists['ConstituentID'].astype(str)
artists.dtypes
Out[12]:
In [13]:
artworks.dtypes
Out[13]:
Join two datasets¶
In [14]:
# change both column names to be the same
artworks.rename(columns={'ConstituentID':'UniqueID'}, inplace = True)
artists.rename(columns={'ConstituentID':'UniqueID'}, inplace = True)
In [15]:
artworks.head(5)
Out[15]:
In [16]:
artists.head(5)
Out[16]:
In [17]:
# join by Unique ID
artists_columns = ['UniqueID', 'DisplayName', 'ArtistBio', 'Nationality', 'Gender', 'BeginDate','EndDate']
master = pd.merge(artworks, artists, on='UniqueID', how='outer')
master.shape
Out[17]:
In [18]:
# sanity check on the problem of duplicates by creating another master dataframe
# whereby total count of each UniqueID in artists vs artworks is checked first
# Dedup column is the total count of
# GroupBy.cumcount(self, ascending: bool = True)
# Number each item in each group from 0 to the length of that group - 1.
artworks['Dedup'] = artworks.groupby('UniqueID').cumcount(ascending=False)
artists['Dedup'] = artists.groupby('UniqueID').cumcount(ascending=False)
draft = pd.merge(artworks, artists, on=['UniqueID', 'Dedup'], how='outer')
draft.shape
Out[18]:
In [19]:
draft.head(10)
Out[19]:
In [20]:
master.head(10)
Out[20]:
In [21]:
# rearrange columns of master
master.columns
master = master[['UniqueID', 'Title', 'Artist', 'ArtistBio_x', 'ArtistBio_y', 'Gender_x', 'Gender_y', 'Nationality_x', 'Nationality_y', 'BeginDate_x',
'EndDate_x', 'BeginDate_y', 'EndDate_y', 'DisplayName', 'Date', 'Medium', 'Dimensions', 'CreditLine',
'AccessionNumber', 'Classification', 'Department', 'DateAcquired',
'Cataloged', 'ObjectID', 'URL', 'ThumbnailURL', 'Circumference (cm)',
'Depth (cm)', 'Diameter (cm)', 'Height (cm)', 'Length (cm)',
'Weight (kg)', 'Width (cm)', 'Seat Height (cm)', 'Duration (sec.)',
'Wiki QID', 'ULAN']]
master.head(10)
Out[21]:
Data Cleaning: drop nulls, deduplication, data types, unique counts, and classification¶
Make a copy of master and remove duplicated columns¶
In [22]:
# make copy
master_copy = master
master_copy.head(5)
Out[22]:
In [23]:
# remove duplicated columns "_x" and keep "_y"
master_copy = master_copy.drop(['ArtistBio_x', 'ArtistBio_y', 'Gender_x', 'Nationality_x', 'BeginDate_x', 'EndDate_x', 'DisplayName'], axis=1)
In [24]:
# rename columns for readability
master_copy.rename(columns={'Nationality_y':'Nationality', 'Gender_y':'Gender',
'BeginDate_y': 'Birth', 'EndDate_y': 'Death'}, inplace = True)
Time¶
In [25]:
# time function - to standardize date, begin/end date, acquisition date
#https://www.dataquest.io/blog/data-cleaning-with-python/
# Pattern 1: “1976-77” (year ranges)
# Pattern 2: “c. 1917”
# Pattern 3: “Unknown”
# Pattern 4: “n.d.”
master_copy['Date'].value_counts(dropna=False).head(1000)
Out[25]:
In [26]:
def strip_punctuation(row):
# turn each start date into string
start_date = str(row['Date'])
# remove any whitespace outside of the start up
start_date = start_date.strip()
# replace inside whitespace with punctuation
start_date = start_date.replace(" ", "?")
# separate start date into 2 elemtns if "-" is found
final_date = start_date.translate(str.maketrans({a: None for a in string.punctuation}))
return final_date
# assign the results of " strip_punctuation" to the 'Date' column
# for Pandas to go row-wise so we set "axis = 1"
# for Pandas to go column-wise so we set "axis = 0"
master_copy['Date'] = master_copy.apply(lambda row: strip_punctuation(row), axis=1)
master_copy['Date'].value_counts()
Out[26]:
In [27]:
# for some reason the above function does not remove '-'
def split_dates(row):
# start date contains the current value for the Date column
start_date = str(row['Date'])
# split start_date into two elements if "-" is found
# remove - again
split_date = start_date.split('-')
# if a "-" is found, split_date will contain a list with at least two items
# else not found, split_date will have 1 item, the initial_date
# use final_date from previous function's list
if len(split_date) >1:
final_date = split_date[0]
else:
final_date = start_date
return final_date
master_copy['Date'] = master_copy.apply(lambda row: split_dates(row), axis=1)
master_copy['Date'].value_counts()
Out[27]:
In [28]:
# delete rows with letters
master_copy = master_copy[~master_copy['Date'].str.contains("[a-zA-Z]").fillna(False)]
In [29]:
master_copy['Date'].value_counts()
Out[29]:
In [30]:
master_copy['Date'].dtypes
Out[30]:
In [31]:
# drop dates with len() < 4 or > 4
# year is 4 digit
def drop_dates(row):
# start date contains the current value for the Date column
start_date = row['Date']
# if larger than 4 digits, then drop above four digits
# if less than 4 digits, then keep the 4 digit format
# anything else, keep the same 4 digit format
if len(start_date) > 4:
final_date = np.NaN
elif len(start_date) < 4:
final_date = np.NaN
else:
final_date = start_date
return final_date
In [32]:
# assign the results of " drop_dates" to the 'Date' column
# for Pandas to go row-wise so we set "axis = 1"
# for Pandas to go column-wise so we set "axis = 0"
master_copy['Date'] = master_copy.apply(lambda row: drop_dates(row), axis=1)
master_copy['Date'].value_counts()
Out[32]:
In [33]:
# delete those two rows where the 'Date' is an error = 4271
master_copy = master_copy[master_copy.Date != '4271']
In [34]:
master_copy['Date'].sort_values()
Out[34]:
In [35]:
master_copy['Date'].dropna()
Out[35]:
In [36]:
# convert back to float
master_copy['Date'] = master_copy['Date'].astype(float)
master_copy['Date'].dtype
Out[36]:
In [37]:
master_copy['DateAcquired'].value_counts(dropna=False).head(20)
# convert to pandas
Out[37]:
In [38]:
master_copy['DateAcquired'] = master_copy['DateAcquired'].astype(str)
In [39]:
master_copy['DateAcquired'] = pd.to_datetime(master_copy['DateAcquired'], infer_datetime_format=True, errors = 'coerce')
In [40]:
master_copy['DateAcquired_Year'] = master_copy['DateAcquired'].dt.year
In [41]:
master_copy['DateAcquired_Month'] = master_copy['DateAcquired'].dt.month
In [42]:
master_copy.dtypes
Out[42]:
In [43]:
master_copy['DateAcquired_Year'].sort_values()
Out[43]:
In [44]:
master_copy['Date'] = master_copy['Date'].astype(float)
Birth¶
In [45]:
master_copy['Birth'].value_counts(ascending=False)
Out[45]:
In [46]:
master_copy['Birth'].value_counts(dropna=False).head(1000)
Out[46]:
In [47]:
master_copy['Birth'] = master_copy['Birth'].astype(str)
master_copy = master_copy[~master_copy['Birth'].str.contains("[a-zA-Z]").fillna(False)]
In [48]:
master_copy['Birth'].value_counts(ascending=False)
Out[48]:
In [49]:
master_copy['Birth'].dropna()
Out[49]:
In [50]:
master_copy['Birth'].value_counts(ascending=False)
Out[50]:
In [51]:
# convert back to float
master_copy['Birth'] = master_copy['Birth'].astype(float)
master_copy['Birth'].dtype
Out[51]:
Death¶
In [52]:
master_copy['Death'].value_counts(ascending=False)
Out[52]:
In [53]:
master_copy['Death'].value_counts(dropna=False).head(1000)
Out[53]:
In [54]:
master_copy['Death'] = master_copy['Death'].astype(str)
master_copy = master_copy[~master_copy['Death'].str.contains("[a-zA-Z]").fillna(False)]
In [55]:
master_copy['Death'].value_counts(ascending=False)
Out[55]:
In [56]:
master_copy['Death'].dropna()
Out[56]:
In [57]:
# convert back to float
master_copy['Death'] = master_copy['Death'].astype(float)
master_copy['Death'].dtype
Out[57]:
In [58]:
master_copy['Death'].value_counts(ascending=False)
Out[58]:
In [59]:
master_copy['Date'].value_counts(ascending=False)
Out[59]:
Age¶
In [60]:
master_copy['AcquiredAge_Artwork'] = master_copy['DateAcquired_Year'] - master_copy['Date']
In [61]:
master_copy['AcquiredAge_Artwork'] = master_copy['AcquiredAge_Artwork'].astype(str)
master_copy = master_copy[~master_copy['AcquiredAge_Artwork'].str.contains("-").fillna(False)]
In [62]:
master_copy['AcquiredAge_Artwork'] = master_copy['AcquiredAge_Artwork'].astype(float)
master_copy['AcquiredAge_Artwork'].value_counts(dropna=False).head(1000)
Out[62]:
In [63]:
master_copy['AcquiredAge_Artwork'].describe()
Out[63]:
In [64]:
def drop_acquired_values(row):
# start date contains the current value for the Date column
start_value = row['AcquiredAge_Artwork']
# if larger than 4 digits, then drop above four digits
# if less than 4 digits, then keep the 4 digit format
# anything else, keep the same 4 digit format
if start_value >= 205:
final_value = np.NaN
elif start_value <= -205:
final_value = np.NaN
else:
final_value = start_value
return final_value
In [65]:
master_copy['AcquiredAge_Artwork_2'] = master_copy.apply(lambda row: drop_acquired_values(row), axis=1)
master_copy['AcquiredAge_Artwork_2'].value_counts()
Out[65]:
In [66]:
master_copy['AcquiredAge_Artwork_2'].describe()
Out[66]:
In [67]:
master_copy['AcquiredAge_Artwork_2']
Out[67]:
Age: First Piece of Work Only Per Artist (_2)¶
AcquiredAge_Artwork_2¶
In [68]:
group = master_copy.groupby('Artist')
In [69]:
master_copy_3 = group.apply(lambda x: x['AcquiredAge_Artwork_2'].unique())
In [70]:
master_copy_3 = master_copy_3.apply(pd.Series)
In [71]:
minValues_AcquiredAge_Artwork_2 = master_copy_3.min(axis=1)
print('minimum value in each row : ')
print(minValues_AcquiredAge_Artwork_2)
In [72]:
master_copy_3['minValues_AcquiredAge_Artwork_2']=minValues_AcquiredAge_Artwork_2
In [73]:
master_copy_3['minValues_AcquiredAge_Artwork_2'].describe()
Out[73]:
In [74]:
master_copy_3.head()
Out[74]:
In [75]:
master_copy.merge(master_copy_3, left_on='Artist', right_on='Artist', how='inner')
Out[75]:
In [76]:
master_copy = master_copy.merge(master_copy_3, left_on='Artist', right_on='Artist', how='inner')
In [77]:
# change column name
master_copy.rename(columns={'minValues_AcquiredAge_Artwork_2_y':'MinAcquiredAge_Artwork'}, inplace = True)
In [78]:
master_copy
Out[78]:
In [79]:
plt.figure(figsize=(20,20))
#plt.style.use('seaborn-colorblind')
acquired_age_artwork_bins_2 = pd.cut(master_copy_3['minValues_AcquiredAge_Artwork_2'], 10)
chart_acquiredage_artwork_2 = sns.countplot(
data=master_copy,
x=acquired_age_artwork_bins_2,
)
# set title and subtitle
chart_acquiredage_artwork_2.text(x=-0.3, y=7700, s="MoMA's Acquistions Trends by Age of Artworks", fontweight='bold', fontsize='48')
chart_acquiredage_artwork_2.text(x=-0.3, y=7400, s="First Piece of Work Per Artist", fontweight='bold', fontsize='48')
chart_acquiredage_artwork_2.text(x=-0.3, y=7140, s="Number of artworks by age of artworks", fontsize='36')
# set axis labels
plt.ylabel(ylabel='Number of Artworks', fontsize=24, fontweight='bold')
plt.xlabel(xlabel='Age Group of Artworks', fontsize=24, fontweight='bold', labelpad=15)
plt.xticks(rotation=45, horizontalalignment='right', fontweight='medium', fontsize='24'
)
plt.yticks(
rotation=45,
verticalalignment='top',
fontweight='medium',
fontsize='24'
)
Out[79]:
The Bootstrap¶
Population¶
In [80]:
minValues_AcquiredAge_Artwork_2 = minValues_AcquiredAge_Artwork_2.dropna()
population = minValues_AcquiredAge_Artwork_2.to_frame().reset_index()
population.head(5)
#population.columns
population.rename(columns={0:'PopulationAge'}, inplace = True)
population
Out[80]:
In [81]:
population_observed_mean = np.average(population['PopulationAge'])
population_observed_mean
Out[81]:
In [82]:
population.describe()
Out[82]:
In [83]:
#Histogram of your population
plt.figure(figsize=(20,20))
population.hist(bins=np.arange(0, 186, 13))
# set title and subtitle
plt.title(x=0.5, y=1.1,label="MoMA's Acquistions Trends by Age of Artworks (Population)", fontweight='bold', fontsize='24')
plt.ylabel(ylabel='Number of Artworks', fontsize=20, fontweight='medium')
plt.xlabel(xlabel='Minimum Acquired Age of First Work Per Artist', fontsize=20, fontweight='medium', labelpad=15)
plt.xticks(rotation=45, horizontalalignment='right', fontweight='medium', fontsize='16'
)
plt.yticks(
rotation=45,
verticalalignment='top',
fontweight='medium',
fontsize='16'
)
Out[83]:
In [84]:
population['PopulationAge'] = population['PopulationAge'].astype(int)
population['Artist'] = population['Artist'].astype(str)
Standard error the mean given sample size = 9469¶
It can be seen from the formula that the standard error of the mean decreases as N increases. This is expected because if the mean at each step is calculated using many data points, then a small deviation in one value will cause less effect on the final mean.¶
The standard error of the mean tells us how the mean varies with different experiments measuring the same quantity. Thus if the effect of random changes are significant, then the standard error of the mean will be higher. If there is no change in the data points as experiments are repeated, then the standard error of mean is zero.¶
In [85]:
from math import sqrt
# standard error of the mean
# standard deviation of the observed / sqrt(sample size)
np.std(population['PopulationAge']) / (sqrt(9469))
Out[85]:
Simulate SAME random sample size of 9469 times with Replacement¶
Simulated average of acquired age of artwork¶
In [86]:
avg = []
n = 9469
simulations = 9469
for i in np.arange(simulations):
sample = np.random.choice(minValues_AcquiredAge_Artwork_2, n, replace=True)
avg = np.append(avg,np.average(sample))
avg
Out[86]:
In [87]:
sample_avg_pd = pd.DataFrame(avg)
In [88]:
sample_avg_pd.describe()
Out[88]:
In [89]:
sum(avg)
Out[89]:
In [90]:
sample_avg = np.average(avg)
sample_avg
Out[90]:
In [91]:
sample_avg_pd.describe()
Out[91]:
In [92]:
#Histogram of your sampled means
plt.figure(figsize=(20,20))
sample_avg_pd.hist(bins=np.arange(15, 18, .1))
# set title and subtitle
plt.title(x=0.5, y=1.1,label="MoMA's Acquistions Trends by Average Age of Artworks (Mean)", fontweight='bold', fontsize='24')
plt.ylabel(ylabel='Number of Artworks', fontsize=20, fontweight='medium')
plt.xlabel(xlabel='Average Acquired Age of First Work Per Artist', fontsize=20, fontweight='medium', labelpad=15)
plt.xticks(rotation=45, horizontalalignment='right', fontweight='medium', fontsize='16'
)
plt.yticks(
rotation=45,
verticalalignment='top',
fontweight='medium',
fontsize='16'
)
Out[92]:
Deviation of the simulated avg from the observed mean¶
In [93]:
avg2 = []
n = 9469
simulations = 10000
for i in np.arange(simulations):
sample2 = np.random.choice(minValues_AcquiredAge_Artwork_2, n, replace=True)
avg2 = np.append(avg2,population_observed_mean-np.average(sample2))
avg2
Out[93]:
In [94]:
avg2_pd = pd.DataFrame(avg2)
In [95]:
avg2_pd.describe()
Out[95]:
In [96]:
#Histogram of difference btwn simulated vs population avg
plt.figure(figsize=(20,20))
avg2_pd.hist(bins=np.arange(-1, 1, .1))
# set title and subtitle
plt.title(x=0.5, y=1.1,label="Distribution of Difference between Simulated Values vs Population Average Age of Artworks (Mean)", fontweight='bold', fontsize='24')
plt.ylabel(ylabel='Number of Artworks', fontsize=20, fontweight='medium')
plt.xlabel(xlabel='Difference of Average Acquired Age of First Work Per Artist', fontsize=20, fontweight='medium', labelpad=15)
plt.xticks(rotation=45, horizontalalignment='right', fontweight='medium', fontsize='16'
)
plt.yticks(
rotation=45,
verticalalignment='top',
fontweight='medium',
fontsize='16'
)
Out[96]:
In [97]:
# calculate the z-score
# using the z-score because sample size is larger than 30
# and know standard deviation of population
# difference of sampled mean from population mean / (standard deviation of population/sqrt(sample size))
z_score = (avg2_pd) /((np.std(population['PopulationAge']))/sqrt(n))
z_score
Out[97]:
In [98]:
#Histogram of your sample minimums
plt.figure(figsize=(20,20))
z_score.hist()
# set title and subtitle
plt.title(x=0.5, y=1.1,label="Distribution of Z-Score (Standard Error of the Mean)", fontweight='bold', fontsize='24')
plt.ylabel(ylabel='Number of Artworks', fontsize=20, fontweight='medium')
plt.xlabel(xlabel='Difference of Average Acquired Age of First Work Per Artist', fontsize=20, fontweight='medium', labelpad=15)
plt.xticks(rotation=45, horizontalalignment='right', fontweight='medium', fontsize='16'
)
plt.yticks(
rotation=45,
verticalalignment='top',
fontweight='medium',
fontsize='16'
)
Out[98]:
When you have multiple samples and want to describe the standard deviation of those sample means (the standard error), you use this z-score.¶
This z-score will tell you how many standard errors there are between the sample mean and the population mean.¶
z = (x – μ) / (σ / √n)¶
Since the z-score is on a normal distribution curve, we can use it to compute the probability of finding each sampled mean or the average acquired age. The z-score shows the sampling distribution of means as the standard error. For example, 99% vs 95% vs 68%/ of values fall within 3 vs 2 vs 1 standard deviations from the mean in a normal probability distribution.¶
In [99]:
diff_lower_bound = np.percentile(avg,2.5)
diff_upper_bound = np.percentile(avg,97.5)
print("Bootstrapped 95% confidence interval for Average Acquired Age of First Work Per Artist[{:f}, {:f}]".format(diff_lower_bound, diff_upper_bound))
In [100]:
p_value = np.count_nonzero(avg < population_observed_mean)/10000
p_value
Out[100]:
Simulate minimum 10k times, same sample size with replacement¶
In [101]:
minimum = []
for i in np.arange(10000):
minimum = np.random.choice(minValues_AcquiredAge_Artwork_2, size = 9468, replace=True)
minimum = np.append(minimum,min(minimum))
minimum
Out[101]:
In [102]:
representative_sample_3_mean = np.average(minimum)
representative_sample_3_mean
Out[102]:
In [103]:
minimum_pd = pd.DataFrame(minimum)
In [104]:
minimum_pd.describe()
Out[104]:
In [105]:
#Histogram of your sample minimums
plt.figure(figsize=(20,20))
minimum_pd.hist(bins=np.arange(0, 186, 13))
# set title and subtitle
plt.title(x=0.5, y=1.1,label="MoMA's Acquistions Trends by Average Age of Artworks (Minimum)", fontweight='bold', fontsize='24')
plt.ylabel(ylabel='Number of Artworks', fontsize=20, fontweight='medium')
plt.xlabel(xlabel='Minimum Acquired Age of First Work Per Artist', fontsize=20, fontweight='medium', labelpad=15)
plt.xticks(rotation=45, horizontalalignment='right', fontweight='medium', fontsize='16'
)
plt.yticks(
rotation=45,
verticalalignment='top',
fontweight='medium',
fontsize='16'
)
Out[105]:
Using bootstrap is not a great estimation method for estimating the acquired age of artworks because:¶
- there are rare elements around the min and max value of the population¶
- majority of count of artworks are in the range of [0,20] years of age yet there is a long tail of data in the range of dataset is [20,186]¶
- probability distribution of the statistic is not roughly bell shaped (the shape of the empirical distribution will be a clue)¶
The Hypothesis Test¶
Nationality¶
In [106]:
# nationality
master_copy['Nationality'].value_counts(dropna=False).head(500)
Out[106]:
In [107]:
master_copy['Nationality'].describe()
Out[107]:
In [108]:
master_copy['Nationality_Am'] = master_copy['Nationality'].str.contains('Amer' or 'American')
master_copy['Nationality_Am']
Out[108]:
In [109]:
master_copy['Nationality_Am'] = master_copy['Nationality_Am'].replace({True:'American', False:'International'})
In [110]:
master_copy['Nationality_Am']
Out[110]:
In [111]:
master_copy['Nationality_Am'].value_counts(dropna=True).head(20)
Out[111]:
In [112]:
master_copy['Nationality_Am'].describe()
Out[112]:
In [113]:
Percentage_American = str((46881/103479)*100)
Percentage_International = str(100-((46881/103479)*100))
print('Percentage of American Artist in MoMA Collection :' + (Percentage_American))
print('Percentage of International Artist in MoMA Collection :' + (Percentage_International))
In [114]:
master_copy.head(5)
Out[114]:
Simulate null hypothesis 50%-50% between American vs International Artists¶
In [115]:
def simulate():
share = [0.5, 0.5]
return sample_proportions (1000,share)
simulate()
Out[115]:
In [116]:
test_statistics_under_null_am = make_array()
repetitions = 10000
for i in np.arange(repetitions):
test_statistics_under_null_am = np.append(test_statistics_under_null_am, simulate().item(0))
test_statistics_under_null_am
Out[116]:
In [117]:
Table().with_column('Share of American artists in 1000 artworks', test_statistics_under_null_am).hist(bins=np.arange(0.4,0.6,0.01))
In [118]:
p_value = np.count_nonzero(test_statistics_under_null_am < .453)/ len(test_statistics_under_null_am)
p_value
# Null: MoMA is evenly made up of American and international artists.
# Alternative: MoMA is NOT evenly made up of American and international artists.
# P-value is the chance that under the null hypotheis,
# that the test statistic is equal to the value observed in the data. A large p-value (> 0.05)
# indicates weak evidence against the null hypothesis, so you fail to reject the null hypothesis.
# A smaller p-value (< 0.05) indicates strong evidence against the null hypothesis, so you reject
# the null hypothesis and accept the alternative.
Out[118]:
In [119]:
diff_lower_bound = percentile(2.5, test_statistics_under_null_am)
diff_upper_bound = percentile(97.5, test_statistics_under_null_am)
print("Bootstrapped 95% confidence interval for Share of American artists in 1000 artworks: [{:f}, {:f}]".format(diff_lower_bound, diff_upper_bound))
It turns out the p-value at 0.0013 is much smaller than the cutoff of 0.05. It indicates strong evidence against the null hypothesis and thus, rejection of the null hypothesis. Thus, I reject the null hypothesis that American and international artists have equal representation in MoMA’s art collection. I accept the alternative hypothesis which is the proportion of American and international artists are not evenly split 50-50.¶
In [120]:
test_statistics_under_null_in = make_array()
repetitions = 10000
for i in np.arange(repetitions):
test_statistics_under_null_in = np.append(test_statistics_under_null_in, simulate().item(1))
test_statistics_under_null_in
Out[120]:
In [121]:
Table().with_column(
'Number of International artists in 1000 artworks', test_statistics_under_null_in).hist(bins=np.arange(.4,.6,.01))
In [122]:
def difference():
difference = test_statistics_under_null_in - test_statistics_under_null_am
return difference
sampled_difference = difference()
In [123]:
diff_lower_bound = percentile(2.5, sampled_difference)
diff_upper_bound = percentile(97.5, sampled_difference)
print("Bootstrapped 95% confidence interval for size of International artist's lead over American artists: [{:f}, {:f}]".format(diff_lower_bound, diff_upper_bound))
In [124]:
Table().with_column(
'Size of International artists lead over American artists', sampled_difference).hist(bins=np.arange(-0.1,.1,0.01))
In [125]:
probability_of_in_lead = (10000-sum(test_statistics_under_null_in))/10000
In [126]:
probability_of_in_lead
Out[126]:
In [127]:
probability_of_am_lead = (10000-sum(test_statistics_under_null_am))/10000
In [128]:
probability_of_am_lead
Out[128]:
In [129]:
master_copy['Date'] = master_copy['Date'].astype(float)
plt.style.use('fivethirtyeight')
fig = plt.figure()
ax = sns.scatterplot(x="Date", y="AcquiredAge_Artwork_2", hue="Nationality_Am", data=master_copy)
# move legend
ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2),
fancybox=True, shadow=True, ncol=5)
# set title and subtitle
ax.text(x=1728, y=235, s="How American is MoMA?", fontweight='bold', fontsize='18')
ax.text(x=1728, y=220, s="Age of artworks when acquired vs. year it was created, by nationality", fontsize='14')
# set axis labels
plt.ylabel(ylabel='Age of Artworks when Acquired', fontsize=12, fontweight='bold')
plt.xlabel(xlabel='Year Created', fontsize=12, fontweight='bold', labelpad=15)
plt.show()
In [130]:
plt.style.use('fivethirtyeight')
fig = plt.figure()
ax_dept_am = sns.scatterplot(x="Date", y="DateAcquired_Year", hue="Nationality_Am", data=master_copy)
# move legend
ax_dept_am.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2),
fancybox=True, shadow=True, ncol=5)
# set title and subtitle
ax_dept_am.text(x=1728, y=2035, s="How American is MoMA?", fontweight='bold', fontsize='18')
ax_dept_am.text(x=1728, y=2028, s="Year of a work's acquisition vs. year it was painted, by artist nationality", fontsize='14')
# set axis labels
plt.ylabel(ylabel='Year Acquired', fontsize=12, fontweight='bold')
plt.xlabel(xlabel='Year Created', fontsize=12, fontweight='bold', labelpad=15)
plt.show()