read the hacker_news.csv
file in as a list of lists
from csv import reader
opened_file = open("hacker_news.csv")
read_file = reader(opened_file)
hn_original = list(read_file)
display the first 5 rows
print(hn_original[:5])
[['id', 'title', 'url', 'num_points', 'num_comments', 'author', 'created_at'], ['12224879', 'Interactive Dynamic Video', 'http://www.interactivedynamicvideo.com/', '386', '52', 'ne0phyte', '8/4/2016 11:52'], ['10975351', 'How to Use Open Source and Shut the Fuck Up at the Same Time', 'http://hueniverse.com/2016/01/26/how-to-use-open-source-and-shut-the-fuck-up-at-the-same-time/', '39', '10', 'josep2', '1/26/2016 19:30'], ['11964716', "Florida DJs May Face Felony for April Fools' Water Joke", 'http://www.thewire.com/entertainment/2013/04/florida-djs-april-fools-water-joke/63798/', '2', '1', 'vezycash', '6/23/2016 22:20'], ['11919867', 'Technology ventures: From Idea to Enterprise', 'https://www.amazon.com/Technology-Ventures-Enterprise-Thomas-Byers/dp/0073523429', '3', '1', 'hswarna', '6/17/2016 0:01']]
In order to analyze our data, we need to remove column header below.
headers = hn_original[0]
print(headers)
hn = hn_original[1:]
print('')
print(hn[:5])
['id', 'title', 'url', 'num_points', 'num_comments', 'author', 'created_at'] [['12224879', 'Interactive Dynamic Video', 'http://www.interactivedynamicvideo.com/', '386', '52', 'ne0phyte', '8/4/2016 11:52'], ['10975351', 'How to Use Open Source and Shut the Fuck Up at the Same Time', 'http://hueniverse.com/2016/01/26/how-to-use-open-source-and-shut-the-fuck-up-at-the-same-time/', '39', '10', 'josep2', '1/26/2016 19:30'], ['11964716', "Florida DJs May Face Felony for April Fools' Water Joke", 'http://www.thewire.com/entertainment/2013/04/florida-djs-april-fools-water-joke/63798/', '2', '1', 'vezycash', '6/23/2016 22:20'], ['11919867', 'Technology ventures: From Idea to Enterprise', 'https://www.amazon.com/Technology-Ventures-Enterprise-Thomas-Byers/dp/0073523429', '3', '1', 'hswarna', '6/17/2016 0:01'], ['10301696', 'Note by Note: The Making of Steinway L1037 (2007)', 'http://www.nytimes.com/2007/11/07/movies/07stein.html?_r=0', '8', '2', 'walterbell', '9/30/2015 4:12']]
# print('I love Fay'.startswith('I love'))
# True
use 'startswith' method to separate posts beginning with '''Ask HN''' and '''Show HN'''
create 3 empty lists below
ask_posts = []
show_posts = []
other_posts = []
for row in hn:
title = row[1]
# if lowercase of title starts with "ask hn", append to ask_posts
if title.lower().startswith('ask hn'):
ask_posts.append(row)
# if lowercase of title starts with "show hn", append to show_posts
elif title.lower().startswith('show hn'):
show_posts.append(row)
# else situations, append to other_posts
else:
other_posts.append(row)
#to check number of posts
print('Number of posts starts with ask hn: ', len(ask_posts))
print('Number of posts starts with show hn: ', len(show_posts))
print('Number of other posts: ', len(other_posts))
Number of posts starts with ask hn: 1744 Number of posts starts with show hn: 1162 Number of other posts: 17194
to see the first 5 rows in the list of lists
print(ask_posts[:5])
print()
print(show_posts[:5])
[['12296411', 'Ask HN: How to improve my personal website?', '', '2', '6', 'ahmedbaracat', '8/16/2016 9:55'], ['10610020', 'Ask HN: Am I the only one outraged by Twitter shutting down share counts?', '', '28', '29', 'tkfx', '11/22/2015 13:43'], ['11610310', 'Ask HN: Aby recent changes to CSS that broke mobile?', '', '1', '1', 'polskibus', '5/2/2016 10:14'], ['12210105', 'Ask HN: Looking for Employee #3 How do I do it?', '', '1', '3', 'sph130', '8/2/2016 14:20'], ['10394168', 'Ask HN: Someone offered to buy my browser extension from me. What now?', '', '28', '17', 'roykolak', '10/15/2015 16:38']] [['10627194', 'Show HN: Wio Link ESP8266 Based Web of Things Hardware Development Platform', 'https://iot.seeed.cc', '26', '22', 'kfihihc', '11/25/2015 14:03'], ['10646440', 'Show HN: Something pointless I made', 'http://dn.ht/picklecat/', '747', '102', 'dhotson', '11/29/2015 22:46'], ['11590768', 'Show HN: Shanhu.io, a programming playground powered by e8vm', 'https://shanhu.io', '1', '1', 'h8liu', '4/28/2016 18:05'], ['12178806', 'Show HN: Webscope Easy way for web developers to communicate with Clients', 'http://webscopeapp.com', '3', '3', 'fastbrick', '7/28/2016 7:11'], ['10872799', 'Show HN: GeoScreenshot Easily test Geo-IP based web pages', 'https://www.geoscreenshot.com/', '1', '9', 'kpsychwave', '1/9/2016 20:45']]
# find total number of comments in ask_posts
total_ask_comments = 0
#iterate over the ask_posts to calculate total amounts
for row in ask_posts:
#num_comments is the fifth column in ask_posts
num_comments = int(row[4])
total_ask_comments += num_comments
print('total ask comments:', total_ask_comments)
#compute the average number of comments
avg_ask_comments = total_ask_comments / len(ask_posts)
print('average ask comments:', avg_ask_comments)
total ask comments: 24483 average ask comments: 14.038417431192661
#find total number of comments in show_posts
total_show_comments = 0
#iterate over the show posts to calculate total amounts
for row in show_posts:
num_comments = int(row[4])
total_show_comments += num_comments
print('total show comments:', total_show_comments)
#compute the average number of comments
avg_show_comments = total_show_comments / len(show_posts)
print('average show comments:', avg_show_comments)
total show comments: 11988 average show comments: 10.31669535283993
Total ask posts | Total show posts | avg. ask comments | avg. show comments |
---|---|---|---|
24483 | 11988 | 14.03 | 10.31 |
Total ask posts are 13000 more than total show posts. Besides, since ask posts receives more comments than show posts, we'll focus our remaining analysis on these posts.
Next, we'll analyze if "creating ask posts at certain time are more likely to attract more comments"
import datetime as dt
result_list = []
for row in ask_posts:
time = row[6]
comment = int(row[4])
#use "list.append([column1, column2])" to append two columns
result_list.append([time, comment])
counts_by_hour = {}
comments_by_hour = {}
date_format = "%m/%d/%Y %H:%M"
for row in result_list:
hour = row[0]
comment = row[1]
hr = dt.datetime.strptime(hour, date_format).strftime("%H")
if hr not in counts_by_hour:
counts_by_hour[hr] = 1
comments_by_hour[hr] = comment
else:
counts_by_hour[hr] += 1
comments_by_hour[hr] += comment
print(counts_by_hour)
print(comments_by_hour)
{'20': 80, '01': 60, '13': 85, '15': 116, '02': 58, '06': 44, '05': 46, '04': 47, '14': 107, '12': 73, '22': 71, '10': 59, '09': 45, '08': 48, '21': 109, '16': 108, '17': 100, '11': 58, '07': 34, '03': 54, '18': 109, '19': 110, '00': 55, '23': 68} {'20': 1722, '01': 683, '13': 1253, '15': 4477, '02': 1381, '06': 397, '05': 464, '04': 337, '14': 1416, '12': 687, '22': 479, '10': 793, '09': 251, '08': 492, '21': 1745, '16': 1814, '17': 1146, '11': 641, '07': 267, '03': 421, '18': 1439, '19': 1188, '00': 447, '23': 543}
Next, we'll use two dictionaries to calculate the average number of comments for posts created during each hour of the day.
### to calculate average comments per post during each hour
### 要統計每小時的貼文平均回覆數 ex: 早上10點 793則回覆/59篇貼文=13.44
result_list = []
#append the data you need: hour, number of comments, posts number
for row in ask_posts:
comment_num = int(row[4])
create_time = row[6]
result_list.append([comment_num, create_time])
#number of posts per hour
counts_by_hour = {}
#number of comment per hour
posts_by_hour = {}
for row in result_list:
comment = row[0] # already in int() type at previous step
hour = row[1]
hour = dt.datetime.strptime(hour,"%m/%d/%Y %H:%M")
hour = hour.strftime("%H")
if hour not in counts_by_hour:
counts_by_hour[hour] = 1
comments_by_hour[hour] = 1
else:
counts_by_hour[hour] += 1
comments_by_hour[hour] += comment
print('Counts by hour',counts_by_hour)
print()
print('Comments by hour', comments_by_hour)
Counts by hour {'20': 80, '01': 60, '13': 85, '15': 116, '02': 58, '06': 44, '05': 46, '04': 47, '14': 107, '12': 73, '22': 71, '10': 59, '09': 45, '08': 48, '21': 109, '16': 108, '17': 100, '11': 58, '07': 34, '03': 54, '18': 109, '19': 110, '00': 55, '23': 68} Comments by hour {'20': 1721, '01': 651, '13': 1225, '15': 4477, '02': 1379, '06': 397, '05': 436, '04': 335, '14': 1414, '12': 684, '22': 478, '10': 793, '09': 246, '08': 488, '21': 1742, '16': 1798, '17': 1146, '11': 640, '07': 266, '03': 421, '18': 1438, '19': 1186, '00': 438, '23': 543}
avg_by_hour = []
for comment in comments_by_hour:
for count in counts_by_hour:
if comment == count:
avg_by_hour.append([comment, (comments_by_hour[comment])/(counts_by_hour[count])])
print(avg_by_hour)
[['20', 21.5125], ['01', 10.85], ['13', 14.411764705882353], ['15', 38.5948275862069], ['02', 23.775862068965516], ['06', 9.022727272727273], ['05', 9.478260869565217], ['04', 7.127659574468085], ['14', 13.214953271028037], ['12', 9.36986301369863], ['22', 6.732394366197183], ['10', 13.440677966101696], ['09', 5.466666666666667], ['08', 10.166666666666666], ['21', 15.98165137614679], ['16', 16.64814814814815], ['17', 11.46], ['11', 11.03448275862069], ['07', 7.823529411764706], ['03', 7.796296296296297], ['18', 13.192660550458715], ['19', 10.781818181818181], ['00', 7.963636363636364], ['23', 7.985294117647059]]
The result is kind of hard to identify the hours with the highest values.
swap_avg_by_hour = []
for swap in avg_by_hour:
swap_avg_by_hour.append([swap[1], swap[0]])
# print(swap_avg_by_hour)
# use sorted() function to sort
from operator import itemgetter
sorted_swap = sorted(swap_avg_by_hour, key = itemgetter(0), reverse = True)
print(sorted_swap)
[[38.5948275862069, '15'], [23.775862068965516, '02'], [21.5125, '20'], [16.64814814814815, '16'], [15.98165137614679, '21'], [14.411764705882353, '13'], [13.440677966101696, '10'], [13.214953271028037, '14'], [13.192660550458715, '18'], [11.46, '17'], [11.03448275862069, '11'], [10.85, '01'], [10.781818181818181, '19'], [10.166666666666666, '08'], [9.478260869565217, '05'], [9.36986301369863, '12'], [9.022727272727273, '06'], [7.985294117647059, '23'], [7.963636363636364, '00'], [7.823529411764706, '07'], [7.796296296296297, '03'], [7.127659574468085, '04'], [6.732394366197183, '22'], [5.466666666666667, '09']]
print("Top 5 Hours for Ask Posts Comments")
Top 5 Hours for Ask Posts Comments
for avg, hr in sorted_swap[:5]:
print(
"{}:00 {:.2f} average comments per post".format(hr, avg)
)
15:00 38.59 average comments per post 02:00 23.78 average comments per post 20:00 21.51 average comments per post 16:00 16.65 average comments per post 21:00 15.98 average comments per post
# Conclusion
### 15:00, 2:00, 20:00, 16:00 and 21:00 had the highest comments of posts