In [108]:
import matplotlib as plt
import numpy as np
import pandas as pd
from pyquery import PyQuery as pq
import requests
import re
pd.set_option('display.max_rows', 500)
%matplotlib inline
plt.style.use('ggplot')
In [46]:
def get_content(url):
    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
    result = requests.get(url, headers=headers)
    return result.content
In [117]:
links = []
words = []
names = []
dates = []
spaces = re.compile("[\s.,:]+")
for page in ["http://waitbutwhy.com/archive","http://waitbutwhy.com/archive/page/2"]:
    p = pq(get_content(page))
    for post in p(".post-list li h5 a"):
        link = post.get("href")
        links.append(link)
        l = pq(get_content(link))
        text = l('.entry-content').text()
        footer = l('#social-ads').text()
        names.append(l('.entry-header h1').text())
        wordcount = len(spaces.findall(text))
        footercount = len(spaces.findall(footer))
        words.append(wordcount-footercount)
        dates.append(re.compile("/(20[0-9][0-9]/[0-9][0-9])/").search(link).group(1))
In [118]:
months = ['{}/{}'.format(y,m if m>9 else '0{}'.format(m)) for y in range(2014,2017) for m in range(1,13)]
for month in months:
    links.append(None)
    names.append(None)
    dates.append(month)
    words.append(0)
In [119]:
posts = pd.DataFrame({"name":names,"words":words,"link":links,"date":dates})
posts[posts['words']!=0].sort_values(by='words',ascending=False)
Out[119]:
date link name words
19 2015/06 http://waitbutwhy.com/2015/06/how-tesla-will-c... How Tesla Will Change The World 27627
16 2015/11 http://waitbutwhy.com/2015/11/the-cook-and-the... The Cook and the Chef: Musk’s Secret Sauce 21494
24 2015/01 http://waitbutwhy.com/2015/01/artificial-intel... The AI Revolution: Our Immortality or Extinction 17130
9 2016/03 http://waitbutwhy.com/2016/03/cryonics.html Why Cryonics Makes Sense 15337
8 2016/05 http://waitbutwhy.com/2016/05/mailbag-1.html Mailbag #1 10721
38 2014/09 http://waitbutwhy.com/2014/09/muhammad-isis-ir... From Muhammad to ISIS: Iraq’s Full Story 9568
17 2015/08 http://waitbutwhy.com/2015/08/how-and-why-spac... How (and Why) SpaceX Will Colonize Mars 8984
33 2014/10 http://waitbutwhy.com/2014/10/religion-for-the... Religion for the Nonreligious 8494
109 2014/10 http://waitbutwhy.com/2014/10/religion-for-the... Religion for the Nonreligious 8494
60 2014/10 http://waitbutwhy.com/2014/10/religion-for-the... Religion for the Nonreligious 8494
99 2015/01 http://waitbutwhy.com/2015/01/artificial-intel... The AI Revolution: The Road to Superintelligence 7972
25 2015/01 http://waitbutwhy.com/2015/01/artificial-intel... The AI Revolution: The Road to Superintelligence 7972
50 2015/01 http://waitbutwhy.com/2015/01/artificial-intel... The AI Revolution: The Road to Superintelligence 7972
53 2015/05 http://waitbutwhy.com/2015/05/elon-musk-the-wo... Elon Musk: The World’s Raddest Man 7232
102 2015/05 http://waitbutwhy.com/2015/05/elon-musk-the-wo... Elon Musk: The World’s Raddest Man 7232
20 2015/05 http://waitbutwhy.com/2015/05/elon-musk-the-wo... Elon Musk: The World’s Raddest Man 7232
4 2016/09 http://waitbutwhy.com/2016/09/spacexs-big-fkin... SpaceX’s Big Fucking Rocket – The Full Story 6784
30 2014/11 http://waitbutwhy.com/2014/11/1000000-grahams-... From 1,000,000 to Graham’s Number 6329
40 2014/07 http://waitbutwhy.com/2014/07/japan-and-how-i-... Japan, and How I Failed to Figure it Out 6206
21 2015/03 http://waitbutwhy.com/2015/03/procrastination-... The Procrastination Matrix 6152
37 2014/09 http://waitbutwhy.com/2014/09/but-what-about-g... But What About Greenland? 5394
28 2014/12 http://waitbutwhy.com/2014/12/what-makes-you-y... What Makes You You? 5117
52 2014/05 http://waitbutwhy.com/2014/05/fermi-paradox.html The Fermi Paradox 4827
45 2014/05 http://waitbutwhy.com/2014/05/fermi-paradox.html The Fermi Paradox 4827
101 2014/05 http://waitbutwhy.com/2014/05/fermi-paradox.html The Fermi Paradox 4827
39 2014/08 http://waitbutwhy.com/2014/08/19-things-learne... 19 Things I Learned in Nigeria 4533
5 2016/09 http://waitbutwhy.com/2016/09/marriage-decisio... The Marriage Decision: Everything Forever or N... 4396
13 2016/03 http://waitbutwhy.com/2016/03/doing-a-ted-talk... Doing a TED Talk: The Full Story 4388
59 2014/06 http://waitbutwhy.com/2014/06/taming-mammoth-l... Taming the Mammoth: Why You Should Stop Caring... 4338
43 2014/06 http://waitbutwhy.com/2014/06/taming-mammoth-l... Taming the Mammoth: Why You Should Stop Caring... 4338
108 2014/06 http://waitbutwhy.com/2014/06/taming-mammoth-l... Taming the Mammoth: Why You Should Stop Caring... 4338
29 2014/12 http://waitbutwhy.com/2014/12/10-types-odd-fri... 10 Types of Odd Friendships You’re Probably Pa... 4278
41 2014/07 http://waitbutwhy.com/2014/07/russia-what-you-... Russia: What You Didn’t Know You Don’t Know 3982
84 2013/09 http://waitbutwhy.com/2013/09/20-things-i-lear... 20 Things I Learned While I Was in North Korea 3752
78 2013/11 http://waitbutwhy.com/2013/11/how-to-beat-proc... How to Beat Procrastination 3703
70 2014/01 http://waitbutwhy.com/2014/01/your-family-past... Your Family: Past, Present, and Future 3584
22 2015/03 http://waitbutwhy.com/2015/03/7-3-billion-peop... 7.3 Billion People, One Building 3435
69 2014/02 http://waitbutwhy.com/2014/02/why-bugs-ruin-ev... Why Bugs Ruin Everything 3381
105 2013/07 http://waitbutwhy.com/2013/07/7-ways-to-be-ins... 7 Ways to Be Insufferable on Facebook 3319
98 2013/07 http://waitbutwhy.com/2013/07/7-ways-to-be-ins... 7 Ways to Be Insufferable on Facebook 3319
56 2013/07 http://waitbutwhy.com/2013/07/7-ways-to-be-ins... 7 Ways to Be Insufferable on Facebook 3319
12 2016/03 http://waitbutwhy.com/2016/03/sound.html Everything You Should Know About Sound 3309
14 2016/01 http://waitbutwhy.com/2016/01/horizontal-histo... Horizontal History 3228
66 2014/02 http://waitbutwhy.com/2014/02/american-preside... The American Presidents—Washington to Lincoln 3101
32 2014/10 http://waitbutwhy.com/2014/10/dark-secrets-bir... The Dark Secrets of the Bird World 3053
49 2014/04 http://waitbutwhy.com/2014/04/everything-dont-... Everything You Don’t Know About Tipping 2825
46 2014/05 http://waitbutwhy.com/2014/05/absurdly-famous-... 10 Absurdly Famous People You Probably Don’t K... 2778
65 2014/03 http://waitbutwhy.com/2014/03/why-is-my-laptop... Why is My Laptop On? 2758
31 2014/11 http://waitbutwhy.com/2014/11/from-1-to-100000... From 1 to 1,000,000 2658
68 2014/02 http://waitbutwhy.com/2014/02/pick-life-partne... How to Pick Your Life Partner – Part 1 2604
103 2014/02 http://waitbutwhy.com/2014/02/pick-life-partne... How to Pick Your Life Partner – Part 1 2604
54 2014/02 http://waitbutwhy.com/2014/02/pick-life-partne... How to Pick Your Life Partner – Part 1 2604
34 2014/10 http://waitbutwhy.com/2014/10/how-religion-got... How Religion Got in the Way 2434
3 2016/10 http://waitbutwhy.com/2016/10/second-president... The Second Presidential Debate 2425
72 2013/12 http://waitbutwhy.com/2013/12/wait-but-why-hol... Wait But Why Holiday Post 2322
58 2013/12 http://waitbutwhy.com/2013/12/how-to-name-baby... How to Name a Baby 2288
107 2013/12 http://waitbutwhy.com/2013/12/how-to-name-baby... How to Name a Baby 2288
74 2013/12 http://waitbutwhy.com/2013/12/how-to-name-baby... How to Name a Baby 2288
73 2013/12 http://waitbutwhy.com/2013/12/your-ancestor-is... Meet Your Ancestors (All of Them) 2121
44 2014/05 http://waitbutwhy.com/2014/05/secretly-hate-ba... Why You Secretly Hate Cool Bars 2029
71 2014/01 http://waitbutwhy.com/2014/01/the-great-perils... The Great Perils of Social Interaction 2022
23 2015/02 http://waitbutwhy.com/2015/02/american-preside... The American Presidents—Johnson to McKinley 2011
75 2013/12 http://waitbutwhy.com/2013/12/11-awkward-thing... 11 Awkward Things About Email 1989
67 2014/02 http://waitbutwhy.com/2014/02/pick-life-partne... How to Pick Your Life Partner – Part 2 1926
0 2016/11 http://waitbutwhy.com/2016/11/its-going-to-be-... It’s Going to Be Okay – Follow Up 1864
1 2016/11 http://waitbutwhy.com/2016/11/its-going-to-be-... It’s Going to Be Okay 1780
86 2013/09 http://waitbutwhy.com/2013/09/all-weird-toys-f... All the Weird Toys From Your Childhood 1778
27 2014/12 http://waitbutwhy.com/2014/12/the-teen-years-9... The Teen Years: 9 Cringe-Inducing Realizations 1661
63 2014/03 http://waitbutwhy.com/2014/03/combined-wealth-... What Could You Buy With $241 Trillion? 1609
18 2015/07 http://waitbutwhy.com/2015/07/why-im-always-la... Why I’m Always Late 1605
55 2013/09 http://waitbutwhy.com/2013/09/why-generation-y... Why Generation Y Yuppies Are Unhappy 1599
104 2013/09 http://waitbutwhy.com/2013/09/why-generation-y... Why Generation Y Yuppies Are Unhappy 1599
85 2013/09 http://waitbutwhy.com/2013/09/why-generation-y... Why Generation Y Yuppies Are Unhappy 1599
77 2013/11 http://waitbutwhy.com/2013/11/4-mind-blowing-t... 4 Mind-Blowing Things About Stars 1546
87 2013/08 http://waitbutwhy.com/2013/08/the-apple-game-h... The Apple Game: How Good a Person Are You? 1522
61 2013/10 http://waitbutwhy.com/2013/10/10-types-of-30-y... 10 Types of 30-Year-Old Single Guys 1489
82 2013/10 http://waitbutwhy.com/2013/10/10-types-of-30-y... 10 Types of 30-Year-Old Single Guys 1489
110 2013/10 http://waitbutwhy.com/2013/10/10-types-of-30-y... 10 Types of 30-Year-Old Single Guys 1489
48 2014/04 http://waitbutwhy.com/2014/04/traveling-third-... Traveling To The Third World Is Great And Also... 1486
76 2013/11 http://waitbutwhy.com/2013/11/life-is-picture-... Life is a Picture, But You Live in a Pixel 1443
51 2013/10 http://waitbutwhy.com/2013/10/why-procrastinat... Why Procrastinators Procrastinate 1438
79 2013/10 http://waitbutwhy.com/2013/10/why-procrastinat... Why Procrastinators Procrastinate 1438
100 2013/10 http://waitbutwhy.com/2013/10/why-procrastinat... Why Procrastinators Procrastinate 1438
92 2013/08 http://waitbutwhy.com/2013/08/the-bunny-manife... The Bunny Manifesto 1395
83 2013/10 http://waitbutwhy.com/2013/10/what-does-quadri... What Does a Quadrillion Sour Patch Kids Look L... 1383
7 2016/05 http://waitbutwhy.com/2016/05/clueyness-a-weir... Clueyness: A Weird Kind of Sad 1333
90 2013/08 http://waitbutwhy.com/2013/08/creepy-kids-in-c... Creepy Kids in Creepy Vintage Ads 1324
80 2013/10 http://waitbutwhy.com/2013/10/the-primate-awar... The Primate Awards 1297
95 2013/07 http://waitbutwhy.com/2013/07/12-types-of-peop... 12 Types Of People You’ll Find In Every Hostel 1227
64 2014/03 http://waitbutwhy.com/2014/03/sports-fans-spor... Why Sports Fans are Sports Fans 1205
15 2015/12 http://waitbutwhy.com/2015/12/the-tail-end.html The Tail End 1182
111 2015/12 http://waitbutwhy.com/2015/12/the-tail-end.html The Tail End 1182
62 2015/12 http://waitbutwhy.com/2015/12/the-tail-end.html The Tail End 1182
47 2014/05 http://waitbutwhy.com/2014/05/life-weeks.html Your Life in Weeks 1133
6 2016/08 http://waitbutwhy.com/2016/08/wait-hi-full-rep... Wait But Hi – Full Report 1027
94 2013/07 http://waitbutwhy.com/2013/07/14-shitty-saying... 14 Shitty Sayings 905
97 2013/07 http://waitbutwhy.com/2013/07/medieval-people-... Medieval People In Bad Situations 748
42 2014/07 http://waitbutwhy.com/2014/07/odd-things-odd-p... Odd Things in Odd Places: Intro 683
88 2013/08 http://waitbutwhy.com/2013/08/7-asinine-things... 7 Asinine Things About Society 627
93 2013/08 http://waitbutwhy.com/2013/08/the-death-toll-c... The Death Toll Comparison Breakdown 615
26 2014/12 http://waitbutwhy.com/2014/12/popular-posts-20... Our Most Popular Posts of 2014 587
2 2016/10 http://waitbutwhy.com/2016/10/100-blocks-day.html 100 Blocks a Day 517
106 2013/08 http://waitbutwhy.com/2013/08/putting-time-in-... Putting Time In Perspective – UPDATED 413
57 2013/08 http://waitbutwhy.com/2013/08/putting-time-in-... Putting Time In Perspective – UPDATED 413
89 2013/08 http://waitbutwhy.com/2013/08/putting-time-in-... Putting Time In Perspective – UPDATED 413
36 2014/09 http://waitbutwhy.com/2014/09/genie-question.html The Genie Question 352
35 2014/09 http://waitbutwhy.com/2014/09/odd-things-odd-p... Odd Things in Odd Places – All 7 Travel Posts 300
91 2013/08 http://waitbutwhy.com/2013/08/what-if-all-71-b... What If All 7.1 Billion People Moved To Tunisia? 254
11 2016/03 http://waitbutwhy.com/2016/03/long-email-delay... How I Handle Long Email Delays 181
81 2013/10 http://waitbutwhy.com/2013/10/the-battle-to-lo... The Battle to Lose the Independent Vote 74
10 2016/03 http://waitbutwhy.com/2016/03/my-ted-talk.html My TED Talk 68
96 2013/07 http://waitbutwhy.com/2013/07/gods-wounded-ego... God’s Wounded Ego 60
In [120]:
monthly = posts.groupby('date').sum()
monthly
Out[120]:
words
date
2013/07 12897
2013/08 6976
2013/09 10327
2013/10 11535
2013/11 6692
2013/12 13296
2014/01 5606
2014/02 16220
2014/03 5572
2014/04 4311
2014/05 20421
2014/06 13014
2014/07 10871
2014/08 4533
2014/09 15614
2014/10 30969
2014/11 8987
2014/12 11643
2015/01 41046
2015/02 2011
2015/03 9587
2015/04 0
2015/05 21696
2015/06 27627
2015/07 1605
2015/08 8984
2015/09 0
2015/10 0
2015/11 21494
2015/12 3546
2016/01 3228
2016/02 0
2016/03 23283
2016/04 0
2016/05 12054
2016/06 0
2016/07 0
2016/08 1027
2016/09 11180
2016/10 2942
2016/11 3644
2016/12 0
In [121]:
ax = monthly.plot(kind='bar',figsize=(13,4),title='Total words of WBW long posts per month',lw=2)
ax.set_ylim(-1000,45000)
ax.set_xlabel("Month")
ax.set_ylabel("Words")
Out[121]:
<matplotlib.text.Text at 0x7027cd0>
In [76]:
 
In [ ]: