Companion to Lecture 4 of Harvard CS109: Data Science
import requests
from pattern import web
from BeautifulSoup import BeautifulSoup
url = 'http://www.imdb.com/search/title?sort=num_votes,desc&start=1&title_type=feature&year=1950,2012'
r = requests.get(url)
print r.url
http://www.imdb.com/search/title?sort=num_votes,desc&start=1&title_type=feature&year=1950,2012
url = 'http://www.imdb.com/search/title'
params = dict(sort='num_votes,desc', start=1, title_type='feature', year='1950,2012')
r = requests.get(url, params=params)
print r.url # notice it constructs the full url for you
http://www.imdb.com/search/title?sort=num_votes%2Cdesc&start=1&title_type=feature&year=1950%2C2012
#selection in pattern follows the rules of CSS
dom = web.Element(r.text)
for movie in dom.by_tag('td.title'):
title = movie.by_tag('a')[0].content
genres = movie.by_tag('span.genre')[0].by_tag('a')
genres = [g.content for g in genres]
runtime = movie.by_tag('span.runtime')[0].content
rating = movie.by_tag('span.value')[0].content
print title, genres, runtime, rating
The Shawshank Redemption [u'Crime', u'Drama'] 142 mins. 9.3 The Dark Knight [u'Action', u'Crime', u'Drama', u'Thriller'] 152 mins. 9.0 Inception [u'Action', u'Adventure', u'Mystery', u'Sci-Fi', u'Thriller'] 148 mins. 8.8 Pulp Fiction [u'Crime', u'Drama', u'Thriller'] 154 mins. 9.0 Fight Club [u'Drama'] 139 mins. 8.9 The Lord of the Rings: The Fellowship of the Ring [u'Action', u'Adventure', u'Fantasy'] 178 mins. 8.8 The Matrix [u'Action', u'Adventure', u'Sci-Fi'] 136 mins. 8.7 The Lord of the Rings: The Return of the King [u'Action', u'Adventure', u'Fantasy'] 201 mins. 8.9 The Godfather [u'Crime', u'Drama'] 175 mins. 9.2 Forrest Gump [u'Drama', u'Romance'] 142 mins. 8.7 The Dark Knight Rises [u'Action', u'Crime', u'Thriller'] 165 mins. 8.6 The Lord of the Rings: The Two Towers [u'Action', u'Adventure', u'Fantasy'] 179 mins. 8.7 Se7en [u'Crime', u'Mystery', u'Thriller'] 127 mins. 8.7 Avatar [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] 162 mins. 7.9 Batman Begins [u'Action', u'Adventure', u'Crime', u'Drama'] 140 mins. 8.3 Gladiator [u'Action', u'Adventure', u'Drama'] 155 mins. 8.5 Star Wars [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] 121 mins. 8.8 The Avengers [u'Action', u'Fantasy'] 143 mins. 8.2 Memento [u'Mystery', u'Thriller'] 113 mins. 8.6 American Beauty [u'Drama'] 122 mins. 8.5 Schindler's List [u'Biography', u'Drama', u'History', u'War'] 195 mins. 8.9 Saving Private Ryan [u'Action', u'Drama', u'War'] 169 mins. 8.6 The Departed [u'Crime', u'Drama', u'Thriller'] 151 mins. 8.5 The Silence of the Lambs [u'Crime', u'Drama', u'Thriller'] 118 mins. 8.7 Pirates of the Caribbean: The Curse of the Black Pearl [u'Action', u'Adventure', u'Fantasy'] 143 mins. 8.0 Star Wars: Episode V - The Empire Strikes Back [u'Action', u'Adventure', u'Sci-Fi'] 124 mins. 8.8 Titanic [u'Drama', u'Romance'] 194 mins. 7.6 V for Vendetta [u'Action', u'Crime', u'Fantasy', u'Mystery', u'Sci-Fi', u'Thriller'] 132 mins. 8.2 Inglourious Basterds [u'Adventure', u'Drama', u'War'] 153 mins. 8.3 The Prestige [u'Drama', u'Mystery', u'Thriller'] 130 mins. 8.4 American History X [u'Crime', u'Drama'] 119 mins. 8.6 The Godfather: Part II [u'Crime', u'Drama'] 200 mins. 9.0 The Usual Suspects [u'Crime', u'Mystery', u'Thriller'] 106 mins. 8.7 Braveheart [u'Action', u'Biography', u'Drama', u'History', u'War'] 177 mins. 8.4 Terminator 2: Judgment Day [u'Action', u'Sci-Fi', u'Thriller'] 137 mins. 8.6 The Sixth Sense [u'Drama', u'Mystery', u'Thriller'] 107 mins. 8.2 Kill Bill: Vol. 1 [u'Action', u'Crime'] 111 mins. 8.2 Goodfellas [u'Biography', u'Crime', u'Drama', u'Thriller'] 146 mins. 8.8 Sin City [u'Crime', u'Thriller'] 124 mins. 8.2 Léon: The Professional [u'Crime', u'Drama', u'Thriller'] 110 mins. 8.6 Django Unchained [u'Adventure', u'Drama', u'Western'] 165 mins. 8.5 One Flew Over the Cuckoo's Nest [u'Drama'] 133 mins. 8.8 The Green Mile [u'Crime', u'Drama', u'Fantasy', u'Mystery'] 189 mins. 8.5 Raiders of the Lost Ark [u'Action', u'Adventure'] 115 mins. 8.6 Eternal Sunshine of the Spotless Mind [u'Drama', u'Romance', u'Sci-Fi'] 108 mins. 8.4 Shutter Island [u'Drama', u'Thriller'] 138 mins. 8.0 Iron Man [u'Action', u'Adventure', u'Sci-Fi'] 126 mins. 7.9 Back to the Future [u'Adventure', u'Comedy', u'Sci-Fi'] 116 mins. 8.5 WALL·E [u'Animation', u'Adventure', u'Family', u'Romance', u'Sci-Fi'] 98 mins. 8.5 300 [u'Action', u'Fantasy', u'History', u'War'] 117 mins. 7.7
bs = BeautifulSoup(r.text)
for movie in bs.findAll('td', 'title'):
title = movie.find('a').contents[0]
genres = movie.find('span', 'genre').findAll('a')
genres = [g.contents[0] for g in genres]
runtime = movie.find('span', 'runtime').contents[0]
rating = movie.find('span', 'value').contents[0]
print title, genres, runtime, rating
The Shawshank Redemption [u'Crime', u'Drama'] 142 mins. 9.3 The Dark Knight [u'Action', u'Crime', u'Drama', u'Thriller'] 152 mins. 9.0 Inception [u'Action', u'Adventure', u'Mystery', u'Sci-Fi', u'Thriller'] 148 mins. 8.8 Pulp Fiction [u'Crime', u'Drama', u'Thriller'] 154 mins. 9.0 Fight Club [u'Drama'] 139 mins. 8.9 The Lord of the Rings: The Fellowship of the Ring [u'Action', u'Adventure', u'Fantasy'] 178 mins. 8.8 The Matrix [u'Action', u'Adventure', u'Sci-Fi'] 136 mins. 8.7 The Lord of the Rings: The Return of the King [u'Action', u'Adventure', u'Fantasy'] 201 mins. 8.9 The Godfather [u'Crime', u'Drama'] 175 mins. 9.2 Forrest Gump [u'Drama', u'Romance'] 142 mins. 8.7 The Dark Knight Rises [u'Action', u'Crime', u'Thriller'] 165 mins. 8.6 The Lord of the Rings: The Two Towers [u'Action', u'Adventure', u'Fantasy'] 179 mins. 8.7 Se7en [u'Crime', u'Mystery', u'Thriller'] 127 mins. 8.7 Avatar [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] 162 mins. 7.9 Batman Begins [u'Action', u'Adventure', u'Crime', u'Drama'] 140 mins. 8.3 Gladiator [u'Action', u'Adventure', u'Drama'] 155 mins. 8.5 Star Wars [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] 121 mins. 8.8 The Avengers [u'Action', u'Fantasy'] 143 mins. 8.2 Memento [u'Mystery', u'Thriller'] 113 mins. 8.6 American Beauty [u'Drama'] 122 mins. 8.5 Schindler's List [u'Biography', u'Drama', u'History', u'War'] 195 mins. 8.9 Saving Private Ryan [u'Action', u'Drama', u'War'] 169 mins. 8.6 The Departed [u'Crime', u'Drama', u'Thriller'] 151 mins. 8.5 The Silence of the Lambs [u'Crime', u'Drama', u'Thriller'] 118 mins. 8.7 Pirates of the Caribbean: The Curse of the Black Pearl [u'Action', u'Adventure', u'Fantasy'] 143 mins. 8.0 Star Wars: Episode V - The Empire Strikes Back [u'Action', u'Adventure', u'Sci-Fi'] 124 mins. 8.8 Titanic [u'Drama', u'Romance'] 194 mins. 7.6 V for Vendetta [u'Action', u'Crime', u'Fantasy', u'Mystery', u'Sci-Fi', u'Thriller'] 132 mins. 8.2 Inglourious Basterds [u'Adventure', u'Drama', u'War'] 153 mins. 8.3 The Prestige [u'Drama', u'Mystery', u'Thriller'] 130 mins. 8.4 American History X [u'Crime', u'Drama'] 119 mins. 8.6 The Godfather: Part II [u'Crime', u'Drama'] 200 mins. 9.0 The Usual Suspects [u'Crime', u'Mystery', u'Thriller'] 106 mins. 8.7 Braveheart [u'Action', u'Biography', u'Drama', u'History', u'War'] 177 mins. 8.4 Terminator 2: Judgment Day [u'Action', u'Sci-Fi', u'Thriller'] 137 mins. 8.6 The Sixth Sense [u'Drama', u'Mystery', u'Thriller'] 107 mins. 8.2 Kill Bill: Vol. 1 [u'Action', u'Crime'] 111 mins. 8.2 Goodfellas [u'Biography', u'Crime', u'Drama', u'Thriller'] 146 mins. 8.8 Sin City [u'Crime', u'Thriller'] 124 mins. 8.2 Léon: The Professional [u'Crime', u'Drama', u'Thriller'] 110 mins. 8.6 Django Unchained [u'Adventure', u'Drama', u'Western'] 165 mins. 8.5 One Flew Over the Cuckoo's Nest [u'Drama'] 133 mins. 8.8 The Green Mile [u'Crime', u'Drama', u'Fantasy', u'Mystery'] 189 mins. 8.5 Raiders of the Lost Ark [u'Action', u'Adventure'] 115 mins. 8.6 Eternal Sunshine of the Spotless Mind [u'Drama', u'Romance', u'Sci-Fi'] 108 mins. 8.4 Shutter Island [u'Drama', u'Thriller'] 138 mins. 8.0 Iron Man [u'Action', u'Adventure', u'Sci-Fi'] 126 mins. 7.9 Back to the Future [u'Adventure', u'Comedy', u'Sci-Fi'] 116 mins. 8.5 WALL·E [u'Animation', u'Adventure', u'Family', u'Romance', u'Sci-Fi'] 98 mins. 8.5 300 [u'Action', u'Fantasy', u'History', u'War'] 117 mins. 7.7