In [2]:
# 爬虫基本原理
from IPython.display import display_html, HTML
HTML('<iframe src=http://www.cnblogs.com/zhaof/p/6898138.html \ width=1000 height=500></iframe>')
Out[2]:
需要解决的问题¶
- 页面解析
- 获取Javascript隐藏源数据
- 自动翻页
- 自动登录
- 连接API接口
- 一般的数据抓取，使用requests和beautifulsoup配合就可以了。
- 尤其是对于翻页时url出现规则变化的网页，只需要处理规则化的url就可以了。
- 以简单的例子是抓取天涯论坛上关于某一个关键词的帖子。
In [3]:
import requests
from bs4 import BeautifulSoup
In [53]:
help(requests.get)
In [5]:
url = 'http://computational-class.github.io/bigdata/data/test.html'
content = requests.get(url)
help(content)
In [6]:
print(content.text)
In [7]:
content.encoding
Out[7]:
Beautiful Soup¶
Beautiful Soup is a Python library designed for quick turnaround projects like screen-scraping. Three features make it powerful:
- Beautiful Soup provides a few simple methods. It doesn't take much code to write an application
- Beautiful Soup automatically converts incoming documents to Unicode and outgoing documents to UTF-8. Then you just have to specify the original encoding.
- Beautiful Soup sits on top of popular Python parsers like
lxmlandhtml5lib.
html.parser¶
Beautiful Soup supports the html.parser included in Python’s standard library
lxml¶
but it also supports a number of third-party Python parsers. One is the lxml parser lxml. Depending on your setup, you might install lxml with one of these commands:
$ apt-get install python-lxml
$ easy_install lxml
$ pip install lxml
html5lib¶
Another alternative is the pure-Python html5lib parser html5lib, which parses HTML the way a web browser does. Depending on your setup, you might install html5lib with one of these commands:
$ apt-get install python-html5lib
$ easy_install html5lib
$ pip install html5lib
In [9]:
url = 'http://computational-class.github.io/bigdata/data/test.html'
content = requests.get(url)
content = content.text
soup = BeautifulSoup(content, 'html.parser')
soup
Out[9]:
In [10]:
print(soup.prettify())
- html
- head
- title
- body
- p (class = 'title', 'story' )
- a (class = 'sister')
- href/id
- a (class = 'sister')
- p (class = 'title', 'story' )
- head
Select方法三步骤¶
- Inspect (检查)
- Copy
- Copy Selector
- 鼠标选中标题
The Dormouse's story, 右键检查Inspect - 鼠标移动到选中的源代码
- 右键Copy-->Copy Selector
body > p.title > b
In [14]:
soup.select('body > p.title > b')#[0].text
Out[14]:
Select 方法: 通过标签名查找¶
In [68]:
soup.select('title')
Out[68]:
In [65]:
soup.select('a')
Out[65]:
In [66]:
soup.select('b')
Out[66]:
Select 方法: 通过类名查找¶
In [69]:
soup.select('.title')
Out[69]:
In [26]:
soup.select('.sister')
Out[26]:
In [27]:
soup.select('.story')
Out[27]:
Select 方法: 通过id名查找¶
In [15]:
soup.select('#link1')
Out[15]:
In [16]:
soup.select('#link1')[0]['href']
Out[16]:
In [70]:
soup.select('p #link1')
Out[70]:
In [17]:
soup.select("head > title")
Out[17]:
In [72]:
soup.select("body > p")
Out[72]:
find_all方法¶
In [30]:
soup('p')
Out[30]:
In [31]:
soup.find_all('p')
Out[31]:
In [32]:
[i.text for i in soup('p')]
Out[32]:
In [34]:
for i in soup('p'):
print(i.text)
In [35]:
for tag in soup.find_all(True):
print(tag.name)
In [36]:
soup('head') # or soup.head
Out[36]:
In [37]:
soup('body') # or soup.body
Out[37]:
In [38]:
soup('title') # or soup.title
Out[38]:
In [39]:
soup('p')
Out[39]:
In [40]:
soup.p
Out[40]:
In [41]:
soup.title.name
Out[41]:
In [42]:
soup.title.string
Out[42]:
In [43]:
soup.title.text
# 推荐使用text方法
Out[43]:
In [44]:
soup.title.parent.name
Out[44]:
In [45]:
soup.p
Out[45]:
In [46]:
soup.p['class']
Out[46]:
In [47]:
soup.find_all('p', {'class', 'title'})
Out[47]:
In [19]:
soup.find_all('p', class_= 'title')
Out[19]:
In [49]:
soup.find_all('p', {'class', 'story'})
Out[49]:
In [34]:
soup.find_all('p', {'class', 'story'})[0].find_all('a')
Out[34]:
In [51]:
soup.a
Out[51]:
In [52]:
soup('a')
Out[52]:
In [53]:
soup.find(id="link3")
Out[53]:
In [54]:
soup.find_all('a')
Out[54]:
In [55]:
soup.find_all('a', {'class', 'sister'}) # compare with soup.find_all('a')
Out[55]:
In [56]:
soup.find_all('a', {'class', 'sister'})[0]
Out[56]:
In [57]:
soup.find_all('a', {'class', 'sister'})[0].text
Out[57]:
In [58]:
soup.find_all('a', {'class', 'sister'})[0]['href']
Out[58]:
In [59]:
soup.find_all('a', {'class', 'sister'})[0]['id']
Out[59]:
In [71]:
soup.find_all(["a", "b"])
Out[71]:
In [38]:
print(soup.get_text())
In [35]:
from IPython.display import display_html, HTML
HTML('<iframe src=http://mp.weixin.qq.com/s?__biz=MzA3MjQ5MTE3OA==&\
mid=206241627&idx=1&sn=471e59c6cf7c8dae452245dbea22c8f3&3rd=MzA3MDU4NTYzMw==&scene=6#rd\
width=800 height=500></iframe>')
# the webpage we would like to crawl
Out[35]:
查看源代码 Inspect¶
In [36]:
url = "http://mp.weixin.qq.com/s?__biz=MzA3MjQ5MTE3OA==&mid=206241627&idx=1&sn=471e59c6cf7c8dae452245dbea22c8f3&3rd=MzA3MDU4NTYzMw==&scene=6#rd"
content = requests.get(url).text #获取网页的html文本
soup = BeautifulSoup(content, 'html.parser')
In [37]:
title = soup.select("#activity-name")
title[0].text.strip()
Out[37]:
In [40]:
soup.find('h2', {'class', 'rich_media_title'}).text.strip()
Out[40]:
In [185]:
print(soup.find('div', {'class', 'rich_media_meta_list'}) )
In [42]:
soup.find('em').text
Out[42]:
In [43]:
article = soup.find('div', {'class' , 'rich_media_content'}).text
print(article)
In [44]:
rmml = soup.find('div', {'class', 'rich_media_meta_list'})
date = rmml.find(id = 'post-date').text
rmc = soup.find('div', {'class', 'rich_media_content'})
content = rmc.get_text()
print(title[0].text.strip())
print(date)
print(content)
requests + Xpath方法介绍：以豆瓣电影为例¶
Xpath 即为 XML 路径语言（XML Path Language），它是一种用来确定 XML 文档中某部分位置的语言。
Xpath 基于 XML 的树状结构，提供在数据结构树中找寻节点的能力。起初 Xpath 的提出的初衷是将其作为一个通用的、介于 Xpointer 与 XSL 间的语法模型。但是Xpath 很快的被开发者采用来当作小型查询语言。
获取元素的Xpath信息并获得文本： 这里的“元素的Xpath信息”是需要我们手动获取的，获取方式为：
- 定位目标元素
- 在网站上依次点击：右键 > 检查
- copy xpath
- xpath + '/text()'
In [46]:
import requests
from lxml import etree
url = 'https://movie.douban.com/subject/26611804/'
data = requests.get(url).text
s = etree.HTML(data)
豆瓣电影的名称对应的的xpath为xpath_title，那么title表达为：
title = s.xpath('xpath_info/text()')
其中，xpath_info为：
//*[@id="content"]/h1/span[1]
In [47]:
title = s.xpath('//*[@id="content"]/h1/span[1]/text()')[0]
director = s.xpath('//*[@id="info"]/span[1]/span[2]/a/text()')
actors = s.xpath('//*[@id="info"]/span[3]/span[2]/a/text()')
type1 = s.xpath('//*[@id="info"]/span[5]/text()')
type2 = s.xpath('//*[@id="info"]/span[6]/text()')
type3 = s.xpath('//*[@id="info"]/span[7]/text()')
time = s.xpath('//*[@id="info"]/span[11]/text()')
length = s.xpath('//*[@id="info"]/span[13]/text()')
score = s.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()')[0]
In [48]:
print(title, director, actors, type1, type2, type3, time, length, score)
In [51]:
import requests
url = 'https://api.douban.com/v2/movie/26611804'
jsonm = requests.get(url).json()
In [58]:
#jsonm.values()
jsonm.keys(), jsonm['rating']
Out[58]:
In [84]:
jsonm['alt']
Out[84]:
In [87]:
jsonm['attrs']['director']
Out[87]:
In [89]:
jsonm['attrs']['movie_type']
Out[89]:
In [88]:
jsonm['attrs']['cast']
Out[88]:
作业：抓取豆瓣电影 Top 250¶
In [59]:
import requests
from bs4 import BeautifulSoup
from lxml import etree
url0 = 'https://movie.douban.com/top250?start=0&filter='
data = requests.get(url0).text
s = etree.HTML(data)
In [222]:
s.xpath('//*[@id="content"]/div/div[1]/ol/li[1]/div/div[2]/div[1]/a/span[1]/text()')[0]
Out[222]:
In [225]:
s.xpath('//*[@id="content"]/div/div[1]/ol/li[2]/div/div[2]/div[1]/a/span[1]/text()')[0]
Out[225]:
In [227]:
s.xpath('//*[@id="content"]/div/div[1]/ol/li[3]/div/div[2]/div[1]/a/span[1]/text()')[0]
Out[227]:
In [60]:
import requests
from bs4 import BeautifulSoup
url0 = 'https://movie.douban.com/top250?start=0&filter='
data = requests.get(url0).text
soup = BeautifulSoup(data, 'lxml')
In [61]:
movies = soup.find_all('div', {'class', 'info'})
In [62]:
len(movies)
Out[62]:
In [63]:
movies[0].a['href']
Out[63]:
In [39]:
movies[0].find('span', {'class', 'title'}).text
Out[39]:
In [26]:
movies[0].find('div', {'class', 'star'})
Out[26]:
In [28]:
movies[0].find('span', {'class', 'rating_num'}).text
Out[28]:
In [90]:
people_num = movies[0].find('div', {'class', 'star'}).find_all('span')[-1]
people_num.text.split('人评价')[0]
Out[90]:
In [64]:
for i in movies:
url = i.a['href']
title = i.find('span', {'class', 'title'}).text
des = i.find('div', {'class', 'star'})
rating = des.find('span', {'class', 'rating_num'}).text
rating_num = des.find_all('span')[-1].text.split('人评价')[0]
print(url, title, rating, rating_num)
In [51]:
for i in range(0, 250, 25):
print('https://movie.douban.com/top250?start=%d&filter='% i)
In [65]:
import requests
from bs4 import BeautifulSoup
dat = []
for j in range(0, 250, 25):
urli = 'https://movie.douban.com/top250?start=%d&filter='% j
data = requests.get(urli).text
soup = BeautifulSoup(data, 'lxml')
movies = soup.find_all('div', {'class', 'info'})
for i in movies:
url = i.a['href']
title = i.find('span', {'class', 'title'}).text
des = i.find('div', {'class', 'star'})
rating = des.find('span', {'class', 'rating_num'}).text
rating_num = des.find_all('span')[-1].text.split('人评价')[0]
listi = [url, title, rating, rating_num]
dat.append(listi)
In [66]:
import pandas as pd
df = pd.DataFrame(dat, columns = ['url', 'title', 'rating', 'rating_num'])
df['rating'] = df.rating.astype(float)
df['rating_num'] = df.rating_num.astype(int)
df.head()
Out[66]:
In [3]:
%matplotlib inline
import matplotlib.pyplot as plt
plt.hist(df.rating_num)
plt.show()
In [19]:
plt.hist(df.rating)
plt.show()
In [11]:
fig = plt.figure(figsize=(16, 16),facecolor='white')
plt.plot(df.rating_num, df.rating, 'bo')
for i in df.index:
plt.text(df.rating_num[i], df.rating[i], df.title[i],
fontsize = df.rating[i],
color = 'red', rotation = 45)
plt.show()
In [123]:
df[df.rating > 9.4]
Out[123]:
In [69]:
alist = []
for i in df.index:
alist.append( [df.rating_num[i], df.rating[i], df.title[i] ])
blist =[[df.rating_num[i], df.rating[i], df.title[i] ] for i in df.index]
alist
Out[69]:
In [70]:
from IPython.display import display_html, HTML
HTML('<iframe src=http://nbviewer.jupyter.org/github/computational-class/bigdata/blob/gh-pages/vis/douban250bubble.html \
width=1000 height=500></iframe>')
Out[70]:
作业：¶
- 抓取复旦新媒体微信公众号最新一期的内容