import requests
from bs4 import BeautifulSoup
from IPython.display import display_html, HTML
HTML('<iframe src=http://www.hprc.org.cn/wxzl/wxysl/lczf/ width=1000 height=500></iframe>')
# the webpage we would like to crawl
<td width="274" class="bl">· <a href="./d12qgrdzfbg/201603/t20160318_369509.html" target="_blank" title="2016年政府工作报告">2016年政府工作报告</a></td>
# get the link for each year
url = "http://www.hprc.org.cn/wxzl/wxysl/lczf/"
content = requests.get(url)
content.encoding
'ISO-8859-1'
# Specify the encoding
content.encoding = 'utf8' # 'gb18030'
content = content.text
soup = BeautifulSoup(content, 'html.parser')
# links = soup.find_all('td', {'class', 'bl'})
links = soup.select('.bl a')
print(links[0])
<a href="./dssjqgrmdbdh_1/201903/t20190318_4849567.html" target="_blank" title="2019年政府工作报告">2019年政府工作报告</a>
len(links)
50
links[-1]['href']
'./dishiyijie_10/200908/t20090818_3955459.html'
links[0]['href'].split('./')[1]
'dssjqgrmdbdh_1/201903/t20190318_4849567.html'
url + links[0]['href'].split('./')[1]
'http://www.hprc.org.cn/wxzl/wxysl/lczf/dssjqgrmdbdh_1/201903/t20190318_4849567.html'
hyperlinks = [url + i['href'].split('./')[1] for i in links]
hyperlinks[:5]
['http://www.hprc.org.cn/wxzl/wxysl/lczf/dssjqgrmdbdh_1/201903/t20190318_4849567.html', 'http://www.hprc.org.cn/wxzl/wxysl/lczf/dssjqgrmdbdh_1/201803/t20180323_4240852.html', 'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201703/t20170317_4144138.html', 'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201603/t20160318_4135203.html', 'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201503/t20150318_4106347.html']
hyperlinks[-5:]
['http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_9/200908/t20090818_3955464.html', 'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_10/200908/t20090818_3955462.html', 'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_10/200908/t20090818_3955461.html', 'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_10/200908/t20090818_3955460.html', 'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_10/200908/t20090818_3955459.html']
hyperlinks[12] # 2007年有分页
'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_3955570.html'
from IPython.display import display_html, HTML
HTML('<iframe src=http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_3955570.html width=1000 height=500></iframe>')
# 2007年有分页
url_i = 'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_3955570.html'
content = requests.get(url_i)
content.encoding = 'utf8'
content = content.text
#content = content.text.encode(content.encoding).decode('gb18030')
soup = BeautifulSoup(content, 'html.parser')
#scripts = soup.find_all('script')
#scripts[0]
scripts = soup.select('td script')[0]
scripts
<script>
var currentPage = 0;//所在页从0开始
var prevPage = currentPage-1//上一页
var 下一页Page = currentPage+1//下一页
var countPage = 4//共多少页
//document.write("共"+countPage+"页 ");
//循环
var num = 17;
for(var i=0+(currentPage-1-(currentPage-1)%num) ; i<=(num+(currentPage-1-(currentPage-1)%num))&&(i<countPage) ; i++){
if(countPage >1){
if(currentPage==i)
document.write("【<span style=\"color:#FF0000;\" class=\"hui14_30_h\">"+(i+1)+"</span>】 ");
else if(i==0)
document.write("<a href=\"t20090818_3955570.html\" class=\"hui14_30_h\">【"+(i+1)+"】</a> ");
else
document.write("<a href=\"t20090818_3955570"+"_" + i + "."+"html\" class=\"hui14_30_h\">【"+(i+1)+"】</a> ");
}
}
document.write("<br><br>");
//设置上一页代码
if(countPage>1&¤tPage!=0&¤tPage!=1)
document.write("<a href=\"t20090818_3955570"+"_" + prevPage + "."+"html\"><span style=\"color:#0033FF;font-weight:bold\">上一页</span></a> ");
else if(countPage>1&¤tPage!=0&¤tPage==1)
document.write("<a href=\"t20090818_3955570.html\"><span style=\"color:#0033FF;font-weight:bold\">上一页</span></a> ");
//else
// document.write("上一页 ");
//设置下一页代码
if(countPage>1&¤tPage!=(countPage-1))
document.write("<a href=\"t20090818_3955570"+"_" + 下一页Page + "."+"html\" ><span style=\"color:#0033FF;font-weight:bold\">下一页</span></a> ");
//else
// document.write("下一页 ");
</script>
scripts.text
'\n\tvar currentPage = 0;//所在页从0开始\n\tvar prevPage = currentPage-1//上一页\n\tvar 下一页Page = currentPage+1//下一页\n\tvar countPage = 4//共多少页\n\t//document.write("共"+countPage+"页 ");\n\t\n\t//循环\n\tvar num = 17;\n\tfor(var i=0+(currentPage-1-(currentPage-1)%num) ; i<=(num+(currentPage-1-(currentPage-1)%num))&&(i<countPage) ; i++){\n\t\tif(countPage >1){\n\t\t\tif(currentPage==i)\n\t\t\t\tdocument.write("【<span style=\\"color:#FF0000;\\" class=\\"hui14_30_h\\">"+(i+1)+"</span>】 ");\n\t\t\telse if(i==0)\n\t\t\t\tdocument.write("<a href=\\"t20090818_3955570.html\\" class=\\"hui14_30_h\\">【"+(i+1)+"】</a> ");\n\t\t\telse\n\t\t\t\tdocument.write("<a href=\\"t20090818_3955570"+"_" + i + "."+"html\\" class=\\"hui14_30_h\\">【"+(i+1)+"】</a> ");\n\t\t}\t\n\t}\n\t\n\tdocument.write("<br><br>");\n\t//设置上一页代码\n\tif(countPage>1&¤tPage!=0&¤tPage!=1)\n\t\tdocument.write("<a href=\\"t20090818_3955570"+"_" + prevPage + "."+"html\\"><span style=\\"color:#0033FF;font-weight:bold\\">上一页</span></a> ");\n\telse if(countPage>1&¤tPage!=0&¤tPage==1)\n\t\tdocument.write("<a href=\\"t20090818_3955570.html\\"><span style=\\"color:#0033FF;font-weight:bold\\">上一页</span></a> ");\n\t//else\n\t//\tdocument.write("上一页 ");\n\t\n\t\n\t//设置下一页代码 \n\tif(countPage>1&¤tPage!=(countPage-1))\n\t\tdocument.write("<a href=\\"t20090818_3955570"+"_" + 下一页Page + "."+"html\\" ><span style=\\"color:#0033FF;font-weight:bold\\">下一页</span></a> ");\n\t//else\n\t//\tdocument.write("下一页 ");\n\t\t\t\t\t \n\t'
# countPage = int(''.join(scripts).split('countPage = ')\
# [1].split('//')[0])
# countPage
countPage = int(scripts.text.split('countPage = ')[1].split('//')[0])
countPage
4
import sys
def flushPrint(s):
sys.stdout.write('\r')
sys.stdout.write('%s' % s)
sys.stdout.flush()
def crawler(url_i):
content = requests.get(url_i)
content.encoding = 'utf8'
content = content.text
soup = BeautifulSoup(content, 'html.parser')
year = soup.find('span', {'class', 'huang16c'}).text[:4]
year = int(year)
report = ''.join(s.text for s in soup('p'))
# 找到分页信息
scripts = soup.find_all('script')
countPage = int(''.join(scripts[1]).split('countPage = ')[1].split('//')[0])
if countPage == 1:
pass
else:
for i in range(1, countPage):
url_child = url_i.split('.html')[0] +'_'+str(i)+'.html'
content = requests.get(url_child)
content.encoding = 'gb18030'
content = content.text
soup = BeautifulSoup(content, 'html.parser')
report_child = ''.join(s.text for s in soup('p'))
report = report + report_child
return year, report
# 抓取50年政府工作报告内容
reports = {}
for link in hyperlinks:
year, report = crawler(link)
flushPrint(year)
reports[year] = report
1954
with open('../data/gov_reports1954-2019.txt', 'w', encoding = 'utf8') as f:
for r in reports:
line = str(r)+'\t'+reports[r].replace('\n', '\t') +'\n'
f.write(line)
import pandas as pd
df = pd.read_table('../data/gov_reports1954-2019.txt', names = ['year', 'report'])
df[-5:]
| year | report | |
|---|---|---|
| 45 | 2015 | 国务院总理 李克强 各位代表： 现在，我代表国务院，向大会报告政府工作，请予审议，并... |
| 46 | 2016 | 政府工作报告 |
| 47 | 2017 | 各位代表： 现在，我代表国务院，向大会报告政府工作，请予审议，并请全国政协各位委员提出... |
| 48 | 2018 | 各位代表： 现在，我代表国务院，向大会报告过去五年政府工作，对今年工作提出建议，请予审... |
| 49 | 2019 | 新华社北京3月16日电 政府工作报告 ——2019年3月5日在第十三届全国人民代表... |