In [9]:
import os
os.chdir('/Users/muneebalam/Desktop/ipython/')
import os.path
if not os.path.exists('/Users/muneebalam/Desktop/ipython/chilcot/'):
    os.mkdir('/Users/muneebalam/Desktop/ipython/chilcot/')
os.chdir('/Users/muneebalam/Desktop/ipython/chilcot/')
import urllib.request
urlbase = 'http://www.iraqinquiry.org.uk'
from PyPDF2 import PdfFileReader
In [2]:
def download():
    r = urllib.request.urlopen('http://www.iraqinquiry.org.uk/the-report/')
    page = r.read().decode('utf-8')
    r.close()
    
    page = page.split('.pdf')
    for i in range(len(page)-1):
        url = '{0:s}{1:s}.pdf'.format(urlbase, page[i][page[i].rfind('href="')+6:])
        savefile = '/Users/muneebalam/Desktop/ipython/chilcot/' + url[url.rfind('/')+1:]
        print(i+1, ' of ', len(page)-1, url)
        if not os.path.exists(savefile):
            response = urllib.request.urlopen(url)
            data = response.read()
            response.close()

            w = open(savefile, 'wb')
            w.write(data)
            w.close()
In [12]:
#download()
files = ['/Users/muneebalam/Desktop/ipython/chilcot/' + x for x in os.listdir('/Users/muneebalam/Desktop/ipython/chilcot/')]
In [29]:
def parse(file):
    r = PdfFileReader(file)
    numpages = r.getNumPages()
    
    fbase = file[:file.rfind('.')]
    alltext = []
    for i in range(numpages):
        try:
            text = r.getPage(i).extractText()
            text = text.replace('.\n', '. ').replace('\n', '').replace('  ', ' ')
            alltext.append(text)
        except Exception as e:
            print(file, 'page', i, e, e.args)
    text = '\n'.join(alltext)
    w = open('{0:s}-{1:d}.txt'.format(fbase, i), 'w')
    w.write(text)
    w.close()
    print('Done with', file)
In [30]:
for file in files:
    parse(file)
Done with /Users/muneebalam/Desktop/ipython/chilcot/2016-09-06-sir-john-chilcots-public-statement.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_executive-summary.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_introduction.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-101.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-102.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-103.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-104.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-11.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-111.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-112.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-12.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-121.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-122.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-131.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-132.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-141.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-142.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-15-1.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-152.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-161.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-162.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-163.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-164.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-170.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-20.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-31.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-32.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-33.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-34.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-35.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-36.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-37.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-38.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-40.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-41.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-42.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-43.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-44.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-50.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-61.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-62.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-63.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-64.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-65.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-70.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-80.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-91.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-92.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-93.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-94.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-95.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-96.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-97.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section-98.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section_annex-1.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section_annex-2.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section_annex-3.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section_annex-4.pdf
Done with /Users/muneebalam/Desktop/ipython/chilcot/the-report-of-the-iraq-inquiry_section_annex-5.pdf