This is a similar experiment as in Experiment 3, but with the British Periodicals data from the Stanford Literary Lab.
import spacy
import pandas as pd
%matplotlib inline
from ast import literal_eval
import numpy as np
import re
import json
from nltk.corpus import names
from collections import Counter
from matplotlib import pyplot as plt
plt.rcParams["figure.figsize"] = [16, 6]
plt.style.use('ggplot')
nlp = spacy.load('en')
with open('../middlemarch.txt') as f:
mm = f.read()
textALength = len(mm)
# Get chapter locations
chapterMatches = re.finditer('PRELUDE|CHAPTER|FINALE', mm)
chapterLocations = [match.start() for match in chapterMatches]
chapterLocations.append(textALength) # Add one to account for last chunk.
len(chapterLocations)
89
def getChapters(text):
chapters = []
for i, loc in enumerate(chapterLocations):
if i != len(chapterLocations)-1:
chapter = mm[loc:chapterLocations[i+1]]
chapters.append(chapter)
return chapters
chapters = getChapters(mm)
chapterLengths = [len(nlp(chapter, tag=False, parse=False, entity=False)) for chapter in chapters]
chapterLengthsSeries = pd.Series(chapterLengths)
chapterLengthsSeries.plot(kind='bar', title='Chapter Lengths')
<matplotlib.axes._subplots.AxesSubplot at 0x7f5a79788c18>
with open('../txt/e4.json') as f:
rawData = f.read()
df = pd.read_json(rawData)
df
Locations in A | Locations in B | contributor_first_name | contributor_last_name | contributor_original_form | contributor_person_name | contributor_role | language_code | numMatches | object_type | ... | publication_qualifier | publication_title | record_id | record_title | source_type | text | year | Decade | Quoted Words | Locations in A with Wordcounts | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | [] | [] | None | None | None | None | None | ENG | 0 | Review | ... | London | The Athenaeum | 8862020 | Essays. | Historical Periodicals | Essay 8. LITERATURE By George Eliot. (Blackwoo... | 1884 | 1880 | 0 | [] |
1 | [] | [] | None | None | Dickens, Charles Culliford Boz 1837-1896 | None | Editor | ENG | 0 | Feature | ... | London | All the year round | 8182372 | CHRONICLES OF ENGLISH COUNTIES. | Historical Periodicals | CHRONICLES OF ENGLISH COUNTIES.' WARIVICK... | 1884 | 1880 | 0 | [] |
2 | [[18417, 18603], [35109, 35254], [929122, 9293... | [[26000, 26184], [33676, 33819], [35266, 35510]] | Peter | Bayne | Bayne, Peter | Peter Bayne | WellesleyAttribution | ENG | 3 | Feature | ... | Edinburgh | Blackwood's Edinburgh magazine | 8243995 | SHAKESPEARE AND GEORGE ELIOT. | Historical Periodicals | SHAKESPEARE AND GEORGE ELIOT. Shakespeare has ... | 1883 | 1880 | 124 | [([18417, 18603], 41), ([35109, 35254], 28), (... |
3 | [] | [] | None | None | V W | None | Author | ENG | 0 | Review | ... | London | The Athenaeum | 8975729 | THE ANATOMY OF FICTION | Historical Periodicals | THE ANATOMY OF FICTION L Materials and Methods... | 1919 | 1910 | 0 | [] |
4 | [] | [] | Henry | Reynolds | Reynolds, Henry Robert | Henry Robert Reynolds | Editor | ENG | 0 | Review | ... | London | The British quarterly review | 8288277 | Parted Lives. | Historical Periodicals | Parted Lives. By Mrs. J. K. Spender. Hurst and... | 1873 | 1870 | 0 | [] |
5 | [] | [] | H | LAWRENNY | LAWRENNY, H | H LAWRENNY | Author | ENG | 0 | Review | ... | London | The Academy, 1869-1902, 0269-333X | 6546857 | Middlemarch. | Historical Periodicals | General Literature. Middlemarch. A Study of Pr... | 1873 | 1870 | 0 | [] |
6 | [] | [] | Ernest | Remnant | Remnant, Ernest | Ernest Remnant | Editor | ENG | 0 | Review | ... | London | The English review, 1908-1937 | 2441325 | GEORGE ELIOT: A STUDY. | Historical Periodicals | BIOGRAPHY AND MEMOIRS. GEORGE ELIOT: A STUDY. ... | 1930 | 1930 | 0 | [] |
7 | [] | [] | None | None | None | None | None | ENG | 0 | Article | ... | London | The Sketch | 1638085810 | SMALL TALK of the WEEK | Historical Periodicals | OUR KING, with the wonderful tact whi... | 1904 | 1900 | 0 | [] |
8 | [] | [] | None | None | None | None | None | ENG | 0 | Advertisement | ... | London | Saturday review of politics, literature, scien... | 9672239 | Advertisement | Historical Periodicals | SAMPSON LOW, MARSTON & CO/S NEW BOOKS FOR ... | 1895 | 1890 | 0 | [] |
9 | [] | [] | None | None | None | None | None | ENG | 0 | Review | ... | London | London quarterly review, Oct. 1862-Jan. 1932 | 2549048 | A History of Modern England. | Historical Periodicals | A History of Modern England. By Herbert Paul. ... | 1905 | 1900 | 0 | [] |
10 | [] | [] | None | None | None | None | None | ENG | 0 | Advertisement | ... | London | The Sphere | 1693114534 | Advertisements | Historical Periodicals | HOTELS TO VISIT. BIDEFORD.-- ... | 1907 | 1900 | 0 | [] |
11 | [] | [] | None | None | THE EDITOR | None | Author | ENG | 0 | Feature | ... | London | The Argosy : a magazine of tales, travels, ess... | 3122681 | IN THE WIND | Historical Periodicals | IN THE WIND -)o very successtui, said Mac in e... | 1900 | 1900 | 0 | [] |
12 | [] | [] | John | Chapman | Chapman, John, 1822-1894 | John Chapman | Editor | ENG | 0 | Review | ... | London | Westminster review, Jan. 1852-Jan. 1914 | 4461375 | HISTORY AND BIOGRAPHY. | Historical Periodicals | HISTORY AND BIOGRAPHY. PROFESSOR STUBBS' ... | 1874 | 1870 | 0 | [] |
13 | [[1282, 2098], [1542, 2123], [2850, 3005], [69... | [[1134, 1950], [1394, 1975], [3256, 3411], [36... | None | None | None | None | None | ENG | 13 | Review | ... | London | The Athenaeum | 8778938 | Middlemarch: a Study of Provincial Life. Book ... | Historical Periodicals | Middlemarch : a Stud// of Provincial Life. Boo... | 1871 | 1870 | 815 | [([1282, 2098], 158), ([1542, 2123], 111), ([2... |
14 | [] | [] | None | None | None | None | None | ENG | 0 | Feature | ... | London | Saturday review of politics, literature, scien... | 9180204 | THE OLD AGE OF JOHNNIE EAMES. | Historical Periodicals | THE OLD AGE OF JOHNN1E EAMES. LIVING in the wo... | 1906 | 1900 | 0 | [] |
15 | [] | [] | Edward | Dowden | DOWDEN, EDWARD | Edward Dowden | Author | ENG | 0 | Feature | ... | London | The Contemporary review, 1866-1900 | 6896624 | THE SCIENTIFIC MOVEMENT AND LITERATURE. | Historical Periodicals | THE SCIENTIFIC MOVEMENT AND LITERATURE. A NY i... | 1877 | 1870 | 0 | [] |
16 | [] | [] | Margaret | Oliphant | M O W O | Margaret Oliphant | Author | ENG | 0 | Review | ... | Edinburgh | Blackwood's Edinburgh magazine | 8251993 | THE ANTI-MARRIAGE LEAGUE. | Historical Periodicals | THIE ANTI-MARRIAGE LEAGUE. NOVELS have become,... | 1896 | 1890 | 0 | [] |
17 | [] | [] | EDWARD | BERDOE | BERDOE, EDWARD | EDWARD BERDOE | Author | ENG | 0 | Feature | ... | London | The Universal review | 4211534 | The College of Physicians and the Medical Press | Historical Periodicals | zteoeftstt~ of taos{icias anu tot eatbkd tMes ... | 1888 | 1880 | 0 | [] |
18 | [] | [] | None | None | None | None | None | ENG | 0 | Article | ... | London | The Graphic | 1693145986 | BISHOP THIRLWALL | Historical Periodicals | The volumes containing Bishop ... | 1881 | 1880 | 0 | [] |
19 | [] | [] | James | Cotton | Cotton, James Sutherland, 1847-1918 | James Sutherland Cotton | Editor | ENG | 0 | Advertisement | ... | London | The Academy, 1869-1902, 0269-333X | 6726352 | Advertisement | Historical Periodicals | MESSRS. WILLIAM BLACKWOOD & SONS' NEW... | 1885 | 1880 | 0 | [] |
20 | [] | [] | Charles | Hind | Hind, Charles Lewis, 1862-1927 | Charles Lewis Hind | Editor | ENG | 0 | Feature | ... | London | The Academy and literature, 1902-1905 | 6782246 | COMPARATIVE CHRONOLOGICAL TABLE OF ENGLISH AND... | Historical Periodicals | COMPARATIVE CHRONOLOGICAL TARLI OF EHOLISH AND... | 1902 | 1900 | 0 | [] |
21 | [] | [] | None | None | None | None | None | ENG | 0 | Review | ... | London | The Academy, 1905-1910 | 6702747 | PAULINE EPISTLES | Historical Periodicals | LITERATURE PAULINE EPISTLES Stray Leaves, By H... | 1906 | 1900 | 0 | [] |
22 | [] | [] | None | None | None | None | None | ENG | 0 | Feature | ... | London / New York | The Musical times, 1904-1995 | 7765488 | TCHAIKOVSKY. | Historical Periodicals | TCHAIKOVSKY 1840-1893- Twelve years have come ... | 1905 | 1900 | 0 | [] |
23 | [] | [] | John | Chapman | Chapman, John, 1822-1894 | John Chapman | Editor | ENG | 0 | Review | ... | London | Westminster review, Jan. 1852-Jan. 1914 | 4466529 | ART. III.--DIDEROT AND THE ART OF ACTING. | Historical Periodicals | ART. II.-DIDEROT AND THE ART OF ACTING. "... | 1887 | 1880 | 0 | [] |
24 | [] | [] | None | None | L R F O | None | Author | ENG | 0 | Review | ... | London | The Speaker : the liberal review | 6636414 | FICTION. | Historical Periodicals | FICTION. The Alien. By F. F. Montresor. London... | 1901 | 1900 | 0 | [] |
25 | [] | [] | John | Blackwood | Blackwood, John | John Blackwood | Editor | ENG | 0 | Review | ... | Edinburgh | Blackwood's Edinburgh magazine | 8254762 | NEW BOOKS. | Historical Periodicals | NEW BOOKS. ONE of the greatest of recent natio... | 1875 | 1870 | 0 | [] |
26 | [] | [] | S | Stepniak | STEPNIAK, S | S Stepniak | Author | ENG | 0 | Feature | ... | London | The New review | 6947284 | NIHILISM: AS IT IS. | Historical Periodicals | NIHiLISM: AS IT IS. (A REPIY.) T HE January nu... | 1894 | 1890 | 0 | [] |
27 | [] | [] | None | None | None | None | None | ENG | 0 | Advertisement | ... | London | The Athenaeum | 8803334 | Advertisement | Historical Periodicals | BOOKS FOR THE SEA-SIDE AND THE COUNTRY. MANY C... | 1873 | 1870 | 0 | [] |
28 | [] | [] | John | Hales | HALES, JOHN W | John W Hales | Author | ENG | 0 | Review | ... | London | The Gentleman's magazine | 8295746 | VICTORIAN LITERATURE. | Historical Periodicals | VICTORIAN LITERA TURE. PART I. 1N studying the... | 1888 | 1880 | 0 | [] |
29 | [] | [] | None | None | INDICATOR | None | Author | ENG | 0 | Review | ... | London | The Athenaeum | 8953909 | Adventures in Books. | Historical Periodicals | Adventures in Books. RICHARD HOLT HUTTON, whom... | 1918 | 1910 | 0 | [] |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
604 | [] | [] | A | Loffelt | Loffelt, A C | A C Loffelt | Author | ENG | 0 | Feature | ... | London | The Athenaeum | 8846457 | HOLLAND. | Historical Periodicals | p" HOLLAND. he Among the many works writt... | 1874 | 1870 | 0 | [] |
605 | [] | [] | None | None | C. K. S | None | Author | ENG | 0 | Article | ... | London | The Sphere | 1689026185 | A LITERARY LETTER | Historical Periodicals | Journalism and literature are always ... | 1907 | 1900 | 0 | [] |
606 | [] | [] | Charles | Hind | Hind, Charles Lewis, 1862-1927 | Charles Lewis Hind | Editor | ENG | 0 | General Information | ... | London | The Academy, 1869-1902, 0269-333X | 6805440 | Our Literary Competitions. | Historical Periodicals | Our Literary Competitions. Result of No. 22. 4... | 1899 | 1890 | 0 | [] |
607 | [] | [] | None | None | None | None | None | ENG | 0 | Advertisement | ... | London | The Athenaeum | 9038642 | Advertisement | Historical Periodicals | INVESTMENT for GENTLEMEN.?WANTED, two ot three... | 1873 | 1870 | 0 | [] |
608 | [] | [] | Paul | Chapman | CHAPMAN, PAUL | Paul Chapman | Author | ENG | 0 | Feature | ... | London | The Nineteenth century and after : a monthly r... | 2659976 | A REMINISCENCE OF COVENTRY PATMORE | Historical Periodicals | A REMINISCENCE OF CO VENTR Y PA TMIORE IN 1870... | 1904 | 1900 | 0 | [] |
609 | [] | [] | None | None | None | None | None | ENG | 0 | Review | ... | London | Saturday review of politics, literature, scien... | 9348101 | "The Library: a Quarterly Review of Bibli... | Historical Periodicals | " The Library: a Quarterly Review of Bibl... | 1907 | 1900 | 0 | [] |
610 | [] | [] | None | None | None | None | None | ENG | 0 | Feature | ... | London | Saturday review of politics, literature, scien... | 9447449 | NOVEL-READING. | Historical Periodicals | NOVEL-READING. THE question, What kind of lite... | 1874 | 1870 | 0 | [] |
611 | [] | [] | None | None | G M T | None | Author | ENG | 0 | Correspondence | ... | London | The Athenaeum | 8706228 | NOTES FROM THE UNITED STATES. | Historical Periodicals | NOTES FROM THE UNITED STATES. Boston, March 24... | 1873 | 1870 | 0 | [] |
612 | [] | [] | William | Stevens | Stevens, William | William Stevens | Editor | ENG | 0 | Back Matter | ... | London | The Leisure hour, Jan. 1877-Oct. 1903 | 3710469 | When the Lamps are Lit. | Historical Periodicals | VaIben tbe lamps are lit. PRIZE COMPETITIONS. ... | 1896 | 1890 | 0 | [] |
613 | [] | [] | None | None | None | None | None | ENG | 0 | General Information | ... | London | Saturday review of politics, literature, scien... | 9131440 | THIS WEEK'S BOOKS. | Historical Periodicals | THIS WEEK'S BOOKS. Art. The Print-Collect... | 1901 | 1900 | 0 | [] |
614 | [] | [] | Leigh | Hunt | Hunt, Leigh, 1784-1859 | Leigh Hunt | Editor | ENG | 0 | Review | ... | London | Examiner | 8517863 | MR MORRIS'S MORALITY. | Historical Periodicals | MR MORRIS'S MORALITY. Lore is Enough; or,... | 1872 | 1870 | 0 | [] |
615 | [] | [] | None | None | None | None | None | ENG | 0 | Review | ... | London | Saturday review of politics, literature, scien... | 9233868 | The Passing Hour. | Historical Periodicals | The Passing Hour* By W. L. Courtney. Hutch in-... | 1925 | 1920 | 0 | [] |
616 | [] | [] | Violet | Fane | FANE, VIOLET | Violet Fane | Author | ENG | 0 | Fiction | ... | London | Time | 6426792 | SOPHY, OR THE ADVENTURES OF A SAVAGE. | Historical Periodicals | SOPHY, OR TIIE ADVENTURES OF A SAVAGE. By VIOL... | 1880 | 1880 | 0 | [] |
617 | [] | [] | None | None | None | None | None | ENG | 0 | Review | ... | Edinburgh | The Edinburgh review, 1802-1929 | 6788527 | ART. VI.-Daniel Deronda. | Historical Periodicals | ART. VI.-Daniel Deronda. 4 vols. 12mo. By GEOR... | 1876 | 1870 | 0 | [] |
618 | [] | [] | Alice | Meynell | MEYNELL, ALICE | Alice Meynell | Author | ENG | 0 | Review | ... | London | Merry England | 5830548 | George Eliot. | Historical Periodicals | George Eliot. A S the most concentrated in man... | 1885 | 1880 | 0 | [] |
619 | [] | [] | None | None | None | None | None | ENG | 0 | Feature | ... | London | The Athenaeum | 8819961 | PICTURES AND SCULPTURES FOR THE ACADEMY. | Historical Periodicals | PICTURES asd sculptures for the academy. Sir J... | 1893 | 1890 | 0 | [] |
620 | [] | [] | None | None | None | None | None | ENG | 0 | Advertisement | ... | London | The Speaker : the liberal review | 6640530 | Advertisement | Historical Periodicals | I Messrs. Wm. BL ALCI(WOOD & SONS' PU... | 1893 | 1890 | 0 | [] |
621 | [] | [] | None | None | None | None | None | ENG | 0 | General Information | ... | London | The Contemporary review, 1866-1900 | 6953182 | CONTENTS OF VOLUME XXIX. | Historical Periodicals | CONTENTS OF VOLUME XXIX. DECEMBER, 1876. PAGel... | 1876 | 1870 | 0 | [] |
622 | [] | [] | None | None | None | None | None | ENG | 0 | Review | ... | London | The Athenaeum | 8751705 | George Eliot's Life, as related in her Le... | Historical Periodicals | George Eliot's Life, as related in her Le... | 1885 | 1880 | 0 | [] |
623 | [] | [] | Leigh | Hunt | Hunt, Leigh, 1784-1859 | Leigh Hunt | Editor | ENG | 0 | Review | ... | London | Examiner | 8542920 | THE AUTUMN NOVELS. | Historical Periodicals | THE AUTUMN NOVELS. Stephen the Schoolmaster. A... | 1879 | 1870 | 0 | [] |
624 | [] | [] | None | None | Nicholl, Sir William Robertson, 1851-1923 | None | Editor | ENG | 0 | Review | ... | London | The Bookman | 3196648 | A NEW CHRISTIAN YEAR. | Historical Periodicals | A NEW CHRISTIAN YEAR.* Miss Stoddart has here ... | 1920 | 1920 | 0 | [] |
625 | [] | [] | None | None | None | None | None | ENG | 0 | Review | ... | London | The Athenaeum | 8720800 | LITERATURE | Historical Periodicals | LITERATURE .T/iprseasiWS of Tbeolhra-.ties Sue... | 1879 | 1870 | 0 | [] |
626 | [] | [] | Leigh | Hunt | R | Leigh Hunt | Author | ENG | 0 | Review | ... | London | Examiner | 8572650 | OLD KENSINGTON. | Historical Periodicals | OLD KENSINGTON. Old Kensington. By Miss Tbacke... | 1873 | 1870 | 0 | [] |
627 | [[834834, 834913], [835093, 835391], [909623, ... | [[816, 891], [1063, 1355], [4636, 4743], [4892... | Leigh | Hunt | Hunt, Leigh, 1784-1859 | Leigh Hunt | Editor | ENG | 8 | Review | ... | London | Examiner | 8516886 | MIDDLEMARCH.-BOOK IV. | Historical Periodicals | LITERARY. --V KWDLMcBSK WV. q - I I - _NM A BY... | 1872 | 1870 | 776 | [([834834, 834913], 16), ([835093, 835391], 79... |
628 | [] | [] | C | Sargisson | SARGISSON, C S | C S Sargisson | Author | ENG | 0 | Feature | ... | London | The English illustrated magazine | 3465358 | WHERE A GENIUS WAS NURTURED. | Historical Periodicals | WHERE A GENIUS WAS NURTURED. The Influence on ... | 1913 | 1910 | 0 | [] |
629 | [] | [] | None | None | None | None | None | ENG | 0 | Advertisement | ... | London | Saturday review of politics, literature, scien... | 9495700 | Advertisement | Historical Periodicals | Monthly, ls. MACMILLAN'S MAGAZINE, '... | 1880 | 1880 | 0 | [] |
630 | [[272169, 272252], [500568, 501119]] | [[284, 367], [4793, 5343]] | None | None | None | None | None | ENG | 2 | Review | ... | London | The Athenaeum | 8688126 | Middlemarch. | Historical Periodicals | Middlemarch. SATURDAY, MARCH 30, 1S72. LITERAT... | 1872 | 1870 | 132 | [([272169, 272252], 15), ([500568, 501119], 117)] |
631 | [] | [] | Thomas | Bradfield | BRADFIELD, THOMAS | Thomas Bradfield | Author | ENG | 0 | Review | ... | London | Westminster review, Jan. 1852-Jan. 1914 | 4472311 | SUGGESTIVE FEATURES OF OUR LAST SIXTY YEARS' L... | Historical Periodicals | SUGGESTIVE FEATURES OF OUR LAST SIXTY YEARS&ap... | 1897 | 1890 | 0 | [] |
632 | [] | [] | None | None | None | None | None | ENG | 0 | Feature | ... | London | The Athenaeum | 8957570 | CHRISTMAS BOOKS. | Historical Periodicals | CHRISTMAS BOOKS. Mr. W. J. Stillman, who is ac... | 1898 | 1890 | 0 | [] |
633 | [] | [] | None | None | None | None | None | ENG | 0 | Advertisement | ... | London | The Graphic | 1693094485 | Advertisements | Historical Periodicals | AD VER TISEMENTS. '"T'HEATRH ... | 1872 | 1870 | 0 | [] |
634 rows × 21 columns
df.publication_qualifier.value_counts()
London 599 Edinburgh 24 London / New York 2 Calcutta 2 Dublin 2 Name: publication_qualifier, dtype: int64
df['Decade'] = df['year'] - (df['year'] % 10)
# df['Locations in A'] = df['matches'].apply(lambda x: x[1])
# df['NumMatches'] = df['matches'].apply(lambda x: x[0])
sum([len(item) for item in df['Locations in A'].values])
300
def diachronicAnalysis(df, decades=(1950, 2020), bins=chapterLocations):
decades = np.arange(decades[0], decades[1], 10)
# Make a dictionary of decades.
# Values are a list of locations.
decadeDict = {}
for i, row in df.iterrows():
decade = row['Decade']
locationsAndWordcounts = row['Locations in A with Wordcounts']
if decade not in decadeDict:
decadeDict[decade] = locationsAndWordcounts
else:
decadeDict[decade] += locationsAndWordcounts
# Grab the beginnings of quotes.
decadeStartsWeights = {decade: [(item[0][0], item[1])
for item in loc]
for decade, loc in decadeDict.items()}
decadesBinned = {decade:
np.histogram([loc[0] for loc in locations],
bins=bins,
weights=[loc[1] for loc in locations],
range=(0, textALength))[0]
for decade, locations in decadeStartsWeights.items()
if decade in decades}
decadesDF = pd.DataFrame(decadesBinned).T
#Normalize
decadesDF = decadesDF.div(decadesDF.max(axis=1), axis=0)
return decadesDF
def countWords(locRange):
""" Counts words in middlemarch, given character ranges. """
doc = nlp(mm[locRange[0]:locRange[1]], tag=False, parse=False, entity=False)
return len(doc)
def totalWords(locRangeSet):
""" Counts total words in a list of location ranges. """
return sum([countWords(locRange) for locRange in locRangeSet])
def countsPerSet(locRangeSet):
""" Returns an augmented location range set that includes word counts. """
return [(locRange, countWords(locRange))
for locRange in locRangeSet]
def synchronicAnalysis(df, bins=chapterLocations):
locs = df['Locations in A'].values
locCounts = [(loc, countWords(loc)) for locSet in locs
for loc in locSet]
starts = [loc[0][0] for loc in locCounts]
counts = [loc[1] for loc in locCounts]
binned = np.histogram(starts, bins=bins,
weights=counts, range=(0, textALength))
binnedDF = pd.Series(binned[0])
return binnedDF
def plotDiachronicAnalysis(df):
ylabels = [str(int(decade)) for decade in df.index] #+ ['2020']
plt.pcolor(df, cmap='gnuplot')
plt.yticks(np.arange(len(df.index)+1), ylabels)
plt.gca().invert_yaxis()
plt.ylabel('Decade')
plt.xlabel('Chapter')
# plt.title("Frequency of Quotations from George Eliot's Middlemarch in Criticism, By Decade")
plt.gca().set_xlim((0, len(df.T)))
plt.colorbar(ticks=[])
plt.show()
def plotSynchronicAnalysis(s):
ax = s.plot(kind='bar')
ax.set_xlabel('Chapter')
ax.set_ylabel('Number of Words Quoted')
df['Quoted Words'] = df['Locations in A'].apply(totalWords)
df['Locations in A with Wordcounts'] = df['Locations in A'].apply(countsPerSet)
plotSynchronicAnalysis(synchronicAnalysis(df))
# Adjusted for the number of words in each chapter
ax = (synchronicAnalysis(df) / chapterLengthsSeries).plot(kind='bar')
ax.set_xlabel('Chapter')
ax.set_ylabel('Words Quoted, Normalized')
<matplotlib.text.Text at 0x7f5a2a556668>
plotDiachronicAnalysis(diachronicAnalysis(df, decades=(1870, 1910), bins=chapterLocations))
# Look at the top journals.
journals = df['publication_title'].value_counts()
journals.plot(kind='bar')
<matplotlib.axes._subplots.AxesSubplot at 0x7f5a24b83710>
Compare journals.
journalDict = {title: synchronicAnalysis(df.loc[df['publication_title'] == title]) for title in journals.index}
byJournal = pd.DataFrame(journalDict).T
# byJournal
byJournal
0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | ... | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
All the year round | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Belgravia : a London magazine | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Blackwood's Edinburgh magazine | 0.0 | 756.0 | 198.0 | 0.0 | 0.0 | 112.0 | 0.0 | 198.0 | 0.0 | 55.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 227.0 | 0.0 | 0.0 |
Bow bells : a magazine of general literature and art for family reading | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Calcutta Review | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Chambers's journal of popular literature, science and arts, Jan. 1854- Nov. 1897 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Cosmopolis | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Dublin University Magazine, 1833-1877 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Examiner | 501.0 | 204.0 | 0.0 | 0.0 | 0.0 | 676.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Farm & Country | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Fortnightly review, May 1865-June 1934 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Fraser's magazine | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Golden hours : an illustrated magazine for any time and all times, Jan. 1880-Dec. 1881 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Good words | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Illustrated Sporting and Dramatic News | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Judy : or The London serio-comic journal | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
London Life | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
London journal | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
London quarterly review, Oct. 1862-Jan. 1932 | 2116.0 | 0.0 | 0.0 | 16.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
London society : an illustrated magazine of light and amusing literature for the hours of relaxation, Jan. 1862-Dec. 1886 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 111.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 69.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Longman's magazine, 1882-1905 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Merry England | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Musical opinion and music trade review | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Musical standard | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Novel review | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Our corner | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Saturday review of politics, literature, science and art | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Temple bar : a London magazine for town and country readers, Dec. 1860-Dec. 1881 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Temple bar : a monthly magazine for town and country readers, Jan. 1902-Dec. 1905 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Temple bar : with which is incorporated Bentley's miscellany, Jan. 1882-May 1898 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
The National review | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The New century review | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The New monthly magazine, Jan. 1853-Dec. 1881 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The New quarterly magazine | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The New review | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Nineteenth century and after : a monthly review | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 127.0 | 0.0 | 0.0 |
The Nineteenth century: a monthly review, Mar. 1877-Dec. 1900 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Non-conformist musical journal : a monthly record and review devoted to the interests of worship music in the non-conformist churches, 1888-1905 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Outlook | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Pall mall magazine, May 1893-Sept. 1914 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Powder magazine | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Practical teacher | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Quarterly review | 0.0 | 34.0 | 27.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Review of reviews | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Saint Pauls magazine | 0.0 | 39.0 | 173.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Scottish review, 1882-1920 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Sketch | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Speaker : the liberal review | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Sphere | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The St. James's magazine | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Sunday at home : a family magazine for Sabbath reading | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Tatler | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Tatler and Bystander | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Theological review : a journal of religious thought and life | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Universal review | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Wesleyan-Methodist magazine | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The Windsor magazine : an illustrated monthly for men and women | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
The orchestra, Oct. 1863-Mar. 1881 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Time | 0.0 | 0.0 | 0.0 | 64.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
Westminster review, Jan. 1852-Jan. 1914 | 24.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 38.0 |
87 rows × 88 columns
cutoff = 500 # Number of words
topJournals = byJournal.loc[byJournal.sum(axis=1) >= cutoff]
otherJournals = byJournal.loc[byJournal.sum(axis=1) <= cutoff]
topJournals.loc['Other'] = otherJournals.sum()
/usr/lib/python3.6/site-packages/ipykernel/__main__.py:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
topJournals.sum(axis=1)
Blackwood's Edinburgh magazine 2713.0 Examiner 6089.0 London quarterly review, Oct. 1862-Jan. 1932 3970.0 The Athenaeum 2366.0 The British quarterly review 2228.0 The Cornhill magazine 519.0 The Edinburgh review, 1802-1929 3440.0 The Graphic 624.0 The orchestra, Oct. 1863-Mar. 1881 2141.0 Other 2821.0 dtype: float64
ax = topJournals.T.plot(kind='bar', stacked=True, colormap='nipy_spectral')
ax.set_ylabel = "Chapter"
ax.set_xlabel = "Number of Words Quoted"