import nltk
with open('book.txt', 'r') as file:
text = file.readlines()
Let's take a smaller chunk from the text:
# using a list comprehension to simplify iterating over the the text structure
snippet = " ".join(block.strip() for block in text[175:200])
snippet
'"Do none suggest themselves? You know my methods. Apply them!" "I can only think of the obvious conclusion that the man has practised in town before going to the country." "I think that we might venture a little farther than this. Look at it in this light. On what occasion would it be most probable that such a presentation would be made? When would his friends unite to give him a pledge of their good will? Obviously at the moment when Dr. Mortimer withdrew from the service of the hospital in order to start in practice for himself. We know there has been a presentation. We believe there has been a change from a town hospital to a country practice. Is it, then, stretching our inference too far to say that the presentation was on the occasion of the change?" "It certainly seems probable." "Now, you will observe that he could not have been on the staff of the hospital, since only a man well-established in a London practice could hold such a position, and such a one would not drift into the country. What was he, then? If he was in the hospital and yet not on the staff he could only have been a house-surgeon or a house-physician--little more than a senior student. And he left five years ago--the date is on the stick. So'
# alternative with for-loop
other_snippet = []
for block in text[175:200]:
other_snippet.append(block.strip())
other_snippet = " ".join(other_snippet)
other_snippet
'"Do none suggest themselves? You know my methods. Apply them!" "I can only think of the obvious conclusion that the man has practised in town before going to the country." "I think that we might venture a little farther than this. Look at it in this light. On what occasion would it be most probable that such a presentation would be made? When would his friends unite to give him a pledge of their good will? Obviously at the moment when Dr. Mortimer withdrew from the service of the hospital in order to start in practice for himself. We know there has been a presentation. We believe there has been a change from a town hospital to a country practice. Is it, then, stretching our inference too far to say that the presentation was on the occasion of the change?" "It certainly seems probable." "Now, you will observe that he could not have been on the staff of the hospital, since only a man well-established in a London practice could hold such a position, and such a one would not drift into the country. What was he, then? If he was in the hospital and yet not on the staff he could only have been a house-surgeon or a house-physician--little more than a senior student. And he left five years ago--the date is on the stick. So'
whole_text = " ".join(block.strip() for block in text)
whole_text[5000:7500]
' towards the truth. Not that you are entirely wrong in this instance. The man is certainly a country practitioner. And he walks a good deal." "Then I was right." "To that extent." "But that was all." "No, no, my dear Watson, not all--by no means all. I would suggest, for example, that a presentation to a doctor is more likely to come from a hospital than from a hunt, and that when the initials \'C.C.\' are placed before that hospital the words \'Charing Cross\' very naturally suggest themselves." "You may be right." "The probability lies in that direction. And if we take this as a working hypothesis we have a fresh basis from which to start our construction of this unknown visitor." "Well, then, supposing that \'C.C.H.\' does stand for \'Charing Cross Hospital,\' what further inferences may we draw?" "Do none suggest themselves? You know my methods. Apply them!" "I can only think of the obvious conclusion that the man has practised in town before going to the country." "I think that we might venture a little farther than this. Look at it in this light. On what occasion would it be most probable that such a presentation would be made? When would his friends unite to give him a pledge of their good will? Obviously at the moment when Dr. Mortimer withdrew from the service of the hospital in order to start in practice for himself. We know there has been a presentation. We believe there has been a change from a town hospital to a country practice. Is it, then, stretching our inference too far to say that the presentation was on the occasion of the change?" "It certainly seems probable." "Now, you will observe that he could not have been on the staff of the hospital, since only a man well-established in a London practice could hold such a position, and such a one would not drift into the country. What was he, then? If he was in the hospital and yet not on the staff he could only have been a house-surgeon or a house-physician--little more than a senior student. And he left five years ago--the date is on the stick. So your grave, middle-aged family practitioner vanishes into thin air, my dear Watson, and there emerges a young fellow under thirty, amiable, unambitious, absent-minded, and the possessor of a favourite dog, which I should describe roughly as being larger than a terrier and smaller than a mastiff." I laughed incredulously as Sherlock Holmes leaned back in his settee and blew little wavering rings of smoke up to the ceiling. "As to the latter part'
We'll look at some tools in nltk to help break the raw text into sentences and word tokens.
from nltk.tokenize import sent_tokenize, word_tokenize
str.split?
# you can try to separate sentences by splitting on punctuation
snippet.split('.')
['"Do none suggest themselves? You know my methods', ' Apply them!" "I can only think of the obvious conclusion that the man has practised in town before going to the country', '" "I think that we might venture a little farther than this', ' Look at it in this light', ' On what occasion would it be most probable that such a presentation would be made? When would his friends unite to give him a pledge of their good will? Obviously at the moment when Dr', ' Mortimer withdrew from the service of the hospital in order to start in practice for himself', ' We know there has been a presentation', ' We believe there has been a change from a town hospital to a country practice', ' Is it, then, stretching our inference too far to say that the presentation was on the occasion of the change?" "It certainly seems probable', '" "Now, you will observe that he could not have been on the staff of the hospital, since only a man well-established in a London practice could hold such a position, and such a one would not drift into the country', ' What was he, then? If he was in the hospital and yet not on the staff he could only have been a house-surgeon or a house-physician--little more than a senior student', ' And he left five years ago--the date is on the stick', ' So']
# The sentence tokenizer has some clever tricks to do a better job
sent_tokenize(snippet)
['"Do none suggest themselves?', 'You know my methods.', 'Apply them!"', '"I can only think of the obvious conclusion that the man has practised in town before going to the country."', '"I think that we might venture a little farther than this.', 'Look at it in this light.', 'On what occasion would it be most probable that such a presentation would be made?', 'When would his friends unite to give him a pledge of their good will?', 'Obviously at the moment when Dr. Mortimer withdrew from the service of the hospital in order to start in practice for himself.', 'We know there has been a presentation.', 'We believe there has been a change from a town hospital to a country practice.', 'Is it, then, stretching our inference too far to say that the presentation was on the occasion of the change?"', '"It certainly seems probable."', '"Now, you will observe that he could not have been on the staff of the hospital, since only a man well-established in a London practice could hold such a position, and such a one would not drift into the country.', 'What was he, then?', 'If he was in the hospital and yet not on the staff he could only have been a house-surgeon or a house-physician--little more than a senior student.', 'And he left five years ago--the date is on the stick.', 'So']
# splitting a text into tokens based on white space
snippet.split()
['"Do', 'none', 'suggest', 'themselves?', 'You', 'know', 'my', 'methods.', 'Apply', 'them!"', '"I', 'can', 'only', 'think', 'of', 'the', 'obvious', 'conclusion', 'that', 'the', 'man', 'has', 'practised', 'in', 'town', 'before', 'going', 'to', 'the', 'country."', '"I', 'think', 'that', 'we', 'might', 'venture', 'a', 'little', 'farther', 'than', 'this.', 'Look', 'at', 'it', 'in', 'this', 'light.', 'On', 'what', 'occasion', 'would', 'it', 'be', 'most', 'probable', 'that', 'such', 'a', 'presentation', 'would', 'be', 'made?', 'When', 'would', 'his', 'friends', 'unite', 'to', 'give', 'him', 'a', 'pledge', 'of', 'their', 'good', 'will?', 'Obviously', 'at', 'the', 'moment', 'when', 'Dr.', 'Mortimer', 'withdrew', 'from', 'the', 'service', 'of', 'the', 'hospital', 'in', 'order', 'to', 'start', 'in', 'practice', 'for', 'himself.', 'We', 'know', 'there', 'has', 'been', 'a', 'presentation.', 'We', 'believe', 'there', 'has', 'been', 'a', 'change', 'from', 'a', 'town', 'hospital', 'to', 'a', 'country', 'practice.', 'Is', 'it,', 'then,', 'stretching', 'our', 'inference', 'too', 'far', 'to', 'say', 'that', 'the', 'presentation', 'was', 'on', 'the', 'occasion', 'of', 'the', 'change?"', '"It', 'certainly', 'seems', 'probable."', '"Now,', 'you', 'will', 'observe', 'that', 'he', 'could', 'not', 'have', 'been', 'on', 'the', 'staff', 'of', 'the', 'hospital,', 'since', 'only', 'a', 'man', 'well-established', 'in', 'a', 'London', 'practice', 'could', 'hold', 'such', 'a', 'position,', 'and', 'such', 'a', 'one', 'would', 'not', 'drift', 'into', 'the', 'country.', 'What', 'was', 'he,', 'then?', 'If', 'he', 'was', 'in', 'the', 'hospital', 'and', 'yet', 'not', 'on', 'the', 'staff', 'he', 'could', 'only', 'have', 'been', 'a', 'house-surgeon', 'or', 'a', 'house-physician--little', 'more', 'than', 'a', 'senior', 'student.', 'And', 'he', 'left', 'five', 'years', 'ago--the', 'date', 'is', 'on', 'the', 'stick.', 'So']
words = word_tokenize(snippet)
# word tokenize treats punctuation as a token
words
['``', 'Do', 'none', 'suggest', 'themselves', '?', 'You', 'know', 'my', 'methods', '.', 'Apply', 'them', '!', "''", '``', 'I', 'can', 'only', 'think', 'of', 'the', 'obvious', 'conclusion', 'that', 'the', 'man', 'has', 'practised', 'in', 'town', 'before', 'going', 'to', 'the', 'country', '.', "''", '``', 'I', 'think', 'that', 'we', 'might', 'venture', 'a', 'little', 'farther', 'than', 'this', '.', 'Look', 'at', 'it', 'in', 'this', 'light', '.', 'On', 'what', 'occasion', 'would', 'it', 'be', 'most', 'probable', 'that', 'such', 'a', 'presentation', 'would', 'be', 'made', '?', 'When', 'would', 'his', 'friends', 'unite', 'to', 'give', 'him', 'a', 'pledge', 'of', 'their', 'good', 'will', '?', 'Obviously', 'at', 'the', 'moment', 'when', 'Dr.', 'Mortimer', 'withdrew', 'from', 'the', 'service', 'of', 'the', 'hospital', 'in', 'order', 'to', 'start', 'in', 'practice', 'for', 'himself', '.', 'We', 'know', 'there', 'has', 'been', 'a', 'presentation', '.', 'We', 'believe', 'there', 'has', 'been', 'a', 'change', 'from', 'a', 'town', 'hospital', 'to', 'a', 'country', 'practice', '.', 'Is', 'it', ',', 'then', ',', 'stretching', 'our', 'inference', 'too', 'far', 'to', 'say', 'that', 'the', 'presentation', 'was', 'on', 'the', 'occasion', 'of', 'the', 'change', '?', "''", '``', 'It', 'certainly', 'seems', 'probable', '.', "''", '``', 'Now', ',', 'you', 'will', 'observe', 'that', 'he', 'could', 'not', 'have', 'been', 'on', 'the', 'staff', 'of', 'the', 'hospital', ',', 'since', 'only', 'a', 'man', 'well-established', 'in', 'a', 'London', 'practice', 'could', 'hold', 'such', 'a', 'position', ',', 'and', 'such', 'a', 'one', 'would', 'not', 'drift', 'into', 'the', 'country', '.', 'What', 'was', 'he', ',', 'then', '?', 'If', 'he', 'was', 'in', 'the', 'hospital', 'and', 'yet', 'not', 'on', 'the', 'staff', 'he', 'could', 'only', 'have', 'been', 'a', 'house-surgeon', 'or', 'a', 'house-physician', '--', 'little', 'more', 'than', 'a', 'senior', 'student', '.', 'And', 'he', 'left', 'five', 'years', 'ago', '--', 'the', 'date', 'is', 'on', 'the', 'stick', '.', 'So']
# let's plot the frequency of occurrence of different words
nltk.FreqDist?
fdist = nltk.FreqDist(words)
fdist.plot(30)
Stop words are words that you want to filter out from your text for downstream analysis. They are typically very common words which don't contain much useful information for the task at hand. There is no universal set of stop words and some domain knowledge is helpful for deciding what you want to include when processing your text.
from nltk.corpus import stopwords
stops = stopwords.words('english')
stops
['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', 'couldn', 'didn', 'doesn', 'hadn', 'hasn', 'haven', 'isn', 'ma', 'mightn', 'mustn', 'needn', 'shan', 'shouldn', 'wasn', 'weren', 'won', 'wouldn']
filtered_words = [word.lower() for word in words if word.lower() not in stops]
filtered_words
['``', 'none', 'suggest', '?', 'know', 'methods', '.', 'apply', '!', "''", '``', 'think', 'obvious', 'conclusion', 'man', 'practised', 'town', 'going', 'country', '.', "''", '``', 'think', 'might', 'venture', 'little', 'farther', '.', 'look', 'light', '.', 'occasion', 'would', 'probable', 'presentation', 'would', 'made', '?', 'would', 'friends', 'unite', 'give', 'pledge', 'good', '?', 'obviously', 'moment', 'dr.', 'mortimer', 'withdrew', 'service', 'hospital', 'order', 'start', 'practice', '.', 'know', 'presentation', '.', 'believe', 'change', 'town', 'hospital', 'country', 'practice', '.', ',', ',', 'stretching', 'inference', 'far', 'say', 'presentation', 'occasion', 'change', '?', "''", '``', 'certainly', 'seems', 'probable', '.', "''", '``', ',', 'observe', 'could', 'staff', 'hospital', ',', 'since', 'man', 'well-established', 'london', 'practice', 'could', 'hold', 'position', ',', 'one', 'would', 'drift', 'country', '.', ',', '?', 'hospital', 'yet', 'staff', 'could', 'house-surgeon', 'house-physician', '--', 'little', 'senior', 'student', '.', 'left', 'five', 'years', 'ago', '--', 'date', 'stick', '.']
filtered_fdist = nltk.FreqDist(filtered_words)
filtered_fdist.plot(30)
import string
string.punctuation
'!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
stops = stopwords.words('english') + list(string.punctuation)
stops
['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', 'couldn', 'didn', 'doesn', 'hadn', 'hasn', 'haven', 'isn', 'ma', 'mightn', 'mustn', 'needn', 'shan', 'shouldn', 'wasn', 'weren', 'won', 'wouldn', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~']
filtered_words = [word.lower() for word in words if word.lower() not in stops]
filtered_fdist2 = nltk.FreqDist(filtered_words)
filtered_fdist2.plot(30)
def process_text(text):
# break text into word tokens
tokens = word_tokenize(text)
# remove stopwords
filtered_words = [token.lower() for token in tokens if not token.lower() in stops]
# filter for short punctuation
filtered_words = [w for w in filtered_words if (len(w) > 2)]
return filtered_words
whole_text[:110]
"Project Gutenberg's The Hound of the Baskervilles, by Arthur Conan Doyle This eBook is for the use of anyone "
len(whole_text)
337863
%%time
clean_text = process_text(whole_text)
CPU times: user 1.17 s, sys: 0 ns, total: 1.17 s Wall time: 1.17 s
fdist_whole_text = nltk.FreqDist(clean_text)
fdist_whole_text.plot(25)
boring_words = ['sir', 'upon', 'said', 'one']
stops += boring_words
%%time
cleaned_text = process_text(whole_text)
CPU times: user 1.32 s, sys: 4 ms, total: 1.32 s Wall time: 1.32 s
fdist_whole_text['holmes']
194
fdist_whole_text['watson']
116
from nltk.stem import PorterStemmer
help(nltk.stem)
Help on package nltk.stem in nltk: NAME nltk.stem - NLTK Stemmers DESCRIPTION Interfaces used to remove morphological affixes from words, leaving only the word stem. Stemming algorithms aim to remove those affixes required for eg. grammatical role, tense, derivational morphology leaving only the stem of the word. This is a difficult problem due to irregular words (eg. common verbs in English), complicated morphological rules, and part-of-speech and sense ambiguities (eg. ``ceil-`` is not the stem of ``ceiling``). StemmerI defines a standard interface for stemmers. PACKAGE CONTENTS api isri lancaster porter regexp rslp snowball util wordnet FILE /home/derek/anaconda3/envs/nlp36/lib/python3.6/site-packages/nltk/stem/__init__.py
ps = PorterStemmer()
print(ps.stem('Happy'))
print(ps.stem('Happiness'))
print(ps.stem('Had'))
print(ps.stem('Fishing'))
print(ps.stem('Fish'))
print(ps.stem('Fisher'))
print(ps.stem('Fishes'))
print(ps.stem('Fished'))
happi happi had fish fish fisher fish fish
words = process_text(snippet)
stemmed = [ps.stem(word) for word in words]
for w, stem in zip(words, stemmed):
print('{} ---> {}'.format(w, stem))
none ---> none suggest ---> suggest know ---> know methods ---> method apply ---> appli think ---> think obvious ---> obviou conclusion ---> conclus man ---> man practised ---> practis town ---> town going ---> go country ---> countri think ---> think might ---> might venture ---> ventur little ---> littl farther ---> farther look ---> look light ---> light occasion ---> occas would ---> would probable ---> probabl presentation ---> present would ---> would made ---> made would ---> would friends ---> friend unite ---> unit give ---> give pledge ---> pledg good ---> good obviously ---> obvious moment ---> moment dr. ---> dr. mortimer ---> mortim withdrew ---> withdrew service ---> servic hospital ---> hospit order ---> order start ---> start practice ---> practic know ---> know presentation ---> present believe ---> believ change ---> chang town ---> town hospital ---> hospit country ---> countri practice ---> practic stretching ---> stretch inference ---> infer far ---> far say ---> say presentation ---> present occasion ---> occas change ---> chang certainly ---> certainli seems ---> seem probable ---> probabl observe ---> observ could ---> could staff ---> staff hospital ---> hospit since ---> sinc man ---> man well-established ---> well-establish london ---> london practice ---> practic could ---> could hold ---> hold position ---> posit would ---> would drift ---> drift country ---> countri hospital ---> hospit yet ---> yet staff ---> staff could ---> could house-surgeon ---> house-surgeon house-physician ---> house-physician little ---> littl senior ---> senior student ---> student left ---> left five ---> five years ---> year ago ---> ago date ---> date stick ---> stick
def stem_process(text):
# tokenize
tokens = word_tokenize(text)
# remove stops
filtered_words = [token.lower() for token in tokens if not token.lower() in stops]
filtered_words = [w for w in filtered_words if (len(w) > 2)]
# stem
stemmed_words = [ps.stem(w) for w in filtered_words]
return stemmed_words
%%time
stemmed = stem_process(whole_text)
CPU times: user 2.72 s, sys: 16 ms, total: 2.73 s Wall time: 2.74 s
stemmed
['project', 'gutenberg', 'hound', 'baskervil', 'arthur', 'conan', 'doyl', 'ebook', 'use', 'anyon', 'anywher', 'cost', 'almost', 'restrict', 'whatsoev', 'may', 'copi', 'give', 'away', 're-us', 'term', 'project', 'gutenberg', 'licens', 'includ', 'ebook', 'onlin', 'www.gutenberg.org', 'titl', 'hound', 'baskervil', 'author', 'arthur', 'conan', 'doyl', 'post', 'date', 'octob', '2010', 'releas', 'date', 'februari', '2002', 'etext', '3070', 'languag', 'english', '***', 'start', 'project', 'gutenberg', 'ebook', 'hound', 'baskervil', '***', 'produc', 'etext', 'produc', 'k.pehtla', 'ppehtla', 'nfld.com', 'hound', 'baskervil', 'arthur', 'conan', 'doyl', 'content', 'chapter', 'mr.', 'sherlock', 'holm', 'chapter', 'curs', 'baskervil', 'chapter', 'problem', 'chapter', 'henri', 'baskervil', 'chapter', 'three', 'broken', 'thread', 'chapter', 'baskervil', 'hall', 'chapter', 'stapleton', 'merripit', 'hous', 'chapter', 'first', 'report', 'dr.', 'watson', 'chapter', 'light', 'moor', 'chapter', 'extract', 'diari', 'dr.', 'watson', 'chapter', 'man', 'tor', 'chapter', 'death', 'moor', 'chapter', 'fix', 'net', 'chapter', 'hound', 'baskervil', 'chapter', 'retrospect', 'chapter', 'mr.', 'sherlock', 'holm', 'mr.', 'sherlock', 'holm', 'usual', 'late', 'morn', 'save', 'infrequ', 'occas', 'night', 'seat', 'breakfast', 'tabl', 'stood', 'hearth-rug', 'pick', 'stick', 'visitor', 'left', 'behind', 'night', 'fine', 'thick', 'piec', 'wood', 'bulbous-head', 'sort', 'known', 'penang', 'lawyer', 'head', 'broad', 'silver', 'band', 'nearli', 'inch', 'across', 'jame', 'mortim', 'm.r.c.s.', 'friend', 'c.c.h.', 'engrav', 'date', '1884', 'stick', 'old-fashion', 'famili', 'practition', 'use', 'carri', 'dignifi', 'solid', 'reassur', 'well', 'watson', 'make', 'holm', 'sit', 'back', 'given', 'sign', 'occup', 'know', 'believ', 'eye', 'back', 'head', 'least', 'well-polish', 'silver-pl', 'coffee-pot', 'front', 'tell', 'watson', 'make', 'visitor', 'stick', 'sinc', 'unfortun', 'miss', 'notion', 'errand', 'accident', 'souvenir', 'becom', 'import', 'let', 'hear', 'reconstruct', 'man', 'examin', 'think', 'follow', 'far', 'could', 'method', 'companion', 'dr.', 'mortim', 'success', 'elderli', 'medic', 'man', 'well-esteem', 'sinc', 'know', 'give', 'mark', 'appreci', 'good', 'holm', 'excel', 'think', 'also', 'probabl', 'favour', 'countri', 'practition', 'great', 'deal', 'visit', 'foot', 'stick', 'though', 'origin', 'handsom', 'knock', 'hardli', 'imagin', 'town', 'practition', 'carri', 'thick-iron', 'ferrul', 'worn', 'evid', 'done', 'great', 'amount', 'walk', 'perfectli', 'sound', 'holm', "'friend", 'c.c.h', 'guess', 'someth', 'hunt', 'local', 'hunt', 'whose', 'member', 'possibl', 'given', 'surgic', 'assist', 'made', 'small', 'present', 'return', 'realli', 'watson', 'excel', 'holm', 'push', 'back', 'chair', 'light', 'cigarett', 'bound', 'say', 'account', 'good', 'give', 'small', 'achiev', 'habitu', 'underr', 'abil', 'may', 'lumin', 'conductor', 'light', 'peopl', 'without', 'possess', 'geniu', 'remark', 'power', 'stimul', 'confess', 'dear', 'fellow', 'much', 'debt', 'never', 'much', 'must', 'admit', 'word', 'gave', 'keen', 'pleasur', 'often', 'piqu', 'indiffer', 'admir', 'attempt', 'made', 'give', 'public', 'method', 'proud', 'think', 'far', 'master', 'system', 'appli', 'way', 'earn', 'approv', 'took', 'stick', 'hand', 'examin', 'minut', 'nake', 'eye', 'express', 'interest', 'laid', 'cigarett', 'carri', 'cane', 'window', 'look', 'convex', 'len', 'interest', 'though', 'elementari', 'return', 'favourit', 'corner', 'sette', 'certainli', 'two', 'indic', 'stick', 'give', 'basi', 'sever', 'deduct', 'anyth', 'escap', 'ask', 'self-import', 'trust', 'noth', 'consequ', 'overlook', 'afraid', 'dear', 'watson', 'conclus', 'erron', 'stimul', 'meant', 'frank', 'note', 'fallaci', 'occasion', 'guid', 'toward', 'truth', 'entir', 'wrong', 'instanc', 'man', 'certainli', 'countri', 'practition', 'walk', 'good', 'deal', 'right', 'extent', 'dear', 'watson', 'mean', 'would', 'suggest', 'exampl', 'present', 'doctor', 'like', 'come', 'hospit', 'hunt', 'initi', "'c.c", 'place', 'hospit', 'word', "'chare", 'cross', 'natur', 'suggest', 'may', 'right', 'probabl', 'lie', 'direct', 'take', 'work', 'hypothesi', 'fresh', 'basi', 'start', 'construct', 'unknown', 'visitor', 'well', 'suppos', "'c.c.h", 'stand', "'chare", 'cross', 'hospit', 'infer', 'may', 'draw', 'none', 'suggest', 'know', 'method', 'appli', 'think', 'obviou', 'conclus', 'man', 'practis', 'town', 'go', 'countri', 'think', 'might', 'ventur', 'littl', 'farther', 'look', 'light', 'occas', 'would', 'probabl', 'present', 'would', 'made', 'would', 'friend', 'unit', 'give', 'pledg', 'good', 'obvious', 'moment', 'dr.', 'mortim', 'withdrew', 'servic', 'hospit', 'order', 'start', 'practic', 'know', 'present', 'believ', 'chang', 'town', 'hospit', 'countri', 'practic', 'stretch', 'infer', 'far', 'say', 'present', 'occas', 'chang', 'certainli', 'seem', 'probabl', 'observ', 'could', 'staff', 'hospit', 'sinc', 'man', 'well-establish', 'london', 'practic', 'could', 'hold', 'posit', 'would', 'drift', 'countri', 'hospit', 'yet', 'staff', 'could', 'house-surgeon', 'house-physician', 'littl', 'senior', 'student', 'left', 'five', 'year', 'ago', 'date', 'stick', 'grave', 'middle-ag', 'famili', 'practition', 'vanish', 'thin', 'air', 'dear', 'watson', 'emerg', 'young', 'fellow', 'thirti', 'amiabl', 'unambiti', 'absent-mind', 'possessor', 'favourit', 'dog', 'describ', 'roughli', 'larger', 'terrier', 'smaller', 'mastiff', 'laugh', 'incredul', 'sherlock', 'holm', 'lean', 'back', 'sette', 'blew', 'littl', 'waver', 'ring', 'smoke', 'ceil', 'latter', 'part', 'mean', 'check', 'least', 'difficult', 'find', 'particular', 'man', 'age', 'profession', 'career', 'small', 'medic', 'shelf', 'took', 'medic', 'directori', 'turn', 'name', 'sever', 'mortim', 'could', 'visitor', 'read', 'record', 'aloud', 'mortim', 'jame', 'm.r.c.s.', '1882', 'grimpen', 'dartmoor', 'devon', 'house-surgeon', '1882', '1884', 'chare', 'cross', 'hospit', 'winner', 'jackson', 'prize', 'compar', 'patholog', 'essay', 'entitl', "'i", 'diseas', 'revers', 'correspond', 'member', 'swedish', 'patholog', 'societi', 'author', "'some", 'freak', 'atav', 'lancet', '1882', "'do", 'progress', 'journal', 'psycholog', 'march', '1883', 'medic', 'offic', 'parish', 'grimpen', 'thorsley', 'high', 'barrow', 'mention', 'local', 'hunt', 'watson', 'holm', 'mischiev', 'smile', 'countri', 'doctor', 'astut', 'observ', 'think', 'fairli', 'justifi', 'infer', 'adject', 'rememb', 'right', 'amiabl', 'unambiti', 'absent-mind', 'experi', 'amiabl', 'man', 'world', 'receiv', 'testimoni', 'unambiti', 'abandon', 'london', 'career', 'countri', 'absent-mind', 'leav', 'stick', 'visiting-card', 'wait', 'hour', 'room', 'dog', 'habit', 'carri', 'stick', 'behind', 'master', 'heavi', 'stick', 'dog', 'held', 'tightli', 'middl', 'mark', 'teeth', 'plainli', 'visibl', 'dog', 'jaw', 'shown', 'space', 'mark', 'broad', 'opinion', 'terrier', 'broad', 'enough', 'mastiff', 'may', 'ye', 'jove', 'curly-hair', 'spaniel', 'risen', 'pace', 'room', 'spoke', 'halt', 'recess', 'window', 'ring', 'convict', 'voic', 'glanc', 'surpris', 'dear', 'fellow', 'possibl', 'sure', 'simpl', 'reason', 'see', 'dog', 'door-step', 'ring', 'owner', "n't", 'move', 'beg', 'watson', 'profession', 'brother', 'presenc', 'may', 'assist', 'dramat', 'moment', 'fate', 'watson', 'hear', 'step', 'stair', 'walk', 'life', 'know', 'whether', 'good', 'ill.', 'dr.', 'jame', 'mortim', 'man', 'scienc', 'ask', 'sherlock', 'holm', 'specialist', 'crime', 'come', 'appear', 'visitor', 'surpris', 'sinc', 'expect', 'typic', 'countri', 'practition', 'tall', 'thin', 'man', 'long', 'nose', 'like', 'beak', 'jut', 'two', 'keen', 'gray', 'eye', 'set', 'close', 'togeth', 'sparkl', 'brightli', 'behind', 'pair', 'gold-rim', 'glass', 'clad', 'profession', 'rather', 'slovenli', 'fashion', 'frock-coat', 'dingi', 'trouser', 'fray', 'though', 'young', 'long', 'back', 'alreadi', 'bow', 'walk', 'forward', 'thrust', 'head', 'gener', 'air', 'peer', 'benevol', 'enter', 'eye', 'fell', 'stick', 'holm', 'hand', 'ran', 'toward', 'exclam', 'joy', 'glad', 'sure', 'whether', 'left', 'ship', 'offic', 'would', 'lose', 'stick', 'world', 'present', 'see', 'holm', 'ye', 'chare', 'cross', 'hospit', 'two', 'friend', 'occas', 'marriag', 'dear', 'dear', 'bad', 'holm', 'shake', 'head', 'dr.', 'mortim', 'blink', 'glass', 'mild', 'astonish', 'bad', 'disarrang', 'littl', 'deduct', 'marriag', 'say', 'ye', 'marri', 'left', 'hospit', 'hope', 'consult', 'practic', 'necessari', 'make', 'home', 'come', 'come', 'far', 'wrong', 'holm', 'dr.', 'jame', 'mortim', 'mister', 'mister', 'humbl', 'm.r.c.', 'man', 'precis', 'mind', 'evid', 'dabbler', 'scienc', 'mr.', 'holm', 'picker', 'shell', 'shore', 'great', 'unknown', 'ocean', 'presum', 'mr.', 'sherlock', 'holm', 'address', 'friend', 'watson', 'glad', 'meet', 'heard', 'name', 'mention', 'connect', 'friend', 'interest', 'much', 'mr.', 'holm', 'hardli', 'expect', 'dolichocephal', 'skull', 'well-mark', 'supra-orbit', 'develop', 'would', 'object', 'run', 'finger', 'along', 'pariet', 'fissur', 'cast', 'skull', 'origin', 'avail', 'would', 'ornament', 'anthropolog', 'museum', 'intent', 'fulsom', 'confess', 'covet', 'skull', 'sherlock', 'holm', 'wave', 'strang', 'visitor', 'chair', 'enthusiast', 'line', 'thought', 'perceiv', 'mine', 'observ', 'forefing', 'make', 'cigarett', 'hesit', 'light', 'man', 'drew', 'paper', 'tobacco', 'twirl', 'surpris', 'dexter', 'long', 'quiver', 'finger', 'agil', 'restless', 'antenna', 'insect', 'holm', 'silent', 'littl', 'dart', ...]
fdist_stems = nltk.FreqDist(stemmed)
fdist_stems.plot(30)
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
lemmatizer.lemmatize?
print(lemmatizer.lemmatize('having'))
print(lemmatizer.lemmatize('have'))
print(lemmatizer.lemmatize('had'))
print()
print(lemmatizer.lemmatize('fishing'))
print(lemmatizer.lemmatize('fish'))
print(lemmatizer.lemmatize('fisher'))
print(lemmatizer.lemmatize('fishes'))
print(lemmatizer.lemmatize('fished'))
print()
print(lemmatizer.lemmatize('am'))
print(lemmatizer.lemmatize('is'))
print(lemmatizer.lemmatize('was'))
having have had fishing fish fisher fish fished am is wa
# including POS for the lemmatizer can improve its output
print(lemmatizer.lemmatize('having', pos='v'))
print(lemmatizer.lemmatize('have', pos='v'))
print(lemmatizer.lemmatize('had', pos='v'))
print()
print(lemmatizer.lemmatize('fishing', pos='v'))
print(lemmatizer.lemmatize('fish', pos='v'))
print(lemmatizer.lemmatize('fisher', pos='n'))
print(lemmatizer.lemmatize('fishes', pos='v'))
print(lemmatizer.lemmatize('fished', pos='v'))
print()
print(lemmatizer.lemmatize('am', pos='v'))
print(lemmatizer.lemmatize('is', pos='v'))
print(lemmatizer.lemmatize('was', pos='v'))
have have have fish fish fisher fish fish be be be
lemmatized = [lemmatizer.lemmatize(word) for word in words]
for w, lemma in zip(words, lemmatized):
print('{} ---> {}'.format(w, lemma))
none ---> none suggest ---> suggest know ---> know methods ---> method apply ---> apply think ---> think obvious ---> obvious conclusion ---> conclusion man ---> man practised ---> practised town ---> town going ---> going country ---> country think ---> think might ---> might venture ---> venture little ---> little farther ---> farther look ---> look light ---> light occasion ---> occasion would ---> would probable ---> probable presentation ---> presentation would ---> would made ---> made would ---> would friends ---> friend unite ---> unite give ---> give pledge ---> pledge good ---> good obviously ---> obviously moment ---> moment dr. ---> dr. mortimer ---> mortimer withdrew ---> withdrew service ---> service hospital ---> hospital order ---> order start ---> start practice ---> practice know ---> know presentation ---> presentation believe ---> believe change ---> change town ---> town hospital ---> hospital country ---> country practice ---> practice stretching ---> stretching inference ---> inference far ---> far say ---> say presentation ---> presentation occasion ---> occasion change ---> change certainly ---> certainly seems ---> seems probable ---> probable observe ---> observe could ---> could staff ---> staff hospital ---> hospital since ---> since man ---> man well-established ---> well-established london ---> london practice ---> practice could ---> could hold ---> hold position ---> position would ---> would drift ---> drift country ---> country hospital ---> hospital yet ---> yet staff ---> staff could ---> could house-surgeon ---> house-surgeon house-physician ---> house-physician little ---> little senior ---> senior student ---> student left ---> left five ---> five years ---> year ago ---> ago date ---> date stick ---> stick
lemmatizer.lemmatize('running', pos='v')
'run'
def lemma_process(text):
# tokenize
tokens = word_tokenize(text)
# remove stops
filtered_words = [token.lower() for token in tokens if not token.lower() in stops]
filtered_words = [w for w in filtered_words if (len(w) > 2)]
# lemmatize
lemmatized_words = [lemmatizer.lemmatize(w) for w in filtered_words]
return lemmatized_words
%%time
lemma_text = lemma_process(whole_text)
CPU times: user 1.88 s, sys: 4 ms, total: 1.88 s Wall time: 1.89 s
lemma_fdist = nltk.FreqDist(lemma_text)
lemma_fdist.plot(30)
The process of marking up a word in a text as corresponding to a particular part of speech, based on both its definition and its context.
POS tagging is tricky because some words can have more than one POS depending on the context.
"Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo."
nltk.help.upenn_tagset()
$: dollar $ -$ --$ A$ C$ HK$ M$ NZ$ S$ U.S.$ US$ '': closing quotation mark ' '' (: opening parenthesis ( [ { ): closing parenthesis ) ] } ,: comma , --: dash -- .: sentence terminator . ! ? :: colon or ellipsis : ; ... CC: conjunction, coordinating & 'n and both but either et for less minus neither nor or plus so therefore times v. versus vs. whether yet CD: numeral, cardinal mid-1890 nine-thirty forty-two one-tenth ten million 0.5 one forty- seven 1987 twenty '79 zero two 78-degrees eighty-four IX '60s .025 fifteen 271,124 dozen quintillion DM2,000 ... DT: determiner all an another any both del each either every half la many much nary neither no some such that the them these this those EX: existential there there FW: foreign word gemeinschaft hund ich jeux habeas Haementeria Herr K'ang-si vous lutihaw alai je jour objets salutaris fille quibusdam pas trop Monte terram fiche oui corporis ... IN: preposition or conjunction, subordinating astride among uppon whether out inside pro despite on by throughout below within for towards near behind atop around if like until below next into if beside ... JJ: adjective or numeral, ordinal third ill-mannered pre-war regrettable oiled calamitous first separable ectoplasmic battery-powered participatory fourth still-to-be-named multilingual multi-disciplinary ... JJR: adjective, comparative bleaker braver breezier briefer brighter brisker broader bumper busier calmer cheaper choosier cleaner clearer closer colder commoner costlier cozier creamier crunchier cuter ... JJS: adjective, superlative calmest cheapest choicest classiest cleanest clearest closest commonest corniest costliest crassest creepiest crudest cutest darkest deadliest dearest deepest densest dinkiest ... LS: list item marker A A. B B. C C. D E F First G H I J K One SP-44001 SP-44002 SP-44005 SP-44007 Second Third Three Two * a b c d first five four one six three two MD: modal auxiliary can cannot could couldn't dare may might must need ought shall should shouldn't will would NN: noun, common, singular or mass common-carrier cabbage knuckle-duster Casino afghan shed thermostat investment slide humour falloff slick wind hyena override subhumanity machinist ... NNP: noun, proper, singular Motown Venneboerger Czestochwa Ranzer Conchita Trumplane Christos Oceanside Escobar Kreisler Sawyer Cougar Yvette Ervin ODI Darryl CTCA Shannon A.K.C. Meltex Liverpool ... NNPS: noun, proper, plural Americans Americas Amharas Amityvilles Amusements Anarcho-Syndicalists Andalusians Andes Andruses Angels Animals Anthony Antilles Antiques Apache Apaches Apocrypha ... NNS: noun, common, plural undergraduates scotches bric-a-brac products bodyguards facets coasts divestitures storehouses designs clubs fragrances averages subjectivists apprehensions muses factory-jobs ... PDT: pre-determiner all both half many quite such sure this POS: genitive marker ' 's PRP: pronoun, personal hers herself him himself hisself it itself me myself one oneself ours ourselves ownself self she thee theirs them themselves they thou thy us PRP$: pronoun, possessive her his mine my our ours their thy your RB: adverb occasionally unabatingly maddeningly adventurously professedly stirringly prominently technologically magisterially predominately swiftly fiscally pitilessly ... RBR: adverb, comparative further gloomier grander graver greater grimmer harder harsher healthier heavier higher however larger later leaner lengthier less- perfectly lesser lonelier longer louder lower more ... RBS: adverb, superlative best biggest bluntest earliest farthest first furthest hardest heartiest highest largest least less most nearest second tightest worst RP: particle aboard about across along apart around aside at away back before behind by crop down ever fast for forth from go high i.e. in into just later low more off on open out over per pie raising start teeth that through under unto up up-pp upon whole with you SYM: symbol % & ' '' ''. ) ). * + ,. < = > @ A[fj] U.S U.S.S.R * ** *** TO: "to" as preposition or infinitive marker to UH: interjection Goodbye Goody Gosh Wow Jeepers Jee-sus Hubba Hey Kee-reist Oops amen huh howdy uh dammit whammo shucks heck anyways whodunnit honey golly man baby diddle hush sonuvabitch ... VB: verb, base form ask assemble assess assign assume atone attention avoid bake balkanize bank begin behold believe bend benefit bevel beware bless boil bomb boost brace break bring broil brush build ... VBD: verb, past tense dipped pleaded swiped regummed soaked tidied convened halted registered cushioned exacted snubbed strode aimed adopted belied figgered speculated wore appreciated contemplated ... VBG: verb, present participle or gerund telegraphing stirring focusing angering judging stalling lactating hankerin' alleging veering capping approaching traveling besieging encrypting interrupting erasing wincing ... VBN: verb, past participle multihulled dilapidated aerosolized chaired languished panelized used experimented flourished imitated reunifed factored condensed sheared unsettled primed dubbed desired ... VBP: verb, present tense, not 3rd person singular predominate wrap resort sue twist spill cure lengthen brush terminate appear tend stray glisten obtain comprise detest tease attract emphasize mold postpone sever return wag ... VBZ: verb, present tense, 3rd person singular bases reconstructs marks mixes displeases seals carps weaves snatches slumps stretches authorizes smolders pictures emerges stockpiles seduces fizzes uses bolsters slaps speaks pleads ... WDT: WH-determiner that what whatever which whichever WP: WH-pronoun that what whatever whatsoever which who whom whosoever WP$: WH-pronoun, possessive whose WRB: Wh-adverb how however whence whenever where whereby whereever wherein whereof why ``: opening quotation mark ` ``
snippet
'"Do none suggest themselves? You know my methods. Apply them!" "I can only think of the obvious conclusion that the man has practised in town before going to the country." "I think that we might venture a little farther than this. Look at it in this light. On what occasion would it be most probable that such a presentation would be made? When would his friends unite to give him a pledge of their good will? Obviously at the moment when Dr. Mortimer withdrew from the service of the hospital in order to start in practice for himself. We know there has been a presentation. We believe there has been a change from a town hospital to a country practice. Is it, then, stretching our inference too far to say that the presentation was on the occasion of the change?" "It certainly seems probable." "Now, you will observe that he could not have been on the staff of the hospital, since only a man well-established in a London practice could hold such a position, and such a one would not drift into the country. What was he, then? If he was in the hospital and yet not on the staff he could only have been a house-surgeon or a house-physician--little more than a senior student. And he left five years ago--the date is on the stick. So'
nltk.pos_tag(word_tokenize(sent_tokenize(snippet)[1]))
[('You', 'PRP'), ('know', 'VBP'), ('my', 'PRP$'), ('methods', 'NNS'), ('.', '.')]
def process_POS(text):
sentences = sent_tokenize(text)
tagged_words = []
for sentence in sentences:
words = word_tokenize(sentence)
tagged = nltk.pos_tag(words)
tagged_words.append(tagged)
return tagged_words
tagged_sentences = process_POS(snippet)
tagged_sentences
[[('``', '``'), ('Do', 'VBP'), ('none', 'RB'), ('suggest', 'VB'), ('themselves', 'PRP'), ('?', '.')], [('You', 'PRP'), ('know', 'VBP'), ('my', 'PRP$'), ('methods', 'NNS'), ('.', '.')], [('Apply', 'VB'), ('them', 'PRP'), ('!', '.'), ("''", "''")], [('``', '``'), ('I', 'PRP'), ('can', 'MD'), ('only', 'RB'), ('think', 'VB'), ('of', 'IN'), ('the', 'DT'), ('obvious', 'JJ'), ('conclusion', 'NN'), ('that', 'IN'), ('the', 'DT'), ('man', 'NN'), ('has', 'VBZ'), ('practised', 'VBN'), ('in', 'IN'), ('town', 'NN'), ('before', 'IN'), ('going', 'VBG'), ('to', 'TO'), ('the', 'DT'), ('country', 'NN'), ('.', '.'), ("''", "''")], [('``', '``'), ('I', 'PRP'), ('think', 'VBP'), ('that', 'IN'), ('we', 'PRP'), ('might', 'MD'), ('venture', 'NN'), ('a', 'DT'), ('little', 'RB'), ('farther', 'JJR'), ('than', 'IN'), ('this', 'DT'), ('.', '.')], [('Look', 'NN'), ('at', 'IN'), ('it', 'PRP'), ('in', 'IN'), ('this', 'DT'), ('light', 'NN'), ('.', '.')], [('On', 'IN'), ('what', 'WP'), ('occasion', 'NN'), ('would', 'MD'), ('it', 'PRP'), ('be', 'VB'), ('most', 'RBS'), ('probable', 'JJ'), ('that', 'IN'), ('such', 'PDT'), ('a', 'DT'), ('presentation', 'NN'), ('would', 'MD'), ('be', 'VB'), ('made', 'VBN'), ('?', '.')], [('When', 'WRB'), ('would', 'MD'), ('his', 'PRP$'), ('friends', 'NNS'), ('unite', 'JJ'), ('to', 'TO'), ('give', 'VB'), ('him', 'PRP'), ('a', 'DT'), ('pledge', 'NN'), ('of', 'IN'), ('their', 'PRP$'), ('good', 'NN'), ('will', 'MD'), ('?', '.')], [('Obviously', 'RB'), ('at', 'IN'), ('the', 'DT'), ('moment', 'NN'), ('when', 'WRB'), ('Dr.', 'NNP'), ('Mortimer', 'NNP'), ('withdrew', 'VBD'), ('from', 'IN'), ('the', 'DT'), ('service', 'NN'), ('of', 'IN'), ('the', 'DT'), ('hospital', 'NN'), ('in', 'IN'), ('order', 'NN'), ('to', 'TO'), ('start', 'VB'), ('in', 'IN'), ('practice', 'NN'), ('for', 'IN'), ('himself', 'PRP'), ('.', '.')], [('We', 'PRP'), ('know', 'VBP'), ('there', 'EX'), ('has', 'VBZ'), ('been', 'VBN'), ('a', 'DT'), ('presentation', 'NN'), ('.', '.')], [('We', 'PRP'), ('believe', 'VBP'), ('there', 'EX'), ('has', 'VBZ'), ('been', 'VBN'), ('a', 'DT'), ('change', 'NN'), ('from', 'IN'), ('a', 'DT'), ('town', 'NN'), ('hospital', 'NN'), ('to', 'TO'), ('a', 'DT'), ('country', 'NN'), ('practice', 'NN'), ('.', '.')], [('Is', 'VBZ'), ('it', 'PRP'), (',', ','), ('then', 'RB'), (',', ','), ('stretching', 'VBG'), ('our', 'PRP$'), ('inference', 'NN'), ('too', 'RB'), ('far', 'RB'), ('to', 'TO'), ('say', 'VB'), ('that', 'IN'), ('the', 'DT'), ('presentation', 'NN'), ('was', 'VBD'), ('on', 'IN'), ('the', 'DT'), ('occasion', 'NN'), ('of', 'IN'), ('the', 'DT'), ('change', 'NN'), ('?', '.'), ("''", "''")], [('``', '``'), ('It', 'PRP'), ('certainly', 'RB'), ('seems', 'VBZ'), ('probable', 'JJ'), ('.', '.'), ("''", "''")], [('``', '``'), ('Now', 'RB'), (',', ','), ('you', 'PRP'), ('will', 'MD'), ('observe', 'VB'), ('that', 'IN'), ('he', 'PRP'), ('could', 'MD'), ('not', 'RB'), ('have', 'VB'), ('been', 'VBN'), ('on', 'IN'), ('the', 'DT'), ('staff', 'NN'), ('of', 'IN'), ('the', 'DT'), ('hospital', 'NN'), (',', ','), ('since', 'IN'), ('only', 'RB'), ('a', 'DT'), ('man', 'NN'), ('well-established', 'JJ'), ('in', 'IN'), ('a', 'DT'), ('London', 'NNP'), ('practice', 'NN'), ('could', 'MD'), ('hold', 'VB'), ('such', 'PDT'), ('a', 'DT'), ('position', 'NN'), (',', ','), ('and', 'CC'), ('such', 'PDT'), ('a', 'DT'), ('one', 'NN'), ('would', 'MD'), ('not', 'RB'), ('drift', 'VB'), ('into', 'IN'), ('the', 'DT'), ('country', 'NN'), ('.', '.')], [('What', 'WP'), ('was', 'VBD'), ('he', 'PRP'), (',', ','), ('then', 'RB'), ('?', '.')], [('If', 'IN'), ('he', 'PRP'), ('was', 'VBD'), ('in', 'IN'), ('the', 'DT'), ('hospital', 'NN'), ('and', 'CC'), ('yet', 'RB'), ('not', 'RB'), ('on', 'IN'), ('the', 'DT'), ('staff', 'NN'), ('he', 'PRP'), ('could', 'MD'), ('only', 'RB'), ('have', 'VB'), ('been', 'VBN'), ('a', 'DT'), ('house-surgeon', 'NN'), ('or', 'CC'), ('a', 'DT'), ('house-physician', 'JJ'), ('--', ':'), ('little', 'RB'), ('more', 'JJR'), ('than', 'IN'), ('a', 'DT'), ('senior', 'JJ'), ('student', 'NN'), ('.', '.')], [('And', 'CC'), ('he', 'PRP'), ('left', 'VBD'), ('five', 'CD'), ('years', 'NNS'), ('ago', 'RB'), ('--', ':'), ('the', 'DT'), ('date', 'NN'), ('is', 'VBZ'), ('on', 'IN'), ('the', 'DT'), ('stick', 'NN'), ('.', '.')], [('So', 'RB')]]
sentences =[]
for sentence in tagged_sentences[:5]:
print(sentence)
lemmas = []
for word, pos in sentence:
if pos == 'VBP':
lemmas.append(lemmatizer.lemmatize(word, 'v'))
elif pos in ['NN', 'NNS']:
lemmas.append(lemmatizer.lemmatize(word, 'n'))
else:
lemmas.append(lemmatizer.lemmatize(word))
sentences.append(lemmas)
[('``', '``'), ('Do', 'VBP'), ('none', 'RB'), ('suggest', 'VB'), ('themselves', 'PRP'), ('?', '.')] [('You', 'PRP'), ('know', 'VBP'), ('my', 'PRP$'), ('methods', 'NNS'), ('.', '.')] [('Apply', 'VB'), ('them', 'PRP'), ('!', '.'), ("''", "''")] [('``', '``'), ('I', 'PRP'), ('can', 'MD'), ('only', 'RB'), ('think', 'VB'), ('of', 'IN'), ('the', 'DT'), ('obvious', 'JJ'), ('conclusion', 'NN'), ('that', 'IN'), ('the', 'DT'), ('man', 'NN'), ('has', 'VBZ'), ('practised', 'VBN'), ('in', 'IN'), ('town', 'NN'), ('before', 'IN'), ('going', 'VBG'), ('to', 'TO'), ('the', 'DT'), ('country', 'NN'), ('.', '.'), ("''", "''")] [('``', '``'), ('I', 'PRP'), ('think', 'VBP'), ('that', 'IN'), ('we', 'PRP'), ('might', 'MD'), ('venture', 'NN'), ('a', 'DT'), ('little', 'RB'), ('farther', 'JJR'), ('than', 'IN'), ('this', 'DT'), ('.', '.')]
from nltk import ngrams
from collections import Counter
bigrams = Counter(ngrams(word_tokenize(whole_text), 2))
for phrase, freq in bigrams.most_common(30):
print("{}\t{}".format(phrase, freq))
("''", '``') 859 (',', 'and') 678 ('.', "''") 671 ('.', '``') 475 ('of', 'the') 461 ('?', "''") 399 ('.', 'I') 253 ('in', 'the') 251 (',', 'but') 211 (',', "''") 186 ('``', 'I') 183 ("''", 'said') 175 ('.', 'The') 157 ('Sir', 'Henry') 153 ('I', 'have') 147 ('to', 'the') 144 ('.', 'It') 137 ('the', 'moor') 135 ('upon', 'the') 132 ('that', 'he') 130 (',', 'I') 129 ('that', 'I') 127 ('.', 'He') 127 ('and', 'the') 120 ('at', 'the') 109 ('it', '.') 109 (',', 'the') 105 ('it', 'was') 105 ('and', 'I') 104 ('I', 'had') 102
trigrams = Counter(ngrams(word_tokenize(whole_text), 3))
for phrase, freq in trigrams.most_common(30):
print("{}\t{}".format(phrase, freq))
('.', "''", '``') 508 ('?', "''", '``') 316 (',', "''", 'said') 150 ("''", '``', 'I') 107 (',', 'and', 'I') 77 ('.', '``', 'I') 66 (',', 'and', 'the') 57 (',', 'sir', ',') 57 ("''", 'said', 'he') 55 ('the', 'moor', '.') 52 ('.', 'It', 'was') 51 ("''", '``', 'No') 50 ("''", '``', 'Yes') 49 ('``', 'No', ',') 46 ("''", '``', 'And') 44 ('upon', 'the', 'moor') 44 ('``', 'Well', ',') 43 (',', 'and', 'that') 43 ('``', 'Yes', ',') 43 (',', 'and', 'he') 41 ("''", 'said', 'Holmes') 39 ('it', '.', "''") 38 (',', 'but', 'I') 36 ("''", '``', 'Well') 34 ('.', 'It', 'is') 34 ('.', '``', 'It') 33 ('said', 'he', '.') 32 ("''", '``', 'But') 32 (',', 'Watson', ',') 31 ('he', '.', '``') 31
stemmed = stem_process(whole_text)
stemmed_bigrams = Counter(ngrams(stemmed, 2))
stemmed_bigrams.most_common(20)
[(('dr.', 'mortim'), 73), (('project', 'gutenberg-tm'), 57), (('sherlock', 'holm'), 34), (('baskervil', 'hall'), 31), (('project', 'gutenberg'), 30), (('dr.', 'watson'), 28), (('electron', 'work'), 27), (('henri', 'baskervil'), 25), (('mr.', 'holm'), 24), (('coomb', 'tracey'), 18), (('gutenberg-tm', 'electron'), 18), (('merripit', 'hous'), 15), (('charl', 'baskervil'), 15), (('mr.', 'sherlock'), 14), (('baker', 'street'), 14), (('grimpen', 'mire'), 14), (('gutenberg', 'literari'), 13), (('literari', 'archiv'), 13), (('archiv', 'foundat'), 13), (('hound', 'baskervil'), 12)]
stemmed_trigrams = Counter(ngrams(stemmed, 3))
stemmed_trigrams.most_common(20)
[(('project', 'gutenberg-tm', 'electron'), 18), (('gutenberg-tm', 'electron', 'work'), 18), (('mr.', 'sherlock', 'holm'), 14), (('project', 'gutenberg', 'literari'), 13), (('gutenberg', 'literari', 'archiv'), 13), (('literari', 'archiv', 'foundat'), 13), (('mrs.', 'laura', 'lyon'), 10), (('project', 'gutenberg-tm', 'work'), 10), (('distribut', 'project', 'gutenberg-tm'), 9), (('great', 'grimpen', 'mire'), 8), (('project', 'gutenberg-tm', 'licens'), 8), (('full', 'project', 'gutenberg-tm'), 6), (('copi', 'project', 'gutenberg-tm'), 5), (('arthur', 'conan', 'doyl'), 4), (('dr.', 'jame', 'mortim'), 4), (('death', 'charl', 'baskervil'), 4), (('frankland', 'lafter', 'hall'), 4), (('dr.', 'mortim', 'look'), 4), (('phrase', 'project', 'gutenberg'), 4), (('set', 'forth', 'paragraph'), 4)]
from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer()
data_corpus = ["John likes to watch movies. Mary likes movies too.",
"John also likes to watch football games."]
X = vectorizer.fit_transform(data_corpus)
print(X.toarray())
print(vectorizer.get_feature_names())
[[0 0 0 1 2 1 2 1 1 1] [1 1 1 1 1 0 0 1 0 1]] ['also', 'football', 'games', 'john', 'likes', 'mary', 'movies', 'to', 'too', 'watch']
from nltk.sentiment.vader import SentimentIntensityAnalyzer
/home/derek/anaconda3/envs/nlp36/lib/python3.6/site-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available. warnings.warn("The twython library has not been installed. "
vader = SentimentIntensityAnalyzer()
text = "I dont hate movies!"
vader.polarity_scores(text)
{'compound': 0.509, 'neg': 0.0, 'neu': 0.378, 'pos': 0.622}