import pandas as pd
import re
def tidy_word_count(data, row):
df = pd.DataFrame({'term': [], 'topic_count':[]})
try:
topic_count = data[row][2:len(data[row])]
df = pd.DataFrame({'topic_count': topic_count})
df['term'] = data[row][1]
except:
pass
return df
data = []
#with open('/Users/dankoban/Documents/EM6575/mallet_command_line/ct.wordtopiccounts','r') as infile:
with open('/Users/dankoban/Documents/EM6575/twitter/hashtag model/hashtags/hashtags.wordtopiccounts','r') as infile:
for line in infile:
line = line.split(' ')
data.append(line)
counter = 0
tidy_dfs = []
for i in range(0, len(data)):
tidy_dfs.append(tidy_word_count(data = data, row = i))
counter += 1
if counter %10000 == 0:
print(str(counter) + ' out of ' + str(len(data)))
df = pd.concat(tidy_dfs)
df['topic_count'] = df['topic_count'].apply(lambda x: re.sub(r'\n', '', x))
df['topic'] = df['topic_count'].apply(lambda x: x.split(":")[0])
df['count'] = df['topic_count'].apply(lambda x: x.split(":")[1])
df = df[['term', 'topic', 'count']]
df.reset_index(inplace = True, drop = True)
#df.to_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/ct_tidy_topics.csv', index=False)
df.to_csv('/Users/dankoban/Documents/EM6575/twitter/twitter_tidy_topics.csv', index=False)
import pandas as pd
data = []
#with open('/Users/dankoban/Documents/EM6575/mallet_command_line/ct.doctopics_sparse','r') as infile:
with open('/Users/dankoban/Documents/EM6575/twitter/hashtag model/hashtags.doctopics_sparse','r') as infile:
for line in infile:
line = line.rstrip().split('\t')
data.append(line)
df = pd.DataFrame(data[1:], columns = ['doc_id', 'name', 'topic_1', 'proportion_1', 'topic_2', 'proportion_2', 'topic_3', 'proportion_3'])
df['topic_1'] = df['topic_1'].astype('float')
df['topic_2'] = df['topic_2'].astype('float')
df['topic_3'] = df['topic_3'].astype('float')
df['proportion_1'] = df['proportion_1'].astype('float')
df['proportion_2'] = df['proportion_2'].astype('float')
df['proportion_3'] = df['proportion_3'].astype('float')
rank1_docs = df[['doc_id', 'name', 'topic_1', 'proportion_1']]
rank1_docs.columns = ['doc_id', 'name', 'topic', 'proportion']
rank1_docs = rank1_docs.assign(rank = 1)
rank2_docs = df[['doc_id', 'name', 'topic_2', 'proportion_2']]
rank2_docs.columns = ['doc_id', 'name', 'topic', 'proportion']
rank2_docs = rank2_docs.assign(rank = 2)
rank3_docs = df[['doc_id', 'name', 'topic_3', 'proportion_3']]
rank3_docs.columns = ['doc_id', 'name', 'topic', 'proportion']
rank3_docs = rank3_docs.assign(rank = 3)
df = pd.concat([rank1_docs, rank2_docs, rank3_docs])
df = df[df['proportion'].notnull()]
#df = df[df['proportion'].isna() == False]
#df.to_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/tidy_docs2topics.csv', index=False)
df.to_csv('/Users/dankoban/Documents/EM6575/twitter/tidy_docs2topics.csv', index=False)
Load the parsed data to save processing time.
import seaborn as sns
import pandas as pd
import math
import numpy as np
num_topics = 50
num_top_terms = 10
#TopicTermFreq = pd.read_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/tidy_topics.csv')
#TopicTermFreq = pd.read_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/ct_tidy_topics.csv')
TopicTermFreq = pd.read_csv('/Users/dankoban/Documents/EM6575/twitter/twitter_tidy_topics.csv')
TopicTermFreq.head()
term | topic | count | |
---|---|---|---|
0 | bud | 34 | 179 |
1 | bud | 49 | 133 |
2 | bud | 31 | 117 |
3 | bud | 29 | 106 |
4 | bud | 39 | 76 |
#docs2topic = pd.read_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/tidy_docs2topics.csv')
docs2topic = pd.read_csv('/Users/dankoban/Documents/EM6575/twitter/tidy_docs2topics.csv')
len(docs2topic['doc_id'].unique())
699836
These descriptions are designed to make sense without mathematical definitions, but we include them for specificity. We'll use the symbols w for a word, k for a topic, and d for a document. Several metrics count tokens. For these we define variables Nd,w,k as the number of times word w occurs in document d assigned to topic k, Nd,k as ∑wNd,w,k. Other metrics involve document frequencies. D(w) is the number of documents that contain at least one token of type w, and D(w1,w2) is the number of documents that contain at least one w1 and one w2. Finally, we define Wk to be the N most probable words in topic k. 𝟙(p) is equal to 1 if predicate p is true, and 0 otherwise.
Tokens. This metric measures the number of word tokens currently assigned to the topic Nk=∑dNd,k. Comparing this number to the sum of the token counts for all topics will give you the proportion of the corpus assigned to the topic. If you are using optimized hyperparameters this number will vary considerably across topics; otherwise it will be roughly the same for all topics. We usually find topics most interesting if they are not on the extremes of this range. Small numbers (relative to other topics) are often a sign that a topic may not be reliable: we don't have enough observations to get a good sense of the topic's word distribution. Large numbers may also be bad: extremely frequent topics are often "not-quite-stopwords". For example, in the sample model the largest topic is "time number term part system form".
tokens_by_topic = (TopicTermFreq.groupby('topic').
sum().
reset_index().
rename(columns = {'count': 'token_count'}))
tokens_by_topic.head()
topic | token_count | |
---|---|---|
0 | 0 | 1322510 |
1 | 1 | 749839 |
2 | 2 | 1972407 |
3 | 3 | 10062309 |
4 | 4 | 1522195 |
sns.distplot(tokens_by_topic["token_count"])
<matplotlib.axes._subplots.AxesSubplot at 0x7fa9904b2640>
def top_n_terms(k, n = 10):
result = (TopicTermFreq[TopicTermFreq['topic'] == k].
sort_values('count', ascending=False).head(n))
return result
pd.set_option('display.max_colwidth', None)
topic_top_n = []
for i in range(0, num_topics):
concat_terms = ', '.join(top_n_terms(i, n = num_top_terms)['term'].tolist())
topic_top_n.append([i, concat_terms])
topic_tbl = pd.DataFrame(topic_top_n, columns = ['topic', 'top_n_terms'])
topic_tbl
topic | top_n_terms | |
---|---|---|
0 | 0 | coronavirus, covid, travel, japan, airlines, aviation, uae, flights, tourism, amp |
1 | 1 | covid, coronavirus, join, bitcoin, contest, ecoins, crypto, free, earn, contestalert |
2 | 2 | covid, support, food, amp, coronavirus, community, donate, ireland, local, people |
3 | 3 | coronavirus, covid, health, pandemic, news, lockdown, outbreak, amp, virus, government |
4 | 4 | covid, coronavirus, follow, stayhome, bts, bbb, highriskcovid, day, mondaythoughts, survival |
5 | 5 | economy, people, urge, coronavirus, million, debt, package, needed, student, stimulate |
6 | 6 | usa, covid, coronavirus, america, project, trump, amp, tuesdaythoughts, topics, amazing |
7 | 7 | coronavirus, iran, covid, amp, yemen, israel, russia, people, health, syria |
8 | 8 | covid, workfromhome, home, quarantinelife, make, coronavirus, extra, online, wfh, work |
9 | 9 | covid, coronavirus, easter, god, hope, jesus, love, ramadan, prayer, pray |
10 | 10 | coronavirus, redbubble, covid, art, findyourthing, support, products, awesome, printed, rbandme |
11 | 11 | china, coronavirus, covid, wuhan, chinesevirus, wuhanvirus, chinese, chinavirus, world, ccp |
12 | 12 | covid, coronavirus, u.s, education, students, school, trump, kids, pandemic, learning |
13 | 13 | covid, coronavirus, australia, auspol, government, alert, world, australian, community, aus |
14 | 14 | covid, coronavirus, dogs, cats, amp, animals, dog, pets, cat, animal |
15 | 15 | covid, coronavirus, art, design, artist, corona, pandemic, drawing, quedateencasa, confinement |
16 | 16 | covid, pakistan, coronavirus, kashmir, cases, amp, karachi, positive, lahore, islamabad |
17 | 17 | pandemic, covid, coronavirus, amp, world, global, earthday, climatechange, crisis, virus |
18 | 18 | covid, coronavirus, memes, italia, italy, tiktok, funny, corona, news, meme |
19 | 19 | covid, coronavirus, nhs, coronavirusuk, borisjohnson, lockdown, amp, boris, people, news |
20 | 20 | coronavirus, covid, economy, oil, stocks, market, markets, stockmarket, trading, recession |
21 | 21 | coronavirus, covid, live, youtube, watch, twitch, gaming, video, gta, xbox |
22 | 22 | covid, coronavirus, technology, tech, data, cybersecurity, google, apple, app, pandemic |
23 | 23 | covid, music, coronavirus, love, nyc, london, nowplaying, unite, paris, hiphop |
24 | 24 | covid, lockdown, coronavirus, india, indiafightscorona, corona, stayhomestaysafe, coronavirusindia, fight, stay |
25 | 25 | covid, stayhome, staysafe, stayathome, coronavirus, stay, home, safe, stayhomesavelives, flattenthecurve |
26 | 26 | coronavirus, covid, nba, sports, football, due, season, nfl, mlb, olympics |
27 | 27 | covid, coronavirus, vaccine, patients, amp, sarscov, treatment, testing, test, cdc |
28 | 28 | covid, coronavirus, cases, india, positive, total, amp, delhi, state, lockdown |
29 | 29 | covid, coronavirus, corona, coronavirusoutbreak, coronaviruspandemic, coronavirusupdate, coronavirusupdates, virus, coronaoutbreak, cases |
30 | 30 | covid, coronavirus, amp, sign, relief, petition, give, american, stimulus, pandemic |
31 | 31 | covid, coronavirus, horny, sex, porn, sexy, ass, onlyfans, nudes, cum |
32 | 32 | covid, coronavirus, quarantine, lockdown, quarantinelife, stayhome, day, socialdistancing, stayathome, home |
33 | 33 | socialdistancing, covid, coronavirus, coronalockdown, stayathomeandstaysafe, listen, great, coronaupdate, click, music |
34 | 34 | coronavirus, covid, people, amp, time, it's, don't, dont, good, i'm |
35 | 35 | covid, amp, healthcare, workers, nurses, doctors, coronavirus, ppe, health, care |
36 | 36 | covid, coronavirus, read, life, poetry, book, books, blog, free, motivation |
37 | 37 | covid, coronavirus, maga, qanon, wwg, kag, fakenews, wga, amp, billgates |
38 | 38 | coronavirus, covid, due, news, facebook, marketing, movie, twitter, socialmedia, film |
39 | 39 | covid, coronavirus, lockdown, day, photography, nature, socialdistancing, love, stayhome, beautiful |
40 | 40 | covid, coronavirus, health, mentalhealth, amp, anxiety, pandemic, care, support, tips |
41 | 41 | covid, coronavirus, mask, masks, face, facemask, amp, facemasks, hands, virus |
42 | 42 | covid, nigeria, lockdown, africa, coronavirus, day, oflockdown, lagos, kenya, ghana |
43 | 43 | covid, georgia, atlanta, realestate, news, coronavirus, conspiracy, university, truth, medical |
44 | 44 | coronavirus, covid, breaking, cases, nyc, state, newyork, florida, california, positive |
45 | 45 | covid, coronavirus, amp, business, latest, crisis, pandemic, webinar, impact, read |
46 | 46 | cases, covid, coronavirus, deaths, death, total, confirmed, italy, recovered, spain |
47 | 47 | coronavirus, covid, trump, amp, trumpvirus, americans, president, cnn, donaldtrump, people |
48 | 48 | covid, canada, news, coronavirus, latest, cdnpoli, ontario, daily, toronto, covidcanada |
49 | 49 | covid, coronavirus, food, healthy, cannabis, health, coffee, lockdown, stayhome, immunity |
We usually think of the probability of a topic given a document. For this metric we calculate the probability of documents given a topic.
# from scipy.stats import entropy
# k = 1
# docs2topic_nonull = docs2topic[docs2topic['topic'].isnull() == False]
# topic_freq_over_all_docs = docs2topic[docs2topic['topic'] == k]['proportion'].tolist()
# def calc_doc_entropy(topics, base=None):
# value,counts = np.unique(topics, return_counts=True)
# return entropy(counts, base=base)
# doc_entropy = []
# for k in range(0, num_topics):
# topic_freq_over_all_docs = docs2topic[docs2topic['topic'] == k]['proportion'].tolist()
# doc_entropy.append([k, calc_doc_entropy(topic_freq_over_all_docs)])
# doc_entropy = pd.DataFrame(doc_entropy, columns = ['topic', 'doc_entropy'])
# doc_entropy.head()
The average length, in characters, of the displayed top words.
topics = TopicTermFreq['topic'].unique().tolist()
topic_ids = []
avg_word_lengths = []
for i in range(0, num_topics):
top_n = top_n_terms(k = i, n = num_top_terms)
avg_len = top_n['term'].apply(len).mean()
avg_word_lengths.append(avg_len)
topic_ids.append(i)
word_len_by_topic = pd.DataFrame({'topic': topic_ids,
'word-length': avg_word_lengths})
word_len_by_topic = word_len_by_topic.sort_values('word-length', ascending = False)
word_len_by_topic.head()
topic | word-length | |
---|---|---|
29 | 29 | 11.9 |
33 | 33 | 10.0 |
24 | 24 | 9.3 |
32 | 32 | 8.9 |
25 | 25 | 8.6 |
This metric measures whether the words in a topic tend to co-occur together. We add up a score for each distinct pair of top ranked words. The score is the log of the probability that a document containing at least one instance of the higher-ranked word also contains at least one instance of the lower-ranked word.
D(wj,wi) = number of documents containing wi and wj
D(wi) = number of topics containing wi
W = set of words describing the topic
# See coherence_calculation.ipynb
We want topics to be specific. This metric measures the distance from a topic's distribution over words to a uniform distribution. We calculate distance using Kullback-Leibler divergence.
def calc_KL_diverg_terms(TopicTermFreq, k, n):
term_count_corpus = len(TopicTermFreq['term'].unique())
token_count_topic_k = sum(TopicTermFreq[(TopicTermFreq['topic'] == k)]['count'])
topic_k_terms = top_n_terms(k = k, n = n)['term'].tolist()
KL_divergence_terms = []
for term in topic_k_terms:
term_count = TopicTermFreq[(TopicTermFreq['topic'] == k) &
(TopicTermFreq['term'] == term)]['count'].tolist()[0]
P_w_k = term_count/token_count_topic_k
KL_diverg_term = P_w_k*math.log(P_w_k/(1/term_count_corpus))
KL_divergence_terms.append([term, KL_diverg_term])
KL_divergence_terms = pd.DataFrame(KL_divergence_terms, columns = ['Term', 'uniform_dist'])
return KL_divergence_terms
KL_divergence_terms = calc_KL_diverg_terms(TopicTermFreq, k = 0, n = num_top_terms)
KL_divergence_terms.head()
Term | uniform_dist | |
---|---|---|
0 | coronavirus | 0.646345 |
1 | covid | 0.549688 |
2 | travel | 0.311497 |
3 | japan | 0.155116 |
4 | airlines | 0.099394 |
def calc_KL_diverg_unif(TopicTermFreq, k):
# calc distribution of topic k
token_counts_topic_k = TopicTermFreq[(TopicTermFreq['topic'] == k)]['count'].tolist()
token_count_topic_k = sum(token_counts_topic_k)
P_w_k = [count/token_count_topic_k for count in token_counts_topic_k]
# determine number of terms in the corpus to calc uniform dist (1/term_count_corpus)
term_count_corpus = len(TopicTermFreq['term'].unique())
KL_diverg_topic = sum([p*math.log(p/(1/term_count_corpus)) for p in P_w_k])
return KL_diverg_topic
unif_dist = []
for k in range(0, num_topics):
unif_dist.append([k, calc_KL_diverg_unif(TopicTermFreq, k = k)])
unif_dist = pd.DataFrame(unif_dist, columns = ['topic', 'uniform_dist'])
unif_dist.head()
topic | uniform_dist | |
---|---|---|
0 | 0 | 6.889917 |
1 | 1 | 8.155044 |
2 | 2 | 6.823867 |
3 | 3 | 6.630374 |
4 | 4 | 6.574626 |
This metric measures how far a topic is from the overall distribution of words in the corpus — essentially what you would get if you "trained" a model with one topic. We calculate distance using Kullback-Leibler divergence. A greater distance means the topic is more distinct; a smaller distanace means that the topic is more similar to the corpus distribution. Not surprisingly, it correlates with number of tokens since a topic that makes up a large proportion of the tokens in the corpus is likely to be more similar to the overall corpus distribution. The closest topic to the corpus distribution is "time number term part system form".
def calc_KL_diverg_corpus(TopicTermFreq, k):
# count terms across topics and corpus
num_tokens_in_corpus = TopicTermFreq['count'].sum()
num_tokens_in_topic = TopicTermFreq['count'][TopicTermFreq['topic']==k].sum()
term_count_corpus_df = TopicTermFreq.groupby('term')['count'].sum().reset_index()
term_count_corpus_df.columns = ['term', 'corpus_count']
term_count_topic_k_df = TopicTermFreq[TopicTermFreq['topic'] == k].groupby('term')['count'].sum().reset_index()
dist_df = term_count_topic_k_df.merge(term_count_corpus_df, how = 'left', on = 'term')
# calc corpus distribution
dist_df['corpus_dist'] = dist_df['corpus_count'].apply(lambda x: x/num_tokens_in_corpus)
# calc corpus distribution
dist_df['topic_dist'] = dist_df['count'].apply(lambda x: x/num_tokens_in_topic)
# calc KL-divergence
KL_diverg = []
for i in range(0, len(dist_df)):
KL_diverg.append(dist_df['topic_dist'][i]*math.log(dist_df['topic_dist'][i]/dist_df['corpus_dist'][i]))
KL_diverg = sum(KL_diverg)
return KL_diverg
corpus_dist = []
for k in range(0, num_topics):
corpus_dist.append([k, calc_KL_diverg_corpus(TopicTermFreq, k = k)])
corpus_dist = pd.DataFrame(corpus_dist, columns = ['topic', 'corpus_dist'])
corpus_dist.head()
topic | corpus_dist | |
---|---|---|
0 | 0 | 2.343306 |
1 | 1 | 3.408515 |
2 | 2 | 1.863144 |
3 | 3 | 0.708723 |
4 | 4 | 2.564010 |
This metric is equivalent to the effective number of parties metric in Political Science. For each word we calculate the inverse of the squared probability of the word in the topic, and then add those numbers up. Larger numbers indicate more specificity. It is similar to distance from the uniform distribution, but produces a value that may be more interpretable.
def calc_effective_num_words(TopicTermFreq, k):
num_tokens_in_topic = TopicTermFreq['count'][TopicTermFreq['topic']==k].sum()
topic_dist = TopicTermFreq[['term', 'count']][TopicTermFreq['topic'] == k]
topic_dist['freq'] = topic_dist['count'].apply(lambda x: x/num_tokens_in_topic)
topic_dist['sq_prob'] = topic_dist['freq'].apply(lambda x: (x*x))
effective_num_words = 1/sum(topic_dist['sq_prob'])
return effective_num_words
effective_words = []
for k in range(0, num_topics):
effective_words.append([k, calc_effective_num_words(TopicTermFreq, k = k)])
effective_words = pd.DataFrame(effective_words, columns = ['topic', 'eff_num_words'])
effective_words.head()
topic | eff_num_words | |
---|---|---|
0 | 0 | 126.926597 |
1 | 1 | 81.466377 |
2 | 2 | 128.929208 |
3 | 3 | 140.626300 |
4 | 4 | 115.824467 |
This metric looks for "bursty" words within topics. We compare two distributions over words using Jensen-Shannon distance. The first distribution P(w|k)∝Nk,w is the usual topic-word distribution proportional to the number of tokens of type w that are assigned to the topic. The second distribution Q(w|k)∝∑d𝟙(Nd,w,k>0) is proportional to the number of documents that contain at least one token of type w that is assigned to the topic. A words that occurs many times in only a few documents may appear prominently in the sorted list of words, but may not be a good representative word for the topic. This metric compares the number of times a word occurs in a topic (measured in tokens) and the number of documents the word occurs in as that topic (instances of the word assigned to other topics are not counted). The highest ranked topic in this metric is the "polish poland danish denmark sweden swedish na norway norwegian sk red" topic, suggesting that those ill-fitting words may be isolated in a few documents. Although this metric has the same goal as coherence, the two don't appear to correlate well: bursty words aren't necessarily unrelated to the topic, they're just unusually frequent in certain contexts.
Some topics are specific, while others aren't really "topics" but language that comes up because we are writing in a certain context. Academic writing will talk about "paper abstract data", and a Wikipedia article will talk about "list links history". The difference is often measurable in terms of burstiness. A content-ful topic will occur in relatively few documents, but when it does, it will produce a lot of tokens. A "background" topic will occur in many documents and have a high overall token count, but never produce many tokens in any single document. This metric counts the frequency at which a given topic is the single most frequent topic in a document. The rank_1 metric shows how many times each topic is ranked first in terms of document proportion (hence rank_1) for the documents in which it occurs. Specific topics like "music album band song" or "cell cells disease dna blood treatment" are the "rank 1" topic in many documents. High token-count topics often have few rank-1 documents. This metric is often useful as a way to identify corpus-specific stopwords. But rarer topics can also have few rank-1 documents: "day year calendar days years month" is a representative example — topics for days of the week and units of measurement often appear in documents as a distinct discourse, but they are rarely the focus of a document.
def calc_rank_1_docs(docs2topic, k):
rank_1_perc = 0
try:
rank_1_docs = docs2topic[(docs2topic['topic'] == k) & (docs2topic['rank'] == 1)]['doc_id'].unique().tolist()
docs_with_topic_k = docs2topic[docs2topic['topic'] == k]['doc_id'].unique().tolist()
rank_1_perc = len(rank_1_docs)/len(docs_with_topic_k)
except:
pass
return rank_1_perc
rank1_perc = []
for k in range(0, num_topics):
rank1_perc.append([k, calc_rank_1_docs(docs2topic, k)])
rank1_perc = pd.DataFrame(rank1_perc, columns = ['topic', 'rank_1_docs'])
rank1_perc.head()
topic | rank_1_docs | |
---|---|---|
0 | 0 | 0.800777 |
1 | 1 | 0.784936 |
2 | 2 | 0.804646 |
3 | 3 | 0.766618 |
4 | 4 | 0.835698 |
This metric has a similar motivation to the rank-1-docs metric. For each document we can calculate the percentage of that document assigned to a given topic. We can then observe how often that percentage is above a certain threshold, by default 30%
print("docs with topic k proprotion >= 0.3: " + str(len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.3)])))
print("docs with topic k proprotion > 0: " + str(len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0)])))
docs with topic k proprotion >= 0.3: 4222 docs with topic k proprotion > 0: 4222
This metric reports the ratio of allocation counts at two different thresholds, by default 50% and 2%.
numerator = len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.5)])
denominator = len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.3)])
print("docs with topic k proportion >= 0.5: " + str(len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.5)])))
print("docs with topic k proportion >= 0.3: " + str(len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.3)])))
print("allocation ratio: " + str(numerator/denominator))
docs with topic k proportion >= 0.5: 2425 docs with topic k proportion >= 0.3: 4222 allocation ratio: 0.5743723353860729
This metric measures the extent to which the top words for this topic are do not appear as top words in other topics -- i.e., the extent to which its top words are 'exclusive.' The value is the average, over each top word, of the probability of that word in the topic divided by the sum of the probabilities of that word in all topics. Of the top words in the topic, how often do they occur in other topics? Exclusivity correlates (negatively) with token count, but also indicates vaguer, more general topics. "black hand back body cross man" is about the same size as "isbn book press published work books", but is much less exclusive.
import numpy as np
from numpy import mean
def calc_topic_exclusivity(TopicTermFreq, k, n):
top_terms = top_n_terms(k = k, n = n)['term'].tolist()
topic_exclusivity = []
for term in top_terms:
# calc probability of term in topic k
count_term_in_topic_k = TopicTermFreq[(TopicTermFreq['term'] == term) &
(TopicTermFreq['topic'] == k)]['count'].sum()
tokens_topic_k = TopicTermFreq[TopicTermFreq['topic'] == k]['count'].sum()
prob_term_in_topic_k = count_term_in_topic_k/tokens_topic_k
# calc probability of term in all topics
tokens_by_topic = TopicTermFreq.groupby('topic').sum().reset_index().rename(columns = {'count': 'token_count'})
count_term_all_topics = TopicTermFreq[TopicTermFreq['term'] == term]
prob_term_k_all_topics = count_term_all_topics.merge(tokens_by_topic, how = 'left', on = 'topic')
prob_term_k_all_topics['freq'] = prob_term_k_all_topics['count']/prob_term_k_all_topics['token_count']
prob_term_k_all_topics = sum(prob_term_k_all_topics['freq'])
# calc exclusivity
term_exclusivity = prob_term_in_topic_k/prob_term_k_all_topics
#print(term + ": " + str(term_exclusivity))
topic_exclusivity.append(term_exclusivity)
topic_exclusivity = mean(topic_exclusivity)
return topic_exclusivity
topic_exclusivity = []
for k in range(0, num_topics):
topic_exclusivity.append([k, calc_topic_exclusivity(TopicTermFreq, k, n = num_top_terms)])
topic_exclusivity = pd.DataFrame(topic_exclusivity, columns = ['topic', 'exclusivity'])
topic_exclusivity.head()
topic | exclusivity | |
---|---|---|
0 | 0 | 0.556460 |
1 | 1 | 0.650400 |
2 | 2 | 0.235125 |
3 | 3 | 0.059785 |
4 | 4 | 0.379354 |
eval_summary = topic_tbl.merge(corpus_dist, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(unif_dist, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(tokens_by_topic, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(word_len_by_topic, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(effective_words, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(topic_exclusivity, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(rank1_perc, how = 'left', on = 'topic')
eval_summary = eval_summary.sort_values('topic')
eval_summary.reset_index(drop = True, inplace = True)
cols = ['topic', 'token_count', 'word-length', 'uniform_dist', 'corpus_dist',
'eff_num_words', 'rank_1_docs', 'exclusivity', 'top_n_terms']
eval_summary = eval_summary[cols]
#eval_summary.to_csv('/Users/dankoban/Documents/EM6575/twitter/twitter_topic_diagnostics_koban.csv', index=False)
eval_summary.head()
topic | token_count | word-length | uniform_dist | corpus_dist | eff_num_words | rank_1_docs | exclusivity | top_n_terms | |
---|---|---|---|---|---|---|---|---|---|
0 | 0 | 1322510 | 6.3 | 6.889917 | 2.343306 | 126.926597 | 0.800777 | 0.556460 | coronavirus, covid, travel, japan, airlines, aviation, uae, flights, tourism, amp |
1 | 1 | 749839 | 6.6 | 8.155044 | 3.408515 | 81.466377 | 0.784936 | 0.650400 | covid, coronavirus, join, bitcoin, contest, ecoins, crypto, free, earn, contestalert |
2 | 2 | 1972407 | 6.3 | 6.823867 | 1.863144 | 128.929208 | 0.804646 | 0.235125 | covid, support, food, amp, coronavirus, community, donate, ireland, local, people |
3 | 3 | 10062309 | 6.8 | 6.630374 | 0.708723 | 140.626300 | 0.766618 | 0.059785 | coronavirus, covid, health, pandemic, news, lockdown, outbreak, amp, virus, government |
4 | 4 | 1522195 | 7.4 | 6.574626 | 2.564010 | 115.824467 | 0.835698 | 0.379354 | covid, coronavirus, follow, stayhome, bts, bbb, highriskcovid, day, mondaythoughts, survival |