Topic Model Diagnostics¶

Load Word Counts by Topic / Parse into Tidy Structure¶

In [ ]:

import pandas as pd
import re

def tidy_word_count(data, row):
    df = pd.DataFrame({'term': [], 'topic_count':[]})
    try:
        topic_count = data[row][2:len(data[row])]    
        df = pd.DataFrame({'topic_count': topic_count})  
        df['term'] = data[row][1]    
    except:
        pass
    return df

data = []
#with open('/Users/dankoban/Documents/EM6575/mallet_command_line/ct.wordtopiccounts','r') as infile:
with open('/Users/dankoban/Documents/EM6575/twitter/hashtag model/hashtags/hashtags.wordtopiccounts','r') as infile:
    for line in infile:        
        line = line.split(' ')        
        data.append(line)      

counter = 0
tidy_dfs = []
for i in range(0, len(data)):
    tidy_dfs.append(tidy_word_count(data = data, row = i))
    counter += 1
    if counter %10000 == 0:
        print(str(counter) + ' out of ' + str(len(data)))
        
df = pd.concat(tidy_dfs)

df['topic_count'] = df['topic_count'].apply(lambda x: re.sub(r'\n', '', x)) 
df['topic'] = df['topic_count'].apply(lambda x: x.split(":")[0])
df['count'] = df['topic_count'].apply(lambda x: x.split(":")[1])
df = df[['term', 'topic', 'count']]            
df.reset_index(inplace = True, drop = True)
#df.to_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/ct_tidy_topics.csv', index=False)        
df.to_csv('/Users/dankoban/Documents/EM6575/twitter/twitter_tidy_topics.csv', index=False)        

Load Topics by Documents / Parse into Tidy Structure¶

In [ ]:

import pandas as pd
data = []
#with open('/Users/dankoban/Documents/EM6575/mallet_command_line/ct.doctopics_sparse','r') as infile:
with open('/Users/dankoban/Documents/EM6575/twitter/hashtag model/hashtags.doctopics_sparse','r') as infile:
    for line in infile:        
        line = line.rstrip().split('\t')
        data.append(line)

df = pd.DataFrame(data[1:], columns = ['doc_id', 'name', 'topic_1', 'proportion_1', 'topic_2', 'proportion_2', 'topic_3', 'proportion_3'])
df['topic_1'] = df['topic_1'].astype('float')
df['topic_2'] = df['topic_2'].astype('float')
df['topic_3'] = df['topic_3'].astype('float')
df['proportion_1'] = df['proportion_1'].astype('float')
df['proportion_2'] = df['proportion_2'].astype('float')
df['proportion_3'] = df['proportion_3'].astype('float')

rank1_docs = df[['doc_id', 'name', 'topic_1', 'proportion_1']]
rank1_docs.columns = ['doc_id', 'name', 'topic', 'proportion']
rank1_docs = rank1_docs.assign(rank = 1)

rank2_docs = df[['doc_id', 'name', 'topic_2', 'proportion_2']]
rank2_docs.columns = ['doc_id', 'name', 'topic', 'proportion']
rank2_docs = rank2_docs.assign(rank = 2)

rank3_docs = df[['doc_id', 'name', 'topic_3', 'proportion_3']]
rank3_docs.columns = ['doc_id', 'name', 'topic', 'proportion']
rank3_docs = rank3_docs.assign(rank = 3)

df = pd.concat([rank1_docs, rank2_docs, rank3_docs])
df = df[df['proportion'].notnull()]
#df = df[df['proportion'].isna() == False]
#df.to_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/tidy_docs2topics.csv', index=False)  
df.to_csv('/Users/dankoban/Documents/EM6575/twitter/tidy_docs2topics.csv', index=False)  

Load data from step 1 and 2¶

Load the parsed data to save processing time.

In [1]:

import seaborn as sns
import pandas as pd
import math
import numpy as np

num_topics = 50
num_top_terms = 10

#TopicTermFreq = pd.read_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/tidy_topics.csv')
#TopicTermFreq = pd.read_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/ct_tidy_topics.csv')
TopicTermFreq = pd.read_csv('/Users/dankoban/Documents/EM6575/twitter/twitter_tidy_topics.csv')
TopicTermFreq.head()

Out[1]:

	term	topic	count
0	bud	34	179
1	bud	49	133
2	bud	31	117
3	bud	29	106
4	bud	39	76

In [2]:

#docs2topic = pd.read_csv('/Users/dankoban/Documents/EM6575/mallet_command_line/tidy_docs2topics.csv')
docs2topic = pd.read_csv('/Users/dankoban/Documents/EM6575/twitter/tidy_docs2topics.csv')
len(docs2topic['doc_id'].unique())

Out[2]:

Tokens per Topic¶

These descriptions are designed to make sense without mathematical definitions, but we include them for specificity. We'll use the symbols $w$ for a word, $k$ for a topic, and $d$ for a document. Several metrics count tokens. For these we define variables $N_{d,w,k}$ as the number of times word $w$ occurs in document $d$ assigned to topic $k$ , $N_{d,k}$ as $\sum_wN_{d,w,k}$ . Other metrics involve document frequencies. $D(w)$ is the number of documents that contain at least one token of type $w$ , and $D(w_1,w_2)$ is the number of documents that contain at least one $w_1$ and one $w_2$ . Finally, we define $W_k$ to be the $N$ most probable words in topic $k$ . $𝟙(p)$ is equal to 1 if predicate $p$ is true, and 0 otherwise.

Tokens. This metric measures the number of word tokens currently assigned to the topic $N_k=\sum_dN_{d,k}$ . Comparing this number to the sum of the token counts for all topics will give you the proportion of the corpus assigned to the topic. If you are using optimized hyperparameters this number will vary considerably across topics; otherwise it will be roughly the same for all topics. We usually find topics most interesting if they are not on the extremes of this range. Small numbers (relative to other topics) are often a sign that a topic may not be reliable: we don't have enough observations to get a good sense of the topic's word distribution. Large numbers may also be bad: extremely frequent topics are often "not-quite-stopwords". For example, in the sample model the largest topic is "time number term part system form".

In [3]:

tokens_by_topic = (TopicTermFreq.groupby('topic').
                   sum().
                   reset_index().
                   rename(columns = {'count': 'token_count'}))
tokens_by_topic.head()    

Out[3]:

	topic	token_count
0	0	1322510
1	1	749839
2	2	1972407
3	3	10062309
4	4	1522195

In [4]:

sns.distplot(tokens_by_topic["token_count"])

Out[4]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa9904b2640>

In [5]:

def top_n_terms(k, n = 10):
    result = (TopicTermFreq[TopicTermFreq['topic'] == k].
                  sort_values('count', ascending=False).head(n))
    return result

pd.set_option('display.max_colwidth', None)
topic_top_n = []
for i in range(0, num_topics):
    concat_terms = ', '.join(top_n_terms(i, n = num_top_terms)['term'].tolist())
    topic_top_n.append([i, concat_terms])

topic_tbl = pd.DataFrame(topic_top_n, columns = ['topic', 'top_n_terms'])
topic_tbl

Out[5]:

	topic	top_n_terms
0	0	coronavirus, covid, travel, japan, airlines, aviation, uae, flights, tourism, amp
1	1	covid, coronavirus, join, bitcoin, contest, ecoins, crypto, free, earn, contestalert
2	2	covid, support, food, amp, coronavirus, community, donate, ireland, local, people
3	3	coronavirus, covid, health, pandemic, news, lockdown, outbreak, amp, virus, government
4	4	covid, coronavirus, follow, stayhome, bts, bbb, highriskcovid, day, mondaythoughts, survival
5	5	economy, people, urge, coronavirus, million, debt, package, needed, student, stimulate
6	6	usa, covid, coronavirus, america, project, trump, amp, tuesdaythoughts, topics, amazing
7	7	coronavirus, iran, covid, amp, yemen, israel, russia, people, health, syria
8	8	covid, workfromhome, home, quarantinelife, make, coronavirus, extra, online, wfh, work
9	9	covid, coronavirus, easter, god, hope, jesus, love, ramadan, prayer, pray
10	10	coronavirus, redbubble, covid, art, findyourthing, support, products, awesome, printed, rbandme
11	11	china, coronavirus, covid, wuhan, chinesevirus, wuhanvirus, chinese, chinavirus, world, ccp
12	12	covid, coronavirus, u.s, education, students, school, trump, kids, pandemic, learning
13	13	covid, coronavirus, australia, auspol, government, alert, world, australian, community, aus
14	14	covid, coronavirus, dogs, cats, amp, animals, dog, pets, cat, animal
15	15	covid, coronavirus, art, design, artist, corona, pandemic, drawing, quedateencasa, confinement
16	16	covid, pakistan, coronavirus, kashmir, cases, amp, karachi, positive, lahore, islamabad
17	17	pandemic, covid, coronavirus, amp, world, global, earthday, climatechange, crisis, virus
18	18	covid, coronavirus, memes, italia, italy, tiktok, funny, corona, news, meme
19	19	covid, coronavirus, nhs, coronavirusuk, borisjohnson, lockdown, amp, boris, people, news
20	20	coronavirus, covid, economy, oil, stocks, market, markets, stockmarket, trading, recession
21	21	coronavirus, covid, live, youtube, watch, twitch, gaming, video, gta, xbox
22	22	covid, coronavirus, technology, tech, data, cybersecurity, google, apple, app, pandemic
23	23	covid, music, coronavirus, love, nyc, london, nowplaying, unite, paris, hiphop
24	24	covid, lockdown, coronavirus, india, indiafightscorona, corona, stayhomestaysafe, coronavirusindia, fight, stay
25	25	covid, stayhome, staysafe, stayathome, coronavirus, stay, home, safe, stayhomesavelives, flattenthecurve
26	26	coronavirus, covid, nba, sports, football, due, season, nfl, mlb, olympics
27	27	covid, coronavirus, vaccine, patients, amp, sarscov, treatment, testing, test, cdc
28	28	covid, coronavirus, cases, india, positive, total, amp, delhi, state, lockdown
29	29	covid, coronavirus, corona, coronavirusoutbreak, coronaviruspandemic, coronavirusupdate, coronavirusupdates, virus, coronaoutbreak, cases
30	30	covid, coronavirus, amp, sign, relief, petition, give, american, stimulus, pandemic
31	31	covid, coronavirus, horny, sex, porn, sexy, ass, onlyfans, nudes, cum
32	32	covid, coronavirus, quarantine, lockdown, quarantinelife, stayhome, day, socialdistancing, stayathome, home
33	33	socialdistancing, covid, coronavirus, coronalockdown, stayathomeandstaysafe, listen, great, coronaupdate, click, music
34	34	coronavirus, covid, people, amp, time, it's, don't, dont, good, i'm
35	35	covid, amp, healthcare, workers, nurses, doctors, coronavirus, ppe, health, care
36	36	covid, coronavirus, read, life, poetry, book, books, blog, free, motivation
37	37	covid, coronavirus, maga, qanon, wwg, kag, fakenews, wga, amp, billgates
38	38	coronavirus, covid, due, news, facebook, marketing, movie, twitter, socialmedia, film
39	39	covid, coronavirus, lockdown, day, photography, nature, socialdistancing, love, stayhome, beautiful
40	40	covid, coronavirus, health, mentalhealth, amp, anxiety, pandemic, care, support, tips
41	41	covid, coronavirus, mask, masks, face, facemask, amp, facemasks, hands, virus
42	42	covid, nigeria, lockdown, africa, coronavirus, day, oflockdown, lagos, kenya, ghana
43	43	covid, georgia, atlanta, realestate, news, coronavirus, conspiracy, university, truth, medical
44	44	coronavirus, covid, breaking, cases, nyc, state, newyork, florida, california, positive
45	45	covid, coronavirus, amp, business, latest, crisis, pandemic, webinar, impact, read
46	46	cases, covid, coronavirus, deaths, death, total, confirmed, italy, recovered, spain
47	47	coronavirus, covid, trump, amp, trumpvirus, americans, president, cnn, donaldtrump, people
48	48	covid, canada, news, coronavirus, latest, cdnpoli, ontario, daily, toronto, covidcanada
49	49	covid, coronavirus, food, healthy, cannabis, health, coffee, lockdown, stayhome, immunity

2. Document Entropy¶

We usually think of the probability of a topic given a document. For this metric we calculate the probability of documents given a topic.

$P(d|k)=\frac{N_{d,k}}{\sum_{d′}N_{d′,k}}$
We count the frequency of a topic over all documents, and normalize to get a distribution, then calculate the entropy of that distribution

$H(P(d|k))$ . A topic with low entropy (i.e., high predictability) will be concentrated in a few documents, while a topic with higher entropy will be spread evenly over many documents. In the example topic model, the topics "list links history external information site" and "game team league player players football games" have roughly the same number of tokens, but the "list" topic has much higher entropy. It occurs to a small degree in many documents, while the "game" topic occurs a lot in a smaller number of documents. Low entropy isn't necessarily good: it can indicate unusual documents (did you accidentally import the Mac .DS_Store file?) or the presence of documents in other languages.

In [ ]:

# from scipy.stats import entropy
# k = 1
# docs2topic_nonull = docs2topic[docs2topic['topic'].isnull() == False]
# topic_freq_over_all_docs = docs2topic[docs2topic['topic'] == k]['proportion'].tolist()

# def calc_doc_entropy(topics, base=None):
#     value,counts = np.unique(topics, return_counts=True)
#     return entropy(counts, base=base)

# doc_entropy = []
# for k in range(0, num_topics):
#     topic_freq_over_all_docs = docs2topic[docs2topic['topic'] == k]['proportion'].tolist()
#     doc_entropy.append([k, calc_doc_entropy(topic_freq_over_all_docs)])  
# doc_entropy = pd.DataFrame(doc_entropy, columns = ['topic', 'doc_entropy'])
# doc_entropy.head()

3. Word Length¶

The average length, in characters, of the displayed top words.

$\frac{1}{|W|}\sum_{w∈W}len(w)$
Words are not weighted by probablity or rank position. This is a useful proxy for topic specificity. Longer words often carry more specific meaning, so if the topic brings together lots of short words, it's probably not a very specific topic. This can also be a good way to pick up the "hey, looks like we've got some Spanish!" topic because we often pick up the short words that would be stopwords if we were modeling a corpus in that language. The example topic "de french la france spanish spain italian paris el le" has a number of this type of short words, but they seem to be due to Wikipedia articles talking about French, Spanish, and Italian things rather than actual text in those languages.

In [6]:

topics = TopicTermFreq['topic'].unique().tolist()

topic_ids = []
avg_word_lengths = []
for i in range(0, num_topics):
    top_n = top_n_terms(k = i, n = num_top_terms)
    avg_len = top_n['term'].apply(len).mean()
    avg_word_lengths.append(avg_len)
    topic_ids.append(i)

word_len_by_topic = pd.DataFrame({'topic': topic_ids,
                                  'word-length': avg_word_lengths})

word_len_by_topic = word_len_by_topic.sort_values('word-length', ascending = False)
word_len_by_topic.head()

Out[6]:

	topic	word-length
29	29	11.9
33	33	10.0
24	24	9.3
32	32	8.9
25	25	8.6

4. Coherence¶

This metric measures whether the words in a topic tend to co-occur together. We add up a score for each distinct pair of top ranked words. The score is the log of the probability that a document containing at least one instance of the higher-ranked word also contains at least one instance of the lower-ranked word.

$\sum_i\sum_jlog\frac{D(w_j,w_i)+β}{D(w_i)}$
To avoid log zero errors we add the "beta" topic-word smoothing parameter specified when you calculate diagnostics. Since these scores are log probabilities they are negative. Large negative values indicate words that don't co-occur often; values closer to zero indicate that words tend to co-occur more often. The least coherent topic in the sample file is "polish poland danish denmark sweden swedish na norway norwegian sk red iceland bj baltic copenhagen cave greenland krak gda faroese". This topic seems to be about Northern and Eastern European countries, but the short abbreviations "na" and "sk" and the words "red" and "cave" don't really fit.

$D(w_j,w_i)$ = number of documents containing $w_i$ and $w_j$

$D(w_i)$ = number of topics containing $w_i$

$W$ = set of words describing the topic

In [ ]:

# See coherence_calculation.ipynb

5. Uniform Dist¶

We want topics to be specific. This metric measures the distance from a topic's distribution over words to a uniform distribution. We calculate distance using Kullback-Leibler divergence.

$\sum_wP(w|k)log\frac{P(w|k)}{\frac{1}{|v|}}$
Larger values indicate more specificity.

In [7]:

def calc_KL_diverg_terms(TopicTermFreq, k, n):
    term_count_corpus = len(TopicTermFreq['term'].unique())  
    token_count_topic_k = sum(TopicTermFreq[(TopicTermFreq['topic'] == k)]['count'])
    topic_k_terms = top_n_terms(k = k, n = n)['term'].tolist()
    KL_divergence_terms = []    
    for term in topic_k_terms:    
        term_count = TopicTermFreq[(TopicTermFreq['topic'] == k) & 
                                   (TopicTermFreq['term'] == term)]['count'].tolist()[0]
        P_w_k = term_count/token_count_topic_k
        KL_diverg_term = P_w_k*math.log(P_w_k/(1/term_count_corpus))
        KL_divergence_terms.append([term, KL_diverg_term])
    KL_divergence_terms = pd.DataFrame(KL_divergence_terms, columns = ['Term', 'uniform_dist'])
    return KL_divergence_terms

KL_divergence_terms = calc_KL_diverg_terms(TopicTermFreq, k = 0, n = num_top_terms)
KL_divergence_terms.head()

Out[7]:

	Term	uniform_dist
0	coronavirus	0.646345
1	covid	0.549688
2	travel	0.311497
3	japan	0.155116
4	airlines	0.099394

In [8]:

def calc_KL_diverg_unif(TopicTermFreq, k):
    # calc distribution of topic k
    token_counts_topic_k = TopicTermFreq[(TopicTermFreq['topic'] == k)]['count'].tolist()
    token_count_topic_k = sum(token_counts_topic_k)      
    P_w_k = [count/token_count_topic_k for count in token_counts_topic_k]
    
    # determine number of terms in the corpus to calc uniform dist (1/term_count_corpus)
    term_count_corpus = len(TopicTermFreq['term'].unique())        
    KL_diverg_topic = sum([p*math.log(p/(1/term_count_corpus)) for p in P_w_k])
    return KL_diverg_topic

unif_dist = []
for k in range(0, num_topics):
    unif_dist.append([k, calc_KL_diverg_unif(TopicTermFreq, k = k)])
unif_dist = pd.DataFrame(unif_dist, columns = ['topic', 'uniform_dist'])    
unif_dist.head()

Out[8]:

	topic	uniform_dist
0	0	6.889917
1	1	8.155044
2	2	6.823867
3	3	6.630374
4	4	6.574626

6. Corpus Dist¶

This metric measures how far a topic is from the overall distribution of words in the corpus — essentially what you would get if you "trained" a model with one topic. We calculate distance using Kullback-Leibler divergence. A greater distance means the topic is more distinct; a smaller distanace means that the topic is more similar to the corpus distribution. Not surprisingly, it correlates with number of tokens since a topic that makes up a large proportion of the tokens in the corpus is likely to be more similar to the overall corpus distribution. The closest topic to the corpus distribution is "time number term part system form".

In [9]:

def calc_KL_diverg_corpus(TopicTermFreq, k):
    # count terms across topics and corpus
    num_tokens_in_corpus = TopicTermFreq['count'].sum()
    num_tokens_in_topic = TopicTermFreq['count'][TopicTermFreq['topic']==k].sum()
    term_count_corpus_df = TopicTermFreq.groupby('term')['count'].sum().reset_index()
    term_count_corpus_df.columns = ['term', 'corpus_count']
    term_count_topic_k_df = TopicTermFreq[TopicTermFreq['topic'] == k].groupby('term')['count'].sum().reset_index()
    dist_df = term_count_topic_k_df.merge(term_count_corpus_df, how = 'left', on = 'term')
    
    # calc corpus distribution
    dist_df['corpus_dist'] = dist_df['corpus_count'].apply(lambda x: x/num_tokens_in_corpus)
    
    # calc corpus distribution
    dist_df['topic_dist'] = dist_df['count'].apply(lambda x: x/num_tokens_in_topic)
    
    # calc KL-divergence
    KL_diverg = []
    for i in range(0, len(dist_df)):
        KL_diverg.append(dist_df['topic_dist'][i]*math.log(dist_df['topic_dist'][i]/dist_df['corpus_dist'][i]))
    KL_diverg = sum(KL_diverg)   
    return KL_diverg

corpus_dist = []
for k in range(0, num_topics):
    corpus_dist.append([k, calc_KL_diverg_corpus(TopicTermFreq, k = k)])
corpus_dist = pd.DataFrame(corpus_dist, columns = ['topic', 'corpus_dist']) 
corpus_dist.head()

Out[9]:

	topic	corpus_dist
0	0	2.343306
1	1	3.408515
2	2	1.863144
3	3	0.708723
4	4	2.564010

7. Effective Number of Words¶

This metric is equivalent to the effective number of parties metric in Political Science. For each word we calculate the inverse of the squared probability of the word in the topic, and then add those numbers up. Larger numbers indicate more specificity. It is similar to distance from the uniform distribution, but produces a value that may be more interpretable.

In [10]:

def calc_effective_num_words(TopicTermFreq, k):
    num_tokens_in_topic = TopicTermFreq['count'][TopicTermFreq['topic']==k].sum()
    topic_dist = TopicTermFreq[['term', 'count']][TopicTermFreq['topic'] == k]
    topic_dist['freq'] = topic_dist['count'].apply(lambda x: x/num_tokens_in_topic) 
    topic_dist['sq_prob'] =  topic_dist['freq'].apply(lambda x: (x*x))
    effective_num_words = 1/sum(topic_dist['sq_prob'])
    return effective_num_words

effective_words = []
for k in range(0, num_topics):
    effective_words.append([k, calc_effective_num_words(TopicTermFreq, k = k)])
effective_words = pd.DataFrame(effective_words, columns = ['topic', 'eff_num_words']) 
effective_words.head()

Out[10]:

	topic	eff_num_words
0	0	126.926597
1	1	81.466377
2	2	128.929208
3	3	140.626300
4	4	115.824467

8. Token/Document Discrepancy¶

This metric looks for "bursty" words within topics. We compare two distributions over words using Jensen-Shannon distance. The first distribution $P(w|k)∝N_{k,w}$ is the usual topic-word distribution proportional to the number of tokens of type w that are assigned to the topic. The second distribution $Q(w|k)∝\sum_d𝟙(N_{d,w,k}>0)$ is proportional to the number of documents that contain at least one token of type w that is assigned to the topic. A words that occurs many times in only a few documents may appear prominently in the sorted list of words, but may not be a good representative word for the topic. This metric compares the number of times a word occurs in a topic (measured in tokens) and the number of documents the word occurs in as that topic (instances of the word assigned to other topics are not counted). The highest ranked topic in this metric is the "polish poland danish denmark sweden swedish na norway norwegian sk red" topic, suggesting that those ill-fitting words may be isolated in a few documents. Although this metric has the same goal as coherence, the two don't appear to correlate well: bursty words aren't necessarily unrelated to the topic, they're just unusually frequent in certain contexts.

In [ ]:

9. Rank 1 Documents¶

Some topics are specific, while others aren't really "topics" but language that comes up because we are writing in a certain context. Academic writing will talk about "paper abstract data", and a Wikipedia article will talk about "list links history". The difference is often measurable in terms of burstiness. A content-ful topic will occur in relatively few documents, but when it does, it will produce a lot of tokens. A "background" topic will occur in many documents and have a high overall token count, but never produce many tokens in any single document. This metric counts the frequency at which a given topic is the single most frequent topic in a document. The rank_1 metric shows how many times each topic is ranked first in terms of document proportion (hence rank_1) for the documents in which it occurs. Specific topics like "music album band song" or "cell cells disease dna blood treatment" are the "rank 1" topic in many documents. High token-count topics often have few rank-1 documents. This metric is often useful as a way to identify corpus-specific stopwords. But rarer topics can also have few rank-1 documents: "day year calendar days years month" is a representative example — topics for days of the week and units of measurement often appear in documents as a distinct discourse, but they are rarely the focus of a document.

In [11]:

def calc_rank_1_docs(docs2topic, k):
    rank_1_perc = 0
    try:
        rank_1_docs = docs2topic[(docs2topic['topic'] == k) & (docs2topic['rank'] == 1)]['doc_id'].unique().tolist()
        docs_with_topic_k = docs2topic[docs2topic['topic'] == k]['doc_id'].unique().tolist()
        rank_1_perc = len(rank_1_docs)/len(docs_with_topic_k)
    except:
        pass
    return rank_1_perc

rank1_perc = []
for k in range(0, num_topics):
    rank1_perc.append([k, calc_rank_1_docs(docs2topic, k)])
rank1_perc = pd.DataFrame(rank1_perc, columns = ['topic', 'rank_1_docs']) 
rank1_perc.head()    

Out[11]:

	topic	rank_1_docs
0	0	0.800777
1	1	0.784936
2	2	0.804646
3	3	0.766618
4	4	0.835698

10. Allocation Count¶

This metric has a similar motivation to the rank-1-docs metric. For each document we can calculate the percentage of that document assigned to a given topic. We can then observe how often that percentage is above a certain threshold, by default 30%

In [12]:

print("docs with topic k proprotion >= 0.3: " + str(len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.3)])))
print("docs with topic k proprotion > 0: " + str(len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0)])))

docs with topic k proprotion >= 0.3: 4222
docs with topic k proprotion > 0: 4222

11. Allocation Ratio¶

This metric reports the ratio of allocation counts at two different thresholds, by default 50% and 2%.

In [13]:

numerator = len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.5)])
denominator = len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.3)])
print("docs with topic k proportion >= 0.5: " + str(len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.5)])))
print("docs with topic k proportion >= 0.3: " + str(len(docs2topic[(docs2topic['topic'] == 1) & (docs2topic['proportion'] >= 0.3)])))
print("allocation ratio: " + str(numerator/denominator))

docs with topic k proportion >= 0.5: 2425
docs with topic k proportion >= 0.3: 4222
allocation ratio: 0.5743723353860729

12. Exclusivity¶

This metric measures the extent to which the top words for this topic are do not appear as top words in other topics -- i.e., the extent to which its top words are 'exclusive.' The value is the average, over each top word, of the probability of that word in the topic divided by the sum of the probabilities of that word in all topics. Of the top words in the topic, how often do they occur in other topics? Exclusivity correlates (negatively) with token count, but also indicates vaguer, more general topics. "black hand back body cross man" is about the same size as "isbn book press published work books", but is much less exclusive.

In [14]:

import numpy as np
from numpy import mean

def calc_topic_exclusivity(TopicTermFreq, k, n):
    top_terms = top_n_terms(k = k, n = n)['term'].tolist()    
    topic_exclusivity = []
    for term in top_terms:
        # calc probability of term in topic k
        count_term_in_topic_k = TopicTermFreq[(TopicTermFreq['term'] == term) & 
                                              (TopicTermFreq['topic'] == k)]['count'].sum()
        tokens_topic_k = TopicTermFreq[TopicTermFreq['topic'] == k]['count'].sum()
        prob_term_in_topic_k = count_term_in_topic_k/tokens_topic_k        
    
        # calc probability of term in all topics
        tokens_by_topic = TopicTermFreq.groupby('topic').sum().reset_index().rename(columns = {'count': 'token_count'})
        count_term_all_topics = TopicTermFreq[TopicTermFreq['term'] == term]
        prob_term_k_all_topics = count_term_all_topics.merge(tokens_by_topic, how = 'left', on = 'topic')
        prob_term_k_all_topics['freq'] = prob_term_k_all_topics['count']/prob_term_k_all_topics['token_count']
        prob_term_k_all_topics = sum(prob_term_k_all_topics['freq'])  
    
        # calc  exclusivity 
        term_exclusivity = prob_term_in_topic_k/prob_term_k_all_topics
        #print(term + ": " + str(term_exclusivity))
        topic_exclusivity.append(term_exclusivity)
    topic_exclusivity = mean(topic_exclusivity)
    return topic_exclusivity

topic_exclusivity = []
for k in range(0, num_topics):
    topic_exclusivity.append([k, calc_topic_exclusivity(TopicTermFreq, k, n = num_top_terms)])
topic_exclusivity = pd.DataFrame(topic_exclusivity, columns = ['topic', 'exclusivity'])     
topic_exclusivity.head()

Out[14]:

	topic	exclusivity
0	0	0.556460
1	1	0.650400
2	2	0.235125
3	3	0.059785
4	4	0.379354

In [15]:

eval_summary = topic_tbl.merge(corpus_dist, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(unif_dist, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(tokens_by_topic, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(word_len_by_topic, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(effective_words, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(topic_exclusivity, how = 'left', on = 'topic')
eval_summary = eval_summary.merge(rank1_perc, how = 'left', on = 'topic')
eval_summary = eval_summary.sort_values('topic')
eval_summary.reset_index(drop = True, inplace = True)
cols = ['topic', 'token_count', 'word-length', 'uniform_dist', 'corpus_dist', 
        'eff_num_words', 'rank_1_docs', 'exclusivity', 'top_n_terms']
eval_summary = eval_summary[cols]
#eval_summary.to_csv('/Users/dankoban/Documents/EM6575/twitter/twitter_topic_diagnostics_koban.csv', index=False) 
eval_summary.head()

Out[15]:

	topic	token_count	word-length	uniform_dist	corpus_dist	eff_num_words	rank_1_docs	exclusivity	top_n_terms
0	0	1322510	6.3	6.889917	2.343306	126.926597	0.800777	0.556460	coronavirus, covid, travel, japan, airlines, aviation, uae, flights, tourism, amp
1	1	749839	6.6	8.155044	3.408515	81.466377	0.784936	0.650400	covid, coronavirus, join, bitcoin, contest, ecoins, crypto, free, earn, contestalert
2	2	1972407	6.3	6.823867	1.863144	128.929208	0.804646	0.235125	covid, support, food, amp, coronavirus, community, donate, ireland, local, people
3	3	10062309	6.8	6.630374	0.708723	140.626300	0.766618	0.059785	coronavirus, covid, health, pandemic, news, lockdown, outbreak, amp, virus, government
4	4	1522195	7.4	6.574626	2.564010	115.824467	0.835698	0.379354	covid, coronavirus, follow, stayhome, bts, bbb, highriskcovid, day, mondaythoughts, survival