import pandas as pd
from gensim import corpora, models
from nltk.corpus import stopwords
import nltk
import snowballstemmer
import re
from operator import itemgetter
df = pd.read_csv('../data/US-news.csv',usecols=['dt','title','newstxt'],parse_dates=[0])
documents = df.newstxt.tolist()
stemmer = snowballstemmer.EnglishStemmer()
stoplist = stopwords.words('english')
stoplist = stemmer.stemWords(stoplist)
stoplist = set(stoplist)
texts=[]
for doc in documents:
doc = re.sub('[!"#%\'()*+,-./:;<=>?@\[\]^_`{|}~1234567890’”“′‘\\\]',' ', doc).split()
doc = stemmer.stemWords(doc)
text= [w.strip() for w in doc if len(w.strip())>=3 and w.strip() not in stoplist]
texts.append(text)
dictionary = corpora.Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]
tfidf = models.TfidfModel(corpus)
corpus_tfidf = tfidf[corpus]
n_topics = 100
lda = models.LdaModel(corpus_tfidf, id2word=dictionary, num_topics=n_topics, passes=3, alpha='auto')
for i in range(0, n_topics):
terms = [term[1] for term in lda.show_topic(i, 10)]
print ("#" + str(i) + ": "+ ", ".join(terms))
#0: preced, Index, Poor, tag, Jew, advic, Standard, vehicl, electr, batteri #1: Portland, Dow, Stock, Exchang, Market, trade, taper, index, stock, Japanes #2: enrol, milit, Saudi, trust, Human, tortur, premium, drone, Qaeda, Arabia #3: Ranger, Yanke, Martinez, Croatia, Derek, Jeter, symptom, mountain, Canadien, Oprah #4: Swift, Wisconsin, Denmark, Taylor, Milwauke, Walker, Shah, Shake, Real, statewid #5: Egypt, Egyptian, Bieber, Carney, Morsi, Cairo, Sisi, Moham, coalit, Islamist #6: Uber, Sgt, kidnap, overcom, WikiLeak, ride, taxi, excel, interpret, Space #7: Summer, Udall, Gardner, hefti, swirl, Buck, Mag, Joss, wunderkind, Colorado #8: Colombia, Djokov, AMC, Tyrion, transgend, Novak, Dead, drama, Jarrett, paint #9: Romney, Bush, separatist, Jeb, Warrior, gun, Mitt, POLITICO, Adelson, Sandi #10: Beat, refuge, Solang, rapper, Amanda, eBay, Macklemor, tent, publicist, merit #11: Hous, percent, Obama, Republican, Senat, vote, Democrat, govern, would, said #12: Cuomo, outbreak, Damascus, Anaheim, Walmart, businessman, nerv, watchdog, princip, Dylan #13: Bush, hospit, Georg, list, daughter, diseas, Africa, alway, semifin, Cheney #14: Clinton, Hillari, Amazon, write, Angele, Los, Palin, ABC, may, climat #15: Spitzer, Memphi, Grizzli, Laker, Coach, Soldier, NRA, regret, Statist, McAuliff #16: Knick, Maverick, Cemeteri, scream, Whole, Andre, handgun, royalti, fragment, trilog #17: NBA, Spur, Heat, Pacer, Antonio, Miami, Thunder, Confer, Indiana, Final #18: Turkey, Turkish, Cuba, Erdogan, wed, Tayyip, Recep, Istanbul, Anna, Cuban #19: Kansa, airstrik, NHL, Kurdish, jihadist, Round, Eagl, milit, mom, Islamic #20: Clooney, Dustin, defect, Morri, refineri, POW, Moran, Silva, Ortiz, limo #21: Oklahoma, Harvard, Playoff, Cincinnati, par, mental, Wild, Palestinian, Prevent, Palestin #22: Award, Emmi, award, terribl, actor, Academi, nomin, Nokia, Oscar, comedi #23: polic, protest, juri, Ford, Zimmerman, King, march, Polic, violenc, murder #24: Morgan, marathon, Marathon, hadn, Citizen, timelin, beef, Vine, preview, Gazprom #25: Window, AFPHous, Deni, cat, Centr, Turtl, McDonough, Cat, GIFs, Rutger #26: Jackson, Mount, keynot, DNC, dad, Kenya, Libyan, lockdown, Tripoli, diagnosi #27: Soni, Interview, Joan, Alabama, Pictur, Burn, Shaheen, Augusta, Interior, Hampshir #28: NATO, shame, wealth, Barra, mandatori, pollut, Societi, Maduro, embark, Nicola #29: Abramson, Walter, Jill, outag, Baquet, Editor, cocktail, scientif, newsroom, shootout #30: plea, breast, breakfast, transcript, Taco, shoe, syndic, Columbus, secret, Read #31: ECB, Ireland, Perez, deposit, bank, Cain, Bank, Margaret, Geithner, Northern #32: Rice, Raven, leagu, Goodel, NFL, elev, overturn, Ray, breach, Command #33: plane, Scotland, Graham, District, Airlin, Constitut, pilot, Malaysian, brain, Oregon #34: Hong, Kong, golf, Throne, RNC, Mich, gate, withdrew, Eight, Carey #35: Hagel, Chuck, Qaida, desert, Libya, Jazeera, downgrad, Khattala, Thousand, aficionado #36: Arthur, anybodi, Grant, blank, Better, sampl, Fusion, albeit, Antoni, Shanghai #37: Seri, Leagu, inning, Wood, seventh, Tiger, basebal, plant, pitch, Sox #38: McDaniel, Cochran, Uruguay, Spain, Suarez, Itali, Gruber, Lui, Madrid, Thad #39: Seattl, Correspond, moon, Goldman, sun, Marriag, eclips, Seahawk, Ramadan, Halloween #40: Snowden, River, Indianapoli, Jesus, oversea, Luca, Abram, Alexand, Protect, uniti #41: Taliban, club, Playoff, Beyonc, Nigerian, Scalis, King, behead, fjs, Drew #42: Franco, Late, Letterman, Meyer, Bruce, Fallon, Kent, Seth, Rogen, handout #43: verdict, Blasio, Liu, Angel, southeastern, Stand, cruis, depth, Malley, Ground #44: Ebola, Franci, Pope, virus, pope, Vatican, tornado, church, Sam, quarantin #45: footag, Code, Price, truck, crude, Art, oil, bomber, band, Ansel #46: Cosbi, Stone, Roll, Ecuador, land, Miss, Jerri, Holm, shell, Chipotl #47: Palestinian, tear, Abbas, parad, Vaniti, Charlott, shadow, Barclay, teacher, Kimmel #48: Hachett, Bull, Oliver, Mozilla, Newsweek, Nina, comet, Eich, Reddit, conspir #49: insur, Belgium, Affordabl, Care, Obamacar, Mandela, repeal, hors, health, healthcar #50: Princ, babi, royal, Royal, Baker, Cambridg, Ghana, ferri, midfield, Duke #51: Cup, World, match, Brazil, Argentina, FIFA, Nigeria, Earnest, Germani, goal #52: Kim, Korean, Pride, Jong, memoir, Korea, Lincoln, checkpoint, North, recognit #53: Plazz, random, commentari, tweet, famous, find, pool, funni, clever, But #54: Silver, Luther, Lopez, arena, Hood, FiveThirtyEight, casualti, Fame, slain, Trey #55: weird, formula, infring, backpack, plagiar, clue, Nicol, liquid, layer, Watt #56: probe, planet, Trade, Gordon, NASA, discoveri, solar, Metropolitan, Krugman, space #57: New, York, state, game, Sunday, show, first, year, Saturday, Citi #58: ISIL, Microsoft, Levant, ISIS, Armstrong, Ballmer, softwar, Delawar, cloud, Apple #59: Six, Tallahasse, Palm, Marlin, incomplet, Westwood, lotteri, forcibl, jackpot, transmit #60: Johnson, Perri, Met, Navi, Easter, Rick, Darren, Patrol, Station, Salvador #61: Blackhawk, Stanley, Grime, Draghi, King, Bruin, Lion, Thompson, puck, Alison #62: Iraq, Iraqi, Baghdad, milit, Maliki, ISIS, Rodriguez, Mosul, Islamic, fighter #63: Nixon, Peterson, Harrison, Walsh, Juan, Richardson, gossip, Adrian, Andi, Pablo #64: Colbert, Foley, German, Portug, Merkel, Dempsey, Shiit, Sunni, partner, Comedi #65: Batman, Affleck, Steel, Superman, Emmanuel, earthquak, CPAC, straw, Dunand, quak #66: Clipper, snow, storm, Weather, rain, Connecticut, temperatur, contracept, Brook, Georgia #67: impeach, quell, Gaga, Knowl, HuffPost, heaven, nicknam, downward, gradual, rehears #68: Shinseki, hole, McDonnel, Mart, Liberia, Marc, Death, Penguin, Wal, Logan #69: Choic, amnesti, audit, mock, Live, Key, Allison, comedi, SNL, yell #70: marriag, sex, coupl, gay, marri, Rica, equal, licens, Greec, lesbian #71: IMF, Hard, Monetari, Chavez, Fund, Farm, Oregonian, Michigan, Eid, thwart #72: Gaza, Hama, Israeli, Israel, Netherland, truce, rocket, Strip, milit, Semifin #73: Afghan, Afghanistan, Iowa, Braley, sequel, Kabul, Ernst, Karzai, Joni, Hamid #74: Young, Good, Sebelius, Plus, embargo, forum, Kathleen, ration, digniti, Vote #75: Holi, Highway, Irish, Vatican, cardin, Huff, Joshua, transact, palac, Portman #76: Sterl, Christi, Cantor, primari, Jersey, ballot, Gov, governor, Brat, donor #77: Davi, abort, touchdown, yard, quarterback, Winston, Bronco, Denver, Wendi, AFC #78: Pelosi, IRS, Lerner, Leader, Tea, French, Reid, Minor, amend, Issa #79: Jewish, Seinfeld, Chrome, AFPIt, Movi, chef, Traci, ambul, Semit, browser #80: Donetsk, Poroshenko, rebel, Disney, Petro, soccer, stanc, Seven, Willi, Luhansk #81: Dutch, inaugur, Cowboy, Quinn, rubl, Liber, Jennif, Prior, Bibl, Eve #82: Chelsea, Rodger, immigr, Wire, Summit, Hispan, class, Rand, middl, citizenship #83: Camp, Dunham, Guantanamo, coffe, slogan, Labour, buyer, flock, Ledger, Lena #84: Lynch, Commission, Bratton, Gawker, Officer, loot, reboot, swap, signup, Platt #85: Dame, Lawn, Charger, fish, Emma, closest, hottest, Glass, turnout, curtain #86: recal, Cyrus, Miley, sandwich, MTV, Feed, Petersen, Stop, heir, Video #87: Grimm, AT&T, Cotton, Arlington, Congressman, humanitarian, barrel, OPEC, lime, Cancer #88: Yellen, Fed, Reserv, Feder, Bernank, Janet, Boy, stadium, funer, loan #89: fenc, Appeal, client, bailout, Share, Circuit, exit, Nike, Wife, Mission #90: Kerri, Secretari, Israel, Iran, debt, student, Israeli, sanction, Minist, nuclear #91: Ukrain, Russian, Ukrainian, Kiev, Russia, eastern, pro, Korea, Crimea, Moscow #92: Apple, iPhon, Abdullah, Cook, Phoenix, Dre, iPad, Van, boast, Mac #93: Algeria, patent, Instagram, Sheldon, Paltrow, Android, Parson, Bang, Gillibrand, buyout #94: Syria, Syrian, Assad, chemic, weapon, airport, war, militari, Bashar, Russian #95: Control, massacr, Draper, startup, Fisher, Jare, Hernandez, Wickerham, sneak, gray #96: Pistorius, unrest, Diego, strict, riot, cigarett, Occupi, Filner, McMillan, cellphon #97: Rio, Sanford, runoff, Warner, Alibaba, LGBT, monetari, Janeiro, Scout, subscrib #98: Weiner, Christma, Kardashian, AIDS, Grammi, Anthoni, Nazi, Stapl, holiday, tree #99: Cup, match, seed, coach, Mexico, playoff, round, tournament, NCAA, defeat
lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=n_topics, onepass=False, power_iters=3)
for i in range(0, n_topics):
terms = [term[1] for term in lsi.show_topic(i, 10)]
print ("#" + str(i) + ": "+ ", ".join(terms))
#0: Obama, Hous, Republican, Senat, Presid, govern, said, New, vote, percent #1: random, Plazz, tweet, commentari, famous, find, Below, curat, summar, But #2: Republican, Senat, Hous, bill, vote, Democrat, Boehner, polic, Obama, immigr #3: Ukrain, Russian, Russia, Putin, Ukrainian, militari, Obama, Syria, Presid, game #4: polic, percent, Ferguson, protest, Brown, rate, Feder, Reserv, offic, shoot #5: game, polic, percent, Ferguson, win, team, protest, Cup, Ukrain, Russian #6: Ukrain, Russian, polic, Israel, protest, Israeli, Obama, Ukrainian, Russia, Syria #7: Snowden, Israeli, Israel, NSA, Gaza, Palestinian, Hama, Edward, percent, surveil #8: Clinton, Hous, candid, elect, Christi, Hillari, percent, poll, presidenti, Democrat #9: marriag, Obama, Israeli, Israel, Gaza, Clinton, sex, Palestinian, gay, bill #10: Snowden, NSA, insur, marriag, health, Edward, Senat, Obama, surveil, Ebola #11: marriag, sex, Syria, gay, Syrian, Court, Yellen, Israeli, Reserv, rule #12: Apple, insur, health, Israel, Obama, Israeli, Clinton, Syrian, Syria, Gaza #13: Ebola, Clinton, percent, health, immigr, poll, insur, marriag, Snowden, snow #14: snow, Ebola, storm, Christi, insur, Apple, percent, Yellen, compani, Jersey #15: immigr, percent, Obama, Yellen, Senat, snow, Reserv, Feder, Fed, health #16: Christi, Apple, Jersey, immigr, compani, iPhon, Day, film, Chris, holiday #17: Clinton, Hillari, Christi, snow, Ebola, Jersey, storm, poll, percent, Obama #18: Ebola, snow, storm, virus, insur, immigr, Christi, patient, worker, Africa #19: Cup, World, Christi, Morsi, match, immigr, Brazil, Egypt, Egyptian, Germani #20: immigr, Iran, nuclear, Christi, Cup, World, Jersey, Morsi, Rouhani, Iraq #21: immigr, debt, Morsi, govern, shutdown, Egypt, Egyptian, Christi, Yellen, snow #22: North, Korea, Korean, Morsi, Kim, Christi, nuclear, Syria, wage, Syrian #23: North, Korea, Korean, Apple, Morsi, immigr, Ford, Kim, mayor, Obama #24: immigr, Christi, Hous, bill, Clinton, Blasio, Apple, White, mayor, Korea #25: Zimmerman, Iraq, Blasio, immigr, mayor, Martin, Ford, protest, York, Morsi #26: Zimmerman, Iraq, Martin, gun, Iraqi, chemic, Trayvon, Morsi, weapon, ISIL #27: Heat, game, NBA, Spur, NFL, Pacer, Rice, Miami, Sox, Game #28: Iran, nuclear, Zimmerman, Syrian, Iraq, chemic, Rouhani, Syria, Boehner, Ford #29: wage, minimum, Obama, Iran, immigr, percent, Boehner, worker, Zimmerman, Kerri #30: Apple, Twitter, user, Franci, Pope, iPhon, Bush, Morsi, Facebook, bill #31: Apple, debt, Zimmerman, Snowden, Senat, North, wage, Korea, Blasio, iPhon #32: wage, minimum, Boehner, Zimmerman, Obama, worker, bill, Senat, Christi, percent #33: Franci, Pope, Apple, Vatican, holiday, pope, Clinton, Day, Bush, Christma #34: Boston, Franci, Pope, Sox, Sterl, Ford, Iran, Rice, inning, Vatican #35: Bush, Apple, Romney, Zimmerman, Olympic, Clinton, vote, Ford, wage, Sochi #36: Boston, Apple, York, Bush, Sox, Clinton, compani, billion, vote, poll #37: holiday, Olympic, Franci, Pope, Sochi, Christma, Day, Vatican, Ford, pope #38: Olympic, holiday, Sochi, Bush, Pope, Franci, Christma, Vatican, Day, medal #39: Ford, Toronto, Olympic, gun, York, Rob, job, crack, video, Blasio #40: Sterl, Clipper, protest, debt, White, Boehner, gun, Brown, flight, holiday #41: Sterl, protest, Clipper, Turkey, Turkish, Ford, Pope, Franci, King, Angele #42: Ford, job, Toronto, Blasio, user, York, unemploy, bank, Twitter, billion #43: bill, plane, flight, McConnel, Reid, Cruz, gun, vote, Malaysian, Airlin #44: Bush, bill, gun, Sterl, plane, flight, Malaysian, Airlin, Boehner, Romney #45: gun, Snowden, Hous, marriag, shutdown, compani, Apple, White, price, juri #46: King, film, Ranger, Blackhawk, Sox, Soni, goal, Stanley, Boston, Heat #47: Texa, film, King, Rice, abort, Davi, Bush, Cruz, Perri, Reid #48: Cruz, Texa, Bush, job, King, Sterl, wage, White, unemploy, gun #49: Rice, video, Goodel, Raven, Sterl, Ray, King, Kerri, elev, NFL #50: Scotland, job, King, Cameron, gun, film, British, Romney, protest, Bush #51: Scotland, King, user, bill, gun, Cameron, debt, immigr, Sterl, Award #52: Boehner, Kerri, Reid, Court, marriag, budget, Suprem, Texa, Zimmerman, immigr #53: percent, Boehner, job, elect, Kerri, John, unemploy, White, Romney, CIA #54: Bush, Kerri, Paul, Romney, debt, budget, Texa, World, San, abort #55: Paul, Wimbledon, Djokov, film, bank, William, Putin, Open, Cruz, Kerri #56: gun, Mandela, Bergdahl, Taliban, film, Afghanistan, Snowden, attack, Cuba, Afghan #57: Kerri, gun, Scotland, Cruz, Cameron, Iran, bill, Hagel, Romney, Secretari #58: Romney, Bush, Paul, Mitt, Texa, gun, film, Davi, Boston, Court #59: Putin, film, Blasio, user, Yellen, compani, Kerri, bank, Romney, Cosbi #60: student, film, Cosbi, gun, Kerri, Bush, school, percent, sexual, job #61: Boston, Bergdahl, gun, Taliban, bomb, Marathon, Afghanistan, San, Paul, prison #62: Mandela, Turkey, Weiner, Turkish, climat, Boston, Scotland, Erdogan, Googl, pipelin #63: Romney, Paul, Kerri, Bush, Hagel, Mitt, Reid, vote, Cosbi, Bergdahl #64: gun, Boehner, Weiner, Cosbi, Suprem, Googl, bank, NSA, Bush, IRS #65: debt, Romney, Kerri, Googl, Cosbi, Court, Suprem, episod, Colbert, Mandela #66: Mandela, Paul, debt, South, Weiner, Nelson, McConnel, shutdown, student, White #67: Weiner, debt, Holder, Bergdahl, Anthoni, McConnel, Brown, Taliban, mayor, Paul #68: Cosbi, San, climat, Weiner, Obama, sexual, Scotland, attack, chang, Francisco #69: San, Francisco, William, Texa, student, Cosbi, Giant, season, Reid, Putin #70: Weiner, McConnel, Cruz, Kentucki, Christma, Paul, climat, Grime, Hagel, Mandela #71: Weiner, Mandela, Boston, Kerri, Amazon, South, bomb, Day, Christma, Nelson #72: Cosbi, climat, shutdown, McConnel, debt, Mandela, chang, Kentucki, William, assault #73: bank, Turkey, budget, percent, Turkish, Draghi, protest, Christma, European, Davi #74: Bieber, Cosbi, Brown, attack, debt, Holder, Weiner, shutdown, Blasio, climat #75: Putin, Cosbi, McConnel, rate, Draghi, debt, bank, Abramson, Snowden, Blasio #76: Putin, Googl, Weiner, Cruz, Brown, Donetsk, climat, Hagel, Twitter, Vladimir #77: Googl, Boston, Cosbi, student, Bergdahl, IRS, CIA, Holder, Paul, school #78: vote, Weiner, San, shutdown, Holder, Cruz, Davi, Turkey, abort, Turkish #79: Cuba, McConnel, Mandela, Rubio, Castro, marijuana, William, Cuban, Reid, Cantor #80: student, Cosbi, budget, Hagel, attack, bill, Cheney, Romney, cut, debt #81: King, Jame, bank, Cantor, Rodriguez, seri, Yanke, Colbert, Koch, Miami #82: William, Christma, Colbert, bill, Princ, CIA, Virginia, Day, McDonnel, Cosbi #83: Palin, William, Paul, price, Amazon, Princ, elect, vote, Facebook, Sarah #84: Cuba, Mandela, Christma, Carney, Cheney, CIA, Googl, student, Thanksgiv, White #85: Colbert, Cheney, host, student, Fallon, Soni, Mandela, Yellen, York, Best #86: Googl, video, Cantor, Colbert, Iowa, episod, juri, Braley, Afghanistan, Princ #87: Bergdahl, Putin, militari, attack, Paul, Bieber, Cantor, Boston, Snowden, protest #88: Palin, Romney, Cantor, Amazon, King, Iowa, juri, Sarah, McDaniel, Braley #89: Amazon, Crimea, Russia, Iowa, marijuana, Donetsk, Pacer, Braley, Hagel, Ernst #90: Colbert, Hagel, Amazon, Israel, Bergdahl, Blasio, shutdown, Netanyahu, Afghanistan, student #91: Facebook, Scotland, Cheney, Uber, Palin, Cosbi, Colbert, Warren, Sen, quarter #92: Amazon, Christma, marijuana, Blasio, Googl, Bergdahl, William, retail, Thanksgiv, store #93: Googl, marijuana, Bergdahl, Palin, Garner, juri, Brown, Cuba, Rouhani, state #94: Palin, Brown, Googl, Cruz, compani, Garner, Paul, Cantor, Cuomo, juri #95: Christma, King, Day, quarter, Cantor, oil, Bowl, Yellen, Giant, Palin #96: Brown, marijuana, Palin, Hagel, Cantor, Colbert, Amazon, Blasio, York, Day #97: Cuba, marijuana, Amazon, Colbert, pipelin, Keyston, Hagel, King, Boehner, Landrieu #98: marijuana, Yellen, protest, Hagel, Republican, Putin, Paul, Facebook, Weiner, million #99: Palin, Googl, Russia, Putin, Crimea, marijuana, Servic, women, Thanksgiv, Yellen
i = 0
print ('Topic distribution on an arbitrarily selected document:\n')
print ('Original document: ' + documents[i])
print ('Preprocessed document: ' + str(texts[i]))
Topic distribution on an arbitrarily selected document: Original document: Pope Francis (in white) visits Hagia Sophia Mosque (Santa Sofia) on Saturday in Istanbul. The pope held talks with Turkish President Recep Tayyip Erdogan in Ankara on Friday, calling for dialogue between faiths to end the Islamist extremism plaguing the Middle East. AFP PHOTO / FILIPPO MONTEFORTEPope Francis ended a three-day visit to Turkey aimed at bridging the divide between the Catholic and Orthodox churches while also bringing greater understanding between Muslims and Christians amid the growing threat of the Islamic State on Sunday.Francis spent his first day in Turkey meeting with Turkish President Recep Tayyip Erdogan and other leaders Friday in the nation’s capitol city Ankara, CNN reports. The pope called for people of different religions to unite with respect for all human life and religious freedom in the face of growing religious persecution by terrorist groups such as the Islamic State.“Fanaticism and fundamentalism, as well as irrational fears which foster misunderstanding and discrimination, need to be countered by the solidarity of all believers,” Pope Francis said to Turkish leaders on Friday.During his trip the Pope also traveled to Istanbul, where he participated in a Muslim prayer Saturday with the grand mufti of Istanbul, Rahmi Yaran, in the city’s historic Sultan Ahmet Mosque.Later that day, the pope met with Ecumenical Patriarch Bartholomew for a joint prayer service, where, in what many have termed “a remarkable display of papal deference,” Francis bowed to Bartholomew I and asked for his blessing, Al Jazeera reports. The Orthodox church split from the Catholic church in year 1054 over differences that included the papacy and patriarchs’ subservient role.Pope Francis’ visit this weekend marked the fourth time a pope has visited Turkey, the BBC reports. Preprocessed document: ['Pope', 'Franci', 'white', 'visit', 'Hagia', 'Sophia', 'Mosqu', 'Santa', 'Sofia', 'Saturday', 'Istanbul', 'The', 'pope', 'held', 'talk', 'Turkish', 'Presid', 'Recep', 'Tayyip', 'Erdogan', 'Ankara', 'Friday', 'call', 'dialogu', 'faith', 'end', 'Islamist', 'extrem', 'plagu', 'Middl', 'East', 'AFP', 'PHOTO', 'FILIPPO', 'MONTEFORTEPope', 'Franci', 'end', 'three', 'day', 'visit', 'Turkey', 'aim', 'bridg', 'divid', 'Cathol', 'Orthodox', 'church', 'also', 'bring', 'greater', 'understand', 'Muslim', 'Christian', 'amid', 'grow', 'threat', 'Islamic', 'State', 'Sunday', 'Franci', 'spent', 'first', 'day', 'Turkey', 'meet', 'Turkish', 'Presid', 'Recep', 'Tayyip', 'Erdogan', 'leader', 'Friday', 'nation', 'capitol', 'citi', 'Ankara', 'CNN', 'report', 'The', 'pope', 'call', 'peopl', 'differ', 'religion', 'unit', 'respect', 'human', 'life', 'religi', 'freedom', 'face', 'grow', 'religi', 'persecut', 'terrorist', 'group', 'Islamic', 'State', 'Fanatic', 'fundament', 'well', 'irrat', 'fear', 'foster', 'misunderstand', 'discrimin', 'need', 'counter', 'solidar', 'believ', 'Pope', 'Franci', 'said', 'Turkish', 'leader', 'Friday', 'Dure', 'trip', 'Pope', 'also', 'travel', 'Istanbul', 'particip', 'Muslim', 'prayer', 'Saturday', 'grand', 'mufti', 'Istanbul', 'Rahmi', 'Yaran', 'citi', 'histor', 'Sultan', 'Ahmet', 'Mosqu', 'Later', 'day', 'pope', 'met', 'Ecumen', 'Patriarch', 'Bartholomew', 'joint', 'prayer', 'servic', 'mani', 'term', 'remark', 'display', 'papal', 'defer', 'Franci', 'bow', 'Bartholomew', 'ask', 'bless', 'Jazeera', 'report', 'The', 'Orthodox', 'church', 'split', 'Cathol', 'church', 'year', 'differ', 'includ', 'papaci', 'patriarch', 'subservi', 'role', 'Pope', 'Franci', 'visit', 'weekend', 'mark', 'fourth', 'time', 'pope', 'visit', 'Turkey', 'BBC', 'report']
print ('Which LDA topic maximally describes this document?')
print ('Topic probability mixture: ',end="")
for j,k in lda[corpus[i]]:
print('(%d, %3.2f)' % (j,k), end = ' ')
print ('\nTopic with the highest probability: topic #' + str(max(lda[corpus[i]],key=itemgetter(1))[0]))
Which LDA topic maximally describes this document? Topic probability mixture: (11, 0.17) (14, 0.03) (18, 0.08) (23, 0.02) (44, 0.27) (57, 0.32) (90, 0.04) Topic with the highest probability: topic #57
print ('Which LSI topic maximally describes a document?')
print ('Topic probability mixture: ',end="")
for j,k in lsi[corpus[i]]:
print('(%d, %3.2f)' % (j,k), end = ' ')
print ('\nTopic with the highest probability: topic #' + str(max(lsi[corpus[i]],key=itemgetter(1))[0]))
Which LSI topic maximally describes a document? Topic probability mixture: (0, 3.73) (1, 0.50) (2, 1.55) (3, -1.02) (4, 0.52) (5, 0.07) (6, 0.95) (7, 0.66) (8, -0.06) (9, -0.09) (10, -0.62) (11, 0.50) (12, -1.05) (13, -0.20) (14, 0.59) (15, -0.30) (16, 1.95) (17, -0.24) (18, 0.23) (19, 1.67) (20, -0.12) (21, -0.55) (22, 0.17) (23, -1.22) (24, -0.16) (25, -0.50) (26, -1.52) (27, -0.94) (28, 1.86) (29, 0.15) (30, 3.32) (31, -1.24) (32, -0.21) (33, -4.26) (34, -2.82) (35, 0.00) (36, 0.46) (37, -2.87) (38, 3.02) (39, 0.20) (40, -1.90) (41, 0.08) (42, -1.04) (43, 0.75) (44, -0.84) (45, -0.75) (46, 0.20) (47, -0.67) (48, -0.25) (49, 0.20) (50, -1.37) (51, -0.60) (52, 0.52) (53, -0.84) (54, -0.64) (55, 0.20) (56, -0.04) (57, -0.01) (58, 0.27) (59, 1.17) (60, 0.55) (61, -0.26) (62, -1.98) (63, -0.38) (64, 0.36) (65, 0.43) (66, 0.96) (67, -0.58) (68, -0.32) (69, -0.76) (70, 0.37) (71, 0.76) (72, 0.17) (73, 1.71) (74, -0.56) (75, 1.43) (76, 0.14) (77, 0.09) (78, 1.05) (79, 0.23) (80, 0.32) (81, 0.26) (82, 0.78) (83, -0.85) (84, -0.47) (85, -0.50) (86, -0.24) (87, 0.09) (88, -0.05) (89, -1.03) (90, 0.11) (91, 0.47) (92, 0.08) (93, 0.62) (94, -0.13) (95, 0.34) (96, 0.27) (97, 0.10) (98, -0.05) (99, 0.86) Topic with the highest probability: topic #0