!sudo apt-get install ipython3-notebook python3-pip

!pip3 install git+https://github.com/scikit-learn/scikit-learn.git
!pip3 install git+https://github.com/nltk/nltk/#

import json
import pandas as pd
from wikiclass.models import RFTextModel

!wget https://github.com/halfak/Wiki-Class/blob/master/models/enwiki.rf_text.model?raw=true

!mv enwiki.rf_text.model\?raw\=true enwiki.rf_text.model

model = RFTextModel.from_file(open("enwiki.rf_text.model",'rb'))

classed_items = json.load(open('test_class_data.json','r'))
print(sum([len(l) for l in classed_items.values()]))

standards = {actual: text for actual, text in classed_items.items() if actual in ['Stub', 'Start', 'C', 'B', 'GA', 'FA'] }

print(sum([len(l) for l in standards.values()]))

accuracy_df = pd.DataFrame(index=classed_items.keys(), columns=['actual','correct', 'model_prob', 'actual_prob'])
for actual, text_list in standards.items():
    #see if actual is even here, otherwise no fair comparison
        for text in text_list:
            try:
                assessment, probabilities = model.classify(text)
            except ZeroDivisionError:
                continue
                #print(actual, text)
            accuracy_df = accuracy_df.append({'actual': actual,
                                              'correct':int(assessment == actual),
                                              'model_prob': probabilities[assessment],
                                              'actual_prob': probabilities[actual]}, ignore_index=True)

df  = accuracy_df.dropna(how='all')
df.head()

groups = df.groupby(by='actual')
groups['correct'].mean()

classic_order = ['Stub', 'Start', 'C', 'B', 'GA', 'FA']
enum_classic = enumerate(classic_order)

for enum, classic in dict(enum_classic).items():
    print(enum, classic)

classic_order = ['Stub', 'Start', 'C', 'B', 'GA', 'FA']
classic_dict = dict(zip(classic_order, range(len(classic_order))))

off_by_df = pd.DataFrame(index=classed_items.keys(), columns=['actual','off_by'])

for classic in classic_order:
    for text in standards[classic]:
            try:
                assessment, probabilities = model.classify(text)
            except ZeroDivisionError:
                continue
                #print(actual, text)
            off_by_df = off_by_df.append({'actual': classic,
                                              'off_by':abs(classic_dict[assessment] - classic_dict[classic])}, ignore_index=True)

off_by  = off_by_df.dropna(how='all')
off_by.head()

%pylab inline

ax = off_by.groupby(by='actual',sort=False).mean().plot(title='Prediction Closeness by Quality Class', kind='bar', legend=False)
                                                   
ax.set_ylabel('''Prediction Closeness (lower is more accurate)''')
ax.set_xlabel('''Quality Class''')

from wikiclass.models import RFTextModel
from wikiclass import assessments

import random

train_set = list()
test_set = list()
for actual, text_list in standards.items():
    for text in text_list:
        if random.randint(0,9) >= 8:
            test_set.append( (text, actual) )
        else:
            train_set.append( (text, actual) )

print(len(test_set)/len(train_set))

# Train a model
model = RFTextModel.train(
    train_set,
    assessments=assessments.WP10
)

# Run the test set & print the results
results = model.test(test_set)
print(results)

# Write the model to disk for reuse.
model.to_file(open("36K_random_enwiki.rf_text.model", "wb"))

norm_results = results.apply(lambda col: col / col.sum(), axis=1)
norm_results

for c in classic_order:
    print(c, norm_results.loc[c][c])