import pandas as pd
data = pd.read_csv("train.csv")
len(data)
data.columns
data.Insult.value_counts()
from sklearn.cross_validation import train_test_split
import numpy as np
y_train = np.array(data.Insult)
y_train
text_train = data.Comment.tolist()
data_test = pd.read_csv("test_with_solutions.csv")
data_test
text_test, y_test = data_test.Comment.tolist(), np.array(data_test.Insult)
from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer()
cv.fit(text_train)
len(cv.vocabulary_)
cv.vocabulary_
X_train = cv.transform(text_train)
X_train.shape
text_train[0]
X_train[0, :].nonzero()
X_train[0]
X_test = cv.transform(text_test)
from sklearn.svm import LinearSVC
svm = LinearSVC(C=.01)
svm.fit(X_train, y_train)
svm.score(X_train, y_train)
svm.score(X_test, y_test)
y_test_pred = svm.predict(X_test)
from sklearn.metrics import classification_report
print(classification_report(y_test, y_test_pred))
coef = svm.coef_.ravel()
positive_coefficients = np.argsort(coef)[-25:]
negative_coefficients = np.argsort(coef)[:25]
interesting_coefficients = np.hstack([negative_coefficients, positive_coefficients])
plt.figure(figsize=(15, 5))
plt.bar(np.arange(50), coef[interesting_coefficients], color=["red" if c < 0 else "blue" for c in coef[interesting_coefficients]])
feature_names = np.array(cv.get_feature_names())
plt.xticks(np.arange(1, 51), feature_names[interesting_coefficients], rotation=60, ha="right");
from sklearn.pipeline import Pipeline
pipeline = Pipeline([('vectorizer', cv), ('classifier', svm)])
pipeline.fit(text_train, y_train)
pipeline.score(text_train, y_train)
pipeline.score(text_test, y_test)
from sklearn.grid_search import GridSearchCV
param_grid = {'classifier__C': 10. ** np.arange(-3, 3)}
grid_search = GridSearchCV(pipeline, param_grid=param_grid, verbose=10)
grid_search.fit(text_train, y_train)
grid_search.best_score_
grid_search.best_params_
param_grid = {'classifier__C': 10. ** np.arange(-3, 3), "vectorizer__ngram_range": [(1, 1), (1, 2), (1, 3), (2, 3), (2, 2)]}
grid_search = GridSearchCV(pipeline, param_grid=param_grid, n_jobs=3)
grid_search.fit(text_train, y_train)
grid_search.best_params_
grid_search.best_score_