In [1]:
import import_ipynb
In [4]:
import argparse
import numpy as np
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import itertools
import warnings
from utils.treebank import StanfordSentiment
import utils.glove as glove

from q3_sgd import load_saved_params, sgd
# We will use sklearn here because it will run faster than implementing
# ourselves. However, for other parts of this assignment you must implement
# the functions yourself!

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
warnings.filterwarnings('ignore')
In [11]:
def getSentenceFeatures(tokens, wordVectors, sentence):
    """
    Obtain the sentence feature for sentiment analysis by averaging its
    word vectors
    """

    # Implement computation for the sentence features given a sentence.

    # Inputs:
    # tokens -- a dictionary that maps words to their indices in
    #           the word vector list
    # wordVectors -- word vectors (each row) for all tokens
    # sentence -- a list of words in the sentence of interest

    # Output:
    # - sentVector: feature vector for the sentence

    sentVector = np.zeros((wordVectors.shape[1],))

    ### YOUR CODE HERE
    ### Note:此处不能用向量运算,因为存在重复的word

    sentence_index = [tokens[i] for i in sentence]
    for index in sentence_index:
        sentVector += wordVectors[index, :]

    sentVector /= len(sentence)
    ### END YOUR CODE

    assert sentVector.shape == (wordVectors.shape[1],)
    return sentVector


def getRegularizationValues():
    """Try different regularizations

    Return a sorted list of values to try.
    """
    values = None   # Assign a list of floats in the block below
    ### YOUR CODE HERE
    values = np.logspace(-4, 2, num=100, base=10)
    ### END YOUR CODE
    return sorted(values)


def chooseBestModel(results):
    """Choose the best model based on dev set performance.

    Arguments:
    results -- A list of python dictionaries of the following format:
        {
            "reg": regularization,
            "clf": classifier,
            "train": trainAccuracy,
            "dev": devAccuracy,
            "test": testAccuracy
        }

    Each dictionary represents the performance of one model.

    Returns:
    Your chosen result dictionary.
    """
    bestResult = None

    ### YOUR CODE HERE
    bestResult = max(results, key= lambda x: x['dev'])
    ### END YOUR CODE

    return bestResult


def accuracy(y, yhat):
    """ Precision for classifier """
    assert(y.shape == yhat.shape)
    return np.sum(y == yhat) * 100.0 / y.size


def plotRegVsAccuracy(regValues, results, filename):
    """ Make a plot of regularization vs accuracy """
    plt.plot(regValues, [x["train"] for x in results])
    plt.plot(regValues, [x["dev"] for x in results])
    plt.xscale('log')
    plt.xlabel("regularization")
    plt.ylabel("accuracy")
    plt.legend(['train', 'dev'], loc='upper left')
    plt.savefig(filename)


def outputConfusionMatrix(features, labels, clf, filename):
    """ Generate a confusion matrix """
    pred = clf.predict(features)
    cm = confusion_matrix(labels, pred, labels=range(5))
    plt.figure()
    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Reds)
    plt.colorbar()
    classes = ["- -", "-", "neut", "+", "+ +"]
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes)
    plt.yticks(tick_marks, classes)
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.savefig(filename)


def outputPredictions(dataset, features, labels, clf, filename):
    """ Write the predictions to file """
    pred = clf.predict(features)
    with open(filename, "w") as f:
        print("True\tPredicted\tText", file=f)
        for i in range(len(dataset)):
            print("%d\t%d\t%s" % (
                            labels[i], pred[i], " ".join(dataset[i][0])), file=f)


def main(args):
    """ Train a model to do sentiment analyis"""
    warnings.filterwarnings('ignore')
    # Load the dataset
    dataset = StanfordSentiment()
    tokens = dataset.tokens()
    nWords = len(tokens)
    if args == 'yourvectors':
        _, wordVectors, _ = load_saved_params()
        wordVectors = np.concatenate(
            (wordVectors[:nWords,:], wordVectors[nWords:,:]),
            axis=1)
    elif args == 'pretrained':
        wordVectors = glove.loadWordVectors(tokens)
    dimVectors = wordVectors.shape[1]
    print(dimVectors)
    # Load the train set
    trainset = dataset.getTrainSentences()
    nTrain = len(trainset)
    trainFeatures = np.zeros((nTrain, dimVectors))
    trainLabels = np.zeros((nTrain,), dtype=np.int32)
    for i in range(nTrain):
        words, trainLabels[i] = trainset[i]
        trainFeatures[i, :] = getSentenceFeatures(tokens, wordVectors, words)

    # Prepare dev set features
    devset = dataset.getDevSentences()
    nDev = len(devset)
    devFeatures = np.zeros((nDev, dimVectors))
    devLabels = np.zeros((nDev,), dtype=np.int32)
    for i in range(nDev):
        words, devLabels[i] = devset[i]
        devFeatures[i, :] = getSentenceFeatures(tokens, wordVectors, words)

    # Prepare test set features
    testset = dataset.getTestSentences()
    nTest = len(testset)
    testFeatures = np.zeros((nTest, dimVectors))
    testLabels = np.zeros((nTest,), dtype=np.int32)
    for i in range(nTest):
        words, testLabels[i] = testset[i]
        testFeatures[i, :] = getSentenceFeatures(tokens, wordVectors, words)

    # We will save our results from each run
    results = []
    regValues = getRegularizationValues()
    for reg in regValues:
        print("Training for reg=%f" % reg)
        # Note: add a very small number to regularization to please the library
        clf = LogisticRegression(C=1.0/(reg + 1e-12))
        clf.fit(trainFeatures, trainLabels)

        # Test on train set
        pred = clf.predict(trainFeatures)
        trainAccuracy = accuracy(trainLabels, pred)
        print("Train accuracy (%%): %f" % trainAccuracy)

        # Test on dev set
        pred = clf.predict(devFeatures)
        devAccuracy = accuracy(devLabels, pred)
        print("Dev accuracy (%%): %f" % devAccuracy)

        # Test on test set
        # Note: always running on test is poor style. Typically, you should
        # do this only after validation.
        pred = clf.predict(testFeatures)
        testAccuracy = accuracy(testLabels, pred)
        print("Test accuracy (%%): %f" % testAccuracy)

        results.append({
            "reg": reg,
            "clf": clf,
            "train": trainAccuracy,
            "dev": devAccuracy,
            "test": testAccuracy})

    # Print the accuracies
    print("")
    print("=== Recap ===")
    print("Reg\t\tTrain\tDev\tTest")

    for result in results:
        print("%.2E\t%.3f\t%.3f\t%.3f" % (
            result["reg"],
            result["train"],
            result["dev"],
            result["test"]))
    print("")

    bestResult = chooseBestModel(results)
    print("Best regularization value: %0.2E" % bestResult["reg"])
    print("Test accuracy (%%): %f" % bestResult["test"])

    # do some error analysis
    if args == 'pretrained':
        plotRegVsAccuracy(regValues, results, "q4_reg_v_acc.png")
        outputConfusionMatrix(devFeatures, devLabels, bestResult["clf"],
                              "q4_dev_conf.png")
        outputPredictions(devset, devFeatures, devLabels, bestResult["clf"],
                          "q4_dev_pred.txt")
In [12]:
main('yourvectors')
20000
20
Training for reg=0.000100
Train accuracy (%): 30.770131
Dev accuracy (%): 30.517711
Test accuracy (%): 28.914027
Training for reg=0.000115
Train accuracy (%): 30.770131
Dev accuracy (%): 30.336058
Test accuracy (%): 28.823529
Training for reg=0.000132
Train accuracy (%): 30.805243
Dev accuracy (%): 30.426885
Test accuracy (%): 28.778281
Training for reg=0.000152
Train accuracy (%): 30.770131
Dev accuracy (%): 30.517711
Test accuracy (%): 28.733032
Training for reg=0.000175
Train accuracy (%): 30.676498
Dev accuracy (%): 30.517711
Test accuracy (%): 28.823529
Training for reg=0.000201
Train accuracy (%): 30.711610
Dev accuracy (%): 30.517711
Test accuracy (%): 28.733032
Training for reg=0.000231
Train accuracy (%): 30.723315
Dev accuracy (%): 30.426885
Test accuracy (%): 28.823529
Training for reg=0.000266
Train accuracy (%): 30.723315
Dev accuracy (%): 30.426885
Test accuracy (%): 28.733032
Training for reg=0.000305
Train accuracy (%): 30.723315
Dev accuracy (%): 30.517711
Test accuracy (%): 28.733032
Training for reg=0.000351
Train accuracy (%): 30.711610
Dev accuracy (%): 30.517711
Test accuracy (%): 28.778281
Training for reg=0.000404
Train accuracy (%): 30.676498
Dev accuracy (%): 30.426885
Test accuracy (%): 28.823529
Training for reg=0.000464
Train accuracy (%): 30.699906
Dev accuracy (%): 30.699364
Test accuracy (%): 28.823529
Training for reg=0.000534
Train accuracy (%): 30.711610
Dev accuracy (%): 30.608538
Test accuracy (%): 28.868778
Training for reg=0.000614
Train accuracy (%): 30.664794
Dev accuracy (%): 30.608538
Test accuracy (%): 28.778281
Training for reg=0.000705
Train accuracy (%): 30.699906
Dev accuracy (%): 30.790191
Test accuracy (%): 28.868778
Training for reg=0.000811
Train accuracy (%): 30.723315
Dev accuracy (%): 30.699364
Test accuracy (%): 28.868778
Training for reg=0.000933
Train accuracy (%): 30.711610
Dev accuracy (%): 30.699364
Test accuracy (%): 28.914027
Training for reg=0.001072
Train accuracy (%): 30.699906
Dev accuracy (%): 30.699364
Test accuracy (%): 28.868778
Training for reg=0.001233
Train accuracy (%): 30.688202
Dev accuracy (%): 30.608538
Test accuracy (%): 28.778281
Training for reg=0.001417
Train accuracy (%): 30.653090
Dev accuracy (%): 30.608538
Test accuracy (%): 28.778281
Training for reg=0.001630
Train accuracy (%): 30.723315
Dev accuracy (%): 30.790191
Test accuracy (%): 28.687783
Training for reg=0.001874
Train accuracy (%): 30.676498
Dev accuracy (%): 30.790191
Test accuracy (%): 28.868778
Training for reg=0.002154
Train accuracy (%): 30.664794
Dev accuracy (%): 30.790191
Test accuracy (%): 28.868778
Training for reg=0.002477
Train accuracy (%): 30.653090
Dev accuracy (%): 30.971844
Test accuracy (%): 28.778281
Training for reg=0.002848
Train accuracy (%): 30.688202
Dev accuracy (%): 30.881017
Test accuracy (%): 28.868778
Training for reg=0.003275
Train accuracy (%): 30.688202
Dev accuracy (%): 30.790191
Test accuracy (%): 28.778281
Training for reg=0.003765
Train accuracy (%): 30.699906
Dev accuracy (%): 30.699364
Test accuracy (%): 28.914027
Training for reg=0.004329
Train accuracy (%): 30.676498
Dev accuracy (%): 30.699364
Test accuracy (%): 29.140271
Training for reg=0.004977
Train accuracy (%): 30.653090
Dev accuracy (%): 30.699364
Test accuracy (%): 29.095023
Training for reg=0.005722
Train accuracy (%): 30.805243
Dev accuracy (%): 30.699364
Test accuracy (%): 29.095023
Training for reg=0.006579
Train accuracy (%): 30.816948
Dev accuracy (%): 30.881017
Test accuracy (%): 28.959276
Training for reg=0.007565
Train accuracy (%): 30.781835
Dev accuracy (%): 30.881017
Test accuracy (%): 28.823529
Training for reg=0.008697
Train accuracy (%): 30.758427
Dev accuracy (%): 30.881017
Test accuracy (%): 28.733032
Training for reg=0.010000
Train accuracy (%): 30.898876
Dev accuracy (%): 30.790191
Test accuracy (%): 28.733032
Training for reg=0.011498
Train accuracy (%): 30.781835
Dev accuracy (%): 30.699364
Test accuracy (%): 28.733032
Training for reg=0.013219
Train accuracy (%): 30.711610
Dev accuracy (%): 30.699364
Test accuracy (%): 28.914027
Training for reg=0.015199
Train accuracy (%): 30.852060
Dev accuracy (%): 30.426885
Test accuracy (%): 29.049774
Training for reg=0.017475
Train accuracy (%): 30.758427
Dev accuracy (%): 30.699364
Test accuracy (%): 28.868778
Training for reg=0.020092
Train accuracy (%): 30.641386
Dev accuracy (%): 30.699364
Test accuracy (%): 28.868778
Training for reg=0.023101
Train accuracy (%): 30.676498
Dev accuracy (%): 30.608538
Test accuracy (%): 28.552036
Training for reg=0.026561
Train accuracy (%): 30.489232
Dev accuracy (%): 30.426885
Test accuracy (%): 28.506787
Training for reg=0.030539
Train accuracy (%): 30.383895
Dev accuracy (%): 30.336058
Test accuracy (%): 28.371041
Training for reg=0.035112
Train accuracy (%): 30.301966
Dev accuracy (%): 30.245232
Test accuracy (%): 28.280543
Training for reg=0.040370
Train accuracy (%): 30.208333
Dev accuracy (%): 30.245232
Test accuracy (%): 28.235294
Training for reg=0.046416
Train accuracy (%): 29.997659
Dev accuracy (%): 29.881926
Test accuracy (%): 28.325792
Training for reg=0.053367
Train accuracy (%): 30.021067
Dev accuracy (%): 30.063579
Test accuracy (%): 28.190045
Training for reg=0.061359
Train accuracy (%): 29.880618
Dev accuracy (%): 29.972752
Test accuracy (%): 28.144796
Training for reg=0.070548
Train accuracy (%): 29.798689
Dev accuracy (%): 29.791099
Test accuracy (%): 28.099548
Training for reg=0.081113
Train accuracy (%): 29.822097
Dev accuracy (%): 30.154405
Test accuracy (%): 28.009050
Training for reg=0.093260
Train accuracy (%): 29.740169
Dev accuracy (%): 30.336058
Test accuracy (%): 27.873303
Training for reg=0.107227
Train accuracy (%): 29.623127
Dev accuracy (%): 30.517711
Test accuracy (%): 27.873303
Training for reg=0.123285
Train accuracy (%): 29.634831
Dev accuracy (%): 30.426885
Test accuracy (%): 27.737557
Training for reg=0.141747
Train accuracy (%): 29.506086
Dev accuracy (%): 30.426885
Test accuracy (%): 27.873303
Training for reg=0.162975
Train accuracy (%): 29.447566
Dev accuracy (%): 30.790191
Test accuracy (%): 27.466063
Training for reg=0.187382
Train accuracy (%): 29.517790
Dev accuracy (%): 30.517711
Test accuracy (%): 27.692308
Training for reg=0.215443
Train accuracy (%): 29.588015
Dev accuracy (%): 30.517711
Test accuracy (%): 27.511312
Training for reg=0.247708
Train accuracy (%): 29.389045
Dev accuracy (%): 30.154405
Test accuracy (%): 27.647059
Training for reg=0.284804
Train accuracy (%): 29.272004
Dev accuracy (%): 30.245232
Test accuracy (%): 27.647059
Training for reg=0.327455
Train accuracy (%): 29.248596
Dev accuracy (%): 30.426885
Test accuracy (%): 27.737557
Training for reg=0.376494
Train accuracy (%): 29.248596
Dev accuracy (%): 30.245232
Test accuracy (%): 27.647059
Training for reg=0.432876
Train accuracy (%): 29.108146
Dev accuracy (%): 30.154405
Test accuracy (%): 27.194570
Training for reg=0.497702
Train accuracy (%): 29.096442
Dev accuracy (%): 30.063579
Test accuracy (%): 27.058824
Training for reg=0.572237
Train accuracy (%): 29.026217
Dev accuracy (%): 29.700272
Test accuracy (%): 27.149321
Training for reg=0.657933
Train accuracy (%): 28.897472
Dev accuracy (%): 29.609446
Test accuracy (%): 26.832579
Training for reg=0.756463
Train accuracy (%): 28.862360
Dev accuracy (%): 29.336966
Test accuracy (%): 26.561086
Training for reg=0.869749
Train accuracy (%): 28.710206
Dev accuracy (%): 28.882834
Test accuracy (%): 26.787330
Training for reg=1.000000
Train accuracy (%): 28.593165
Dev accuracy (%): 28.337875
Test accuracy (%): 26.289593
Training for reg=1.149757
Train accuracy (%): 28.417603
Dev accuracy (%): 27.974569
Test accuracy (%): 25.837104
Training for reg=1.321941
Train accuracy (%): 28.347378
Dev accuracy (%): 27.429609
Test accuracy (%): 25.384615
Training for reg=1.519911
Train accuracy (%): 28.101592
Dev accuracy (%): 26.884650
Test accuracy (%): 25.248869
Training for reg=1.747528
Train accuracy (%): 28.054775
Dev accuracy (%): 26.430518
Test accuracy (%): 25.203620
Training for reg=2.009233
Train accuracy (%): 27.879213
Dev accuracy (%): 26.067212
Test accuracy (%): 25.022624
Training for reg=2.310130
Train accuracy (%): 27.633427
Dev accuracy (%): 26.248865
Test accuracy (%): 25.022624
Training for reg=2.656088
Train accuracy (%): 27.551498
Dev accuracy (%): 26.158038
Test accuracy (%): 24.615385
Training for reg=3.053856
Train accuracy (%): 27.375936
Dev accuracy (%): 25.885559
Test accuracy (%): 24.072398
Training for reg=3.511192
Train accuracy (%): 27.457865
Dev accuracy (%): 25.976385
Test accuracy (%): 23.800905
Training for reg=4.037017
Train accuracy (%): 27.223783
Dev accuracy (%): 26.067212
Test accuracy (%): 23.619910
Training for reg=4.641589
Train accuracy (%): 27.176966
Dev accuracy (%): 25.703906
Test accuracy (%): 23.393665
Training for reg=5.336699
Train accuracy (%): 27.106742
Dev accuracy (%): 25.703906
Test accuracy (%): 23.122172
Training for reg=6.135907
Train accuracy (%): 27.130150
Dev accuracy (%): 25.522252
Test accuracy (%): 23.076923
Training for reg=7.054802
Train accuracy (%): 27.095037
Dev accuracy (%): 25.431426
Test accuracy (%): 23.076923
Training for reg=8.111308
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 23.031674
Training for reg=9.326033
Train accuracy (%): 27.036517
Dev accuracy (%): 25.340599
Test accuracy (%): 22.986425
Training for reg=10.722672
Train accuracy (%): 27.048221
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=12.328467
Train accuracy (%): 27.048221
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=14.174742
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=16.297508
Train accuracy (%): 27.059925
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=18.738174
Train accuracy (%): 27.059925
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=21.544347
Train accuracy (%): 27.059925
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=24.770764
Train accuracy (%): 27.059925
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=28.480359
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=32.745492
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=37.649358
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=43.287613
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=49.770236
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=57.223677
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=65.793322
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=75.646333
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=86.974900
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928
Training for reg=100.000000
Train accuracy (%): 27.071629
Dev accuracy (%): 25.340599
Test accuracy (%): 22.895928

=== Recap ===
Reg		Train	Dev	Test
1.00E-04	30.770	30.518	28.914
1.15E-04	30.770	30.336	28.824
1.32E-04	30.805	30.427	28.778
1.52E-04	30.770	30.518	28.733
1.75E-04	30.676	30.518	28.824
2.01E-04	30.712	30.518	28.733
2.31E-04	30.723	30.427	28.824
2.66E-04	30.723	30.427	28.733
3.05E-04	30.723	30.518	28.733
3.51E-04	30.712	30.518	28.778
4.04E-04	30.676	30.427	28.824
4.64E-04	30.700	30.699	28.824
5.34E-04	30.712	30.609	28.869
6.14E-04	30.665	30.609	28.778
7.05E-04	30.700	30.790	28.869
8.11E-04	30.723	30.699	28.869
9.33E-04	30.712	30.699	28.914
1.07E-03	30.700	30.699	28.869
1.23E-03	30.688	30.609	28.778
1.42E-03	30.653	30.609	28.778
1.63E-03	30.723	30.790	28.688
1.87E-03	30.676	30.790	28.869
2.15E-03	30.665	30.790	28.869
2.48E-03	30.653	30.972	28.778
2.85E-03	30.688	30.881	28.869
3.27E-03	30.688	30.790	28.778
3.76E-03	30.700	30.699	28.914
4.33E-03	30.676	30.699	29.140
4.98E-03	30.653	30.699	29.095
5.72E-03	30.805	30.699	29.095
6.58E-03	30.817	30.881	28.959
7.56E-03	30.782	30.881	28.824
8.70E-03	30.758	30.881	28.733
1.00E-02	30.899	30.790	28.733
1.15E-02	30.782	30.699	28.733
1.32E-02	30.712	30.699	28.914
1.52E-02	30.852	30.427	29.050
1.75E-02	30.758	30.699	28.869
2.01E-02	30.641	30.699	28.869
2.31E-02	30.676	30.609	28.552
2.66E-02	30.489	30.427	28.507
3.05E-02	30.384	30.336	28.371
3.51E-02	30.302	30.245	28.281
4.04E-02	30.208	30.245	28.235
4.64E-02	29.998	29.882	28.326
5.34E-02	30.021	30.064	28.190
6.14E-02	29.881	29.973	28.145
7.05E-02	29.799	29.791	28.100
8.11E-02	29.822	30.154	28.009
9.33E-02	29.740	30.336	27.873
1.07E-01	29.623	30.518	27.873
1.23E-01	29.635	30.427	27.738
1.42E-01	29.506	30.427	27.873
1.63E-01	29.448	30.790	27.466
1.87E-01	29.518	30.518	27.692
2.15E-01	29.588	30.518	27.511
2.48E-01	29.389	30.154	27.647
2.85E-01	29.272	30.245	27.647
3.27E-01	29.249	30.427	27.738
3.76E-01	29.249	30.245	27.647
4.33E-01	29.108	30.154	27.195
4.98E-01	29.096	30.064	27.059
5.72E-01	29.026	29.700	27.149
6.58E-01	28.897	29.609	26.833
7.56E-01	28.862	29.337	26.561
8.70E-01	28.710	28.883	26.787
1.00E+00	28.593	28.338	26.290
1.15E+00	28.418	27.975	25.837
1.32E+00	28.347	27.430	25.385
1.52E+00	28.102	26.885	25.249
1.75E+00	28.055	26.431	25.204
2.01E+00	27.879	26.067	25.023
2.31E+00	27.633	26.249	25.023
2.66E+00	27.551	26.158	24.615
3.05E+00	27.376	25.886	24.072
3.51E+00	27.458	25.976	23.801
4.04E+00	27.224	26.067	23.620
4.64E+00	27.177	25.704	23.394
5.34E+00	27.107	25.704	23.122
6.14E+00	27.130	25.522	23.077
7.05E+00	27.095	25.431	23.077
8.11E+00	27.072	25.341	23.032
9.33E+00	27.037	25.341	22.986
1.07E+01	27.048	25.341	22.896
1.23E+01	27.048	25.341	22.896
1.42E+01	27.072	25.341	22.896
1.63E+01	27.060	25.341	22.896
1.87E+01	27.060	25.341	22.896
2.15E+01	27.060	25.341	22.896
2.48E+01	27.060	25.341	22.896
2.85E+01	27.072	25.341	22.896
3.27E+01	27.072	25.341	22.896
3.76E+01	27.072	25.341	22.896
4.33E+01	27.072	25.341	22.896
4.98E+01	27.072	25.341	22.896
5.72E+01	27.072	25.341	22.896
6.58E+01	27.072	25.341	22.896
7.56E+01	27.072	25.341	22.896
8.70E+01	27.072	25.341	22.896
1.00E+02	27.072	25.341	22.896

Best regularization value: 2.48E-03
Test accuracy (%): 28.778281
In [13]:
 main('pretrained')
50
Training for reg=0.000100
Train accuracy (%): 39.325843
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.000115
Train accuracy (%): 39.314139
Dev accuracy (%): 36.148955
Test accuracy (%): 36.968326
Training for reg=0.000132
Train accuracy (%): 39.314139
Dev accuracy (%): 36.148955
Test accuracy (%): 36.968326
Training for reg=0.000152
Train accuracy (%): 39.325843
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.000175
Train accuracy (%): 39.325843
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.000201
Train accuracy (%): 39.314139
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.000231
Train accuracy (%): 39.325843
Dev accuracy (%): 36.148955
Test accuracy (%): 36.923077
Training for reg=0.000266
Train accuracy (%): 39.314139
Dev accuracy (%): 36.148955
Test accuracy (%): 36.968326
Training for reg=0.000305
Train accuracy (%): 39.302434
Dev accuracy (%): 36.148955
Test accuracy (%): 36.968326
Training for reg=0.000351
Train accuracy (%): 39.325843
Dev accuracy (%): 36.148955
Test accuracy (%): 36.923077
Training for reg=0.000404
Train accuracy (%): 39.325843
Dev accuracy (%): 36.148955
Test accuracy (%): 36.923077
Training for reg=0.000464
Train accuracy (%): 39.337547
Dev accuracy (%): 36.148955
Test accuracy (%): 36.923077
Training for reg=0.000534
Train accuracy (%): 39.314139
Dev accuracy (%): 36.148955
Test accuracy (%): 36.923077
Training for reg=0.000614
Train accuracy (%): 39.302434
Dev accuracy (%): 36.148955
Test accuracy (%): 36.968326
Training for reg=0.000705
Train accuracy (%): 39.337547
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.000811
Train accuracy (%): 39.337547
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.000933
Train accuracy (%): 39.360955
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.001072
Train accuracy (%): 39.360955
Dev accuracy (%): 36.148955
Test accuracy (%): 36.968326
Training for reg=0.001233
Train accuracy (%): 39.349251
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.001417
Train accuracy (%): 39.372659
Dev accuracy (%): 36.148955
Test accuracy (%): 36.968326
Training for reg=0.001630
Train accuracy (%): 39.360955
Dev accuracy (%): 36.148955
Test accuracy (%): 36.968326
Training for reg=0.001874
Train accuracy (%): 39.314139
Dev accuracy (%): 36.058129
Test accuracy (%): 37.013575
Training for reg=0.002154
Train accuracy (%): 39.314139
Dev accuracy (%): 36.058129
Test accuracy (%): 36.968326
Training for reg=0.002477
Train accuracy (%): 39.302434
Dev accuracy (%): 36.058129
Test accuracy (%): 36.923077
Training for reg=0.002848
Train accuracy (%): 39.360955
Dev accuracy (%): 36.148955
Test accuracy (%): 37.058824
Training for reg=0.003275
Train accuracy (%): 39.325843
Dev accuracy (%): 36.058129
Test accuracy (%): 36.923077
Training for reg=0.003765
Train accuracy (%): 39.360955
Dev accuracy (%): 36.148955
Test accuracy (%): 36.923077
Training for reg=0.004329
Train accuracy (%): 39.349251
Dev accuracy (%): 36.148955
Test accuracy (%): 36.877828
Training for reg=0.004977
Train accuracy (%): 39.360955
Dev accuracy (%): 36.148955
Test accuracy (%): 36.923077
Training for reg=0.005722
Train accuracy (%): 39.396067
Dev accuracy (%): 36.058129
Test accuracy (%): 37.013575
Training for reg=0.006579
Train accuracy (%): 39.360955
Dev accuracy (%): 35.967302
Test accuracy (%): 37.013575
Training for reg=0.007565
Train accuracy (%): 39.360955
Dev accuracy (%): 35.967302
Test accuracy (%): 36.968326
Training for reg=0.008697
Train accuracy (%): 39.384363
Dev accuracy (%): 36.058129
Test accuracy (%): 36.968326
Training for reg=0.010000
Train accuracy (%): 39.419476
Dev accuracy (%): 36.058129
Test accuracy (%): 36.968326
Training for reg=0.011498
Train accuracy (%): 39.396067
Dev accuracy (%): 35.876476
Test accuracy (%): 37.013575
Training for reg=0.013219
Train accuracy (%): 39.431180
Dev accuracy (%): 35.785649
Test accuracy (%): 36.968326
Training for reg=0.015199
Train accuracy (%): 39.431180
Dev accuracy (%): 35.694823
Test accuracy (%): 37.013575
Training for reg=0.017475
Train accuracy (%): 39.431180
Dev accuracy (%): 35.694823
Test accuracy (%): 36.968326
Training for reg=0.020092
Train accuracy (%): 39.384363
Dev accuracy (%): 35.694823
Test accuracy (%): 37.058824
Training for reg=0.023101
Train accuracy (%): 39.337547
Dev accuracy (%): 35.876476
Test accuracy (%): 37.104072
Training for reg=0.026561
Train accuracy (%): 39.360955
Dev accuracy (%): 35.967302
Test accuracy (%): 37.058824
Training for reg=0.030539
Train accuracy (%): 39.360955
Dev accuracy (%): 36.148955
Test accuracy (%): 37.104072
Training for reg=0.035112
Train accuracy (%): 39.442884
Dev accuracy (%): 36.330609
Test accuracy (%): 37.104072
Training for reg=0.040370
Train accuracy (%): 39.431180
Dev accuracy (%): 36.603088
Test accuracy (%): 37.058824
Training for reg=0.046416
Train accuracy (%): 39.419476
Dev accuracy (%): 36.421435
Test accuracy (%): 37.104072
Training for reg=0.053367
Train accuracy (%): 39.407772
Dev accuracy (%): 36.512262
Test accuracy (%): 37.194570
Training for reg=0.061359
Train accuracy (%): 39.454588
Dev accuracy (%): 36.421435
Test accuracy (%): 37.194570
Training for reg=0.070548
Train accuracy (%): 39.442884
Dev accuracy (%): 36.512262
Test accuracy (%): 37.104072
Training for reg=0.081113
Train accuracy (%): 39.384363
Dev accuracy (%): 36.603088
Test accuracy (%): 37.149321
Training for reg=0.093260
Train accuracy (%): 39.360955
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.107227
Train accuracy (%): 39.360955
Dev accuracy (%): 36.148955
Test accuracy (%): 37.058824
Training for reg=0.123285
Train accuracy (%): 39.314139
Dev accuracy (%): 36.148955
Test accuracy (%): 37.013575
Training for reg=0.141747
Train accuracy (%): 39.267322
Dev accuracy (%): 35.694823
Test accuracy (%): 37.194570
Training for reg=0.162975
Train accuracy (%): 39.279026
Dev accuracy (%): 35.603996
Test accuracy (%): 37.149321
Training for reg=0.187382
Train accuracy (%): 39.232210
Dev accuracy (%): 35.513170
Test accuracy (%): 37.149321
Training for reg=0.215443
Train accuracy (%): 39.232210
Dev accuracy (%): 35.513170
Test accuracy (%): 37.058824
Training for reg=0.247708
Train accuracy (%): 39.161985
Dev accuracy (%): 35.603996
Test accuracy (%): 37.058824
Training for reg=0.284804
Train accuracy (%): 39.080056
Dev accuracy (%): 35.603996
Test accuracy (%): 37.013575
Training for reg=0.327455
Train accuracy (%): 39.033240
Dev accuracy (%): 35.694823
Test accuracy (%): 37.104072
Training for reg=0.376494
Train accuracy (%): 38.986423
Dev accuracy (%): 35.694823
Test accuracy (%): 37.194570
Training for reg=0.432876
Train accuracy (%): 39.056648
Dev accuracy (%): 35.694823
Test accuracy (%): 37.149321
Training for reg=0.497702
Train accuracy (%): 39.080056
Dev accuracy (%): 35.694823
Test accuracy (%): 37.058824
Training for reg=0.572237
Train accuracy (%): 39.091760
Dev accuracy (%): 35.513170
Test accuracy (%): 37.013575
Training for reg=0.657933
Train accuracy (%): 39.115169
Dev accuracy (%): 35.694823
Test accuracy (%): 37.104072
Training for reg=0.756463
Train accuracy (%): 39.115169
Dev accuracy (%): 35.785649
Test accuracy (%): 37.104072
Training for reg=0.869749
Train accuracy (%): 39.115169
Dev accuracy (%): 35.694823
Test accuracy (%): 37.104072
Training for reg=1.000000
Train accuracy (%): 39.103464
Dev accuracy (%): 35.785649
Test accuracy (%): 37.013575
Training for reg=1.149757
Train accuracy (%): 39.115169
Dev accuracy (%): 35.785649
Test accuracy (%): 36.923077
Training for reg=1.321941
Train accuracy (%): 39.068352
Dev accuracy (%): 35.694823
Test accuracy (%): 36.832579
Training for reg=1.519911
Train accuracy (%): 39.080056
Dev accuracy (%): 35.694823
Test accuracy (%): 36.651584
Training for reg=1.747528
Train accuracy (%): 38.974719
Dev accuracy (%): 35.785649
Test accuracy (%): 36.651584
Training for reg=2.009233
Train accuracy (%): 38.986423
Dev accuracy (%): 35.967302
Test accuracy (%): 36.696833
Training for reg=2.310130
Train accuracy (%): 38.904494
Dev accuracy (%): 36.058129
Test accuracy (%): 36.696833
Training for reg=2.656088
Train accuracy (%): 38.904494
Dev accuracy (%): 36.148955
Test accuracy (%): 36.606335
Training for reg=3.053856
Train accuracy (%): 38.904494
Dev accuracy (%): 36.239782
Test accuracy (%): 36.696833
Training for reg=3.511192
Train accuracy (%): 38.822566
Dev accuracy (%): 36.330609
Test accuracy (%): 36.832579
Training for reg=4.037017
Train accuracy (%): 38.752341
Dev accuracy (%): 36.512262
Test accuracy (%): 36.787330
Training for reg=4.641589
Train accuracy (%): 38.834270
Dev accuracy (%): 36.603088
Test accuracy (%): 36.696833
Training for reg=5.336699
Train accuracy (%): 38.728933
Dev accuracy (%): 36.421435
Test accuracy (%): 36.651584
Training for reg=6.135907
Train accuracy (%): 38.717228
Dev accuracy (%): 36.239782
Test accuracy (%): 37.104072
Training for reg=7.054802
Train accuracy (%): 38.553371
Dev accuracy (%): 36.148955
Test accuracy (%): 37.466063
Training for reg=8.111308
Train accuracy (%): 38.295880
Dev accuracy (%): 36.058129
Test accuracy (%): 37.511312
Training for reg=9.326033
Train accuracy (%): 38.190543
Dev accuracy (%): 36.239782
Test accuracy (%): 37.556561
Training for reg=10.722672
Train accuracy (%): 38.237360
Dev accuracy (%): 36.512262
Test accuracy (%): 37.375566
Training for reg=12.328467
Train accuracy (%): 38.132022
Dev accuracy (%): 36.239782
Test accuracy (%): 37.194570
Training for reg=14.174742
Train accuracy (%): 38.026685
Dev accuracy (%): 36.239782
Test accuracy (%): 37.104072
Training for reg=16.297508
Train accuracy (%): 37.874532
Dev accuracy (%): 36.239782
Test accuracy (%): 37.149321
Training for reg=18.738174
Train accuracy (%): 37.804307
Dev accuracy (%): 36.421435
Test accuracy (%): 36.787330
Training for reg=21.544347
Train accuracy (%): 37.710674
Dev accuracy (%): 36.148955
Test accuracy (%): 36.651584
Training for reg=24.770764
Train accuracy (%): 37.453184
Dev accuracy (%): 36.239782
Test accuracy (%): 36.515837
Training for reg=28.480359
Train accuracy (%): 37.359551
Dev accuracy (%): 36.239782
Test accuracy (%): 36.651584
Training for reg=32.745492
Train accuracy (%): 37.183989
Dev accuracy (%): 36.330609
Test accuracy (%): 36.380090
Training for reg=37.649358
Train accuracy (%): 37.137172
Dev accuracy (%): 36.512262
Test accuracy (%): 36.153846
Training for reg=43.287613
Train accuracy (%): 36.938202
Dev accuracy (%): 36.239782
Test accuracy (%): 36.289593
Training for reg=49.770236
Train accuracy (%): 36.750936
Dev accuracy (%): 36.058129
Test accuracy (%): 36.063348
Training for reg=57.223677
Train accuracy (%): 36.692416
Dev accuracy (%): 35.785649
Test accuracy (%): 36.063348
Training for reg=65.793322
Train accuracy (%): 36.481742
Dev accuracy (%): 35.513170
Test accuracy (%): 35.791855
Training for reg=75.646333
Train accuracy (%): 36.235955
Dev accuracy (%): 35.149864
Test accuracy (%): 35.701357
Training for reg=86.974900
Train accuracy (%): 36.142322
Dev accuracy (%): 34.604905
Test accuracy (%): 35.339367
Training for reg=100.000000
Train accuracy (%): 35.791199
Dev accuracy (%): 34.786558
Test accuracy (%): 35.022624

=== Recap ===
Reg		Train	Dev	Test
1.00E-04	39.326	36.149	37.014
1.15E-04	39.314	36.149	36.968
1.32E-04	39.314	36.149	36.968
1.52E-04	39.326	36.149	37.014
1.75E-04	39.326	36.149	37.014
2.01E-04	39.314	36.149	37.014
2.31E-04	39.326	36.149	36.923
2.66E-04	39.314	36.149	36.968
3.05E-04	39.302	36.149	36.968
3.51E-04	39.326	36.149	36.923
4.04E-04	39.326	36.149	36.923
4.64E-04	39.338	36.149	36.923
5.34E-04	39.314	36.149	36.923
6.14E-04	39.302	36.149	36.968
7.05E-04	39.338	36.149	37.014
8.11E-04	39.338	36.149	37.014
9.33E-04	39.361	36.149	37.014
1.07E-03	39.361	36.149	36.968
1.23E-03	39.349	36.149	37.014
1.42E-03	39.373	36.149	36.968
1.63E-03	39.361	36.149	36.968
1.87E-03	39.314	36.058	37.014
2.15E-03	39.314	36.058	36.968
2.48E-03	39.302	36.058	36.923
2.85E-03	39.361	36.149	37.059
3.27E-03	39.326	36.058	36.923
3.76E-03	39.361	36.149	36.923
4.33E-03	39.349	36.149	36.878
4.98E-03	39.361	36.149	36.923
5.72E-03	39.396	36.058	37.014
6.58E-03	39.361	35.967	37.014
7.56E-03	39.361	35.967	36.968
8.70E-03	39.384	36.058	36.968
1.00E-02	39.419	36.058	36.968
1.15E-02	39.396	35.876	37.014
1.32E-02	39.431	35.786	36.968
1.52E-02	39.431	35.695	37.014
1.75E-02	39.431	35.695	36.968
2.01E-02	39.384	35.695	37.059
2.31E-02	39.338	35.876	37.104
2.66E-02	39.361	35.967	37.059
3.05E-02	39.361	36.149	37.104
3.51E-02	39.443	36.331	37.104
4.04E-02	39.431	36.603	37.059
4.64E-02	39.419	36.421	37.104
5.34E-02	39.408	36.512	37.195
6.14E-02	39.455	36.421	37.195
7.05E-02	39.443	36.512	37.104
8.11E-02	39.384	36.603	37.149
9.33E-02	39.361	36.149	37.014
1.07E-01	39.361	36.149	37.059
1.23E-01	39.314	36.149	37.014
1.42E-01	39.267	35.695	37.195
1.63E-01	39.279	35.604	37.149
1.87E-01	39.232	35.513	37.149
2.15E-01	39.232	35.513	37.059
2.48E-01	39.162	35.604	37.059
2.85E-01	39.080	35.604	37.014
3.27E-01	39.033	35.695	37.104
3.76E-01	38.986	35.695	37.195
4.33E-01	39.057	35.695	37.149
4.98E-01	39.080	35.695	37.059
5.72E-01	39.092	35.513	37.014
6.58E-01	39.115	35.695	37.104
7.56E-01	39.115	35.786	37.104
8.70E-01	39.115	35.695	37.104
1.00E+00	39.103	35.786	37.014
1.15E+00	39.115	35.786	36.923
1.32E+00	39.068	35.695	36.833
1.52E+00	39.080	35.695	36.652
1.75E+00	38.975	35.786	36.652
2.01E+00	38.986	35.967	36.697
2.31E+00	38.904	36.058	36.697
2.66E+00	38.904	36.149	36.606
3.05E+00	38.904	36.240	36.697
3.51E+00	38.823	36.331	36.833
4.04E+00	38.752	36.512	36.787
4.64E+00	38.834	36.603	36.697
5.34E+00	38.729	36.421	36.652
6.14E+00	38.717	36.240	37.104
7.05E+00	38.553	36.149	37.466
8.11E+00	38.296	36.058	37.511
9.33E+00	38.191	36.240	37.557
1.07E+01	38.237	36.512	37.376
1.23E+01	38.132	36.240	37.195
1.42E+01	38.027	36.240	37.104
1.63E+01	37.875	36.240	37.149
1.87E+01	37.804	36.421	36.787
2.15E+01	37.711	36.149	36.652
2.48E+01	37.453	36.240	36.516
2.85E+01	37.360	36.240	36.652
3.27E+01	37.184	36.331	36.380
3.76E+01	37.137	36.512	36.154
4.33E+01	36.938	36.240	36.290
4.98E+01	36.751	36.058	36.063
5.72E+01	36.692	35.786	36.063
6.58E+01	36.482	35.513	35.792
7.56E+01	36.236	35.150	35.701
8.70E+01	36.142	34.605	35.339
1.00E+02	35.791	34.787	35.023

Best regularization value: 4.04E-02
Test accuracy (%): 37.058824
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]: