In [1]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_matching/snli/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
%tensorflow_version 2.x
!pip install tensorflow-addons
!pip install transformers
Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (0.8.3)
Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons) (2.7.1)
Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.0.2)
Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)
Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)
Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)
Requirement already satisfied: tokenizers==0.8.1.rc1 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8.1rc1)
Requirement already satisfied: dataclasses; python_version < "3.7" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)
Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)
Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)
Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)
Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)
Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)
Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)
Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.12.0)
Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)
Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)
In [3]:
from transformers import RobertaTokenizer, TFRobertaModel

import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pprint
import logging
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.2.0
WARNING:tensorflow:From <ipython-input-3-1e763de9ead9>:11: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU Enabled: True
In [4]:
params = {
  'train_path': '../data/train.txt',
  'test_path': '../data/test.txt',
  'pretrain_path': 'roberta-base',
  'num_samples': 550152,
  'buffer_size': 200000,
  'batch_size': 32,
  'max_len': 128 + 3,
  'num_patience': 5,
  'init_lr': 1e-5,
  'max_lr': 3e-5,
}
In [5]:
tokenizer = RobertaTokenizer.from_pretrained(params['pretrain_path'],
                                             lowercase = True,
                                             add_special_tokens = True)
In [6]:
# stream data from text files
def data_generator(f_path, params):
  label2idx = {'neutral': 0, 'entailment': 1, 'contradiction': 2,}
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = line.rstrip()
      label, text1, text2 = line.split('\t')
      if label == '-':
        continue
      text1 = tokenizer.tokenize(text1)
      text2 = tokenizer.tokenize(text2)
      if len(text1) + len(text2) + 3 > params['max_len']:
        _max_len = (params['max_len'] - 3) // 2
        text1 = text1[:_max_len]
        text2 = text2[:_max_len]
      text = ['<s>'] + text1 + ['</s>'] + text2 + ['</s>']
      text = tokenizer.convert_tokens_to_ids(text)
      yield text, label2idx[label]


def dataset(is_training, params):
  _shapes = ([None], ())
  _types = (tf.int32, tf.int32)
  _pads = (1, -1)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['buffer_size'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [7]:
# input stream ids check
text, _ = next(data_generator(params['train_path'], params))
print(text)
Reading ../data/train.txt
[0, 102, 621, 15, 10, 5253, 13855, 81, 10, 3187, 159, 16847, 2, 102, 621, 16, 1058, 39, 5253, 13, 10, 1465, 2]
In [8]:
class RobertaFinetune(tf.keras.Model):
  def __init__(self, params):
    super(RobertaFinetune, self).__init__()
    self.bert = TFRobertaModel.from_pretrained(params['pretrain_path'],
                                               trainable = True)
    self.drop_1 = tf.keras.layers.Dropout(.1)
    self.fc = tf.keras.layers.Dense(300, tf.nn.swish, name='down_stream/fc')
    self.drop_2 = tf.keras.layers.Dropout(.1)
    self.out = tf.keras.layers.Dense(3, name='down_stream/out')

  def call(self, bert_inputs, training):
    bert_inputs = [tf.cast(inp, tf.int32) for inp in bert_inputs]
    x = self.bert(bert_inputs, training=training)[1]
    x = self.drop_1(x, training=training)
    x = self.fc(x)
    x = self.drop_2(x, training=training)
    x = self.out(x)
    return x
In [ ]:
model = RobertaFinetune(params)
model.build([[None, None], [None, None], [None, None]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

step_size = 2 * params['num_samples'] // params['batch_size']
decay_lr = tfa.optimizers.Triangular2CyclicalLearningRate(
  initial_learning_rate = params['init_lr'],
  maximal_learning_rate = params['max_lr'],
  step_size = step_size,)
optim = tf.optimizers.Adam(params['init_lr'])
global_step = 0

best_acc = .0
count = 0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

while True:
  # TRAINING
  for (text, labels) in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      masks = tf.cast(tf.math.not_equal(text, 1), tf.int32)
      logits = model([text, masks], training=True)
      loss = tf.compat.v1.losses.softmax_cross_entropy(
        tf.one_hot(labels, 3, dtype=tf.float32),
        logits = logits,
        label_smoothing = .2,)
      
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, 5.)
    optim.apply_gradients(zip(grads, model.trainable_variables))
    
    if global_step % 100 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  m = tf.keras.metrics.Accuracy()

  for (text, labels) in dataset(is_training=False, params=params):
    masks = tf.cast(tf.math.not_equal(text, 1), tf.int32)
    logits = model([text, masks], training=False)
    m.update_state(y_true=labels, y_pred=tf.argmax(logits, -1))

  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))

  if acc > best_acc:
    best_acc = acc
    # you can save model here
    count = 0
  else:
    count += 1
  logger.info("Best Accuracy: {:.3f}".format(best_acc))

  if count == params['num_patience']:
    print(params['num_patience'], "times not improve the best result, therefore stop training")
    break
Some weights of the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']
- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
All the weights of TFRobertaModel were initialized from the model checkpoint at roberta-base.
If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFRobertaModel for predictions without further training.
[('tf_roberta_model/roberta/encoder/layer_._0/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._0/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._0/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._0/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._1/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._1/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._1/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._2/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._2/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._2/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._3/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._3/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._3/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._4/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._4/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._4/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._5/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._5/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._5/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._6/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._6/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._6/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._7/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._7/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._7/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._8/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._8/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._8/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._9/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._9/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._9/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._10/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._10/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._10/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_roberta_model/roberta/encoder/layer_._11/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_roberta_model/roberta/encoder/layer_._11/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/output/dense/bias:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/encoder/layer_._11/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_roberta_model/roberta/pooler/dense/kernel:0', TensorShape([768, 768])),
 ('tf_roberta_model/roberta/pooler/dense/bias:0', TensorShape([768])),
 ('tf_roberta_model/roberta/embeddings/word_embeddings/weight:0',
  TensorShape([50265, 768])),
 ('tf_roberta_model/roberta/embeddings/position_embeddings/embeddings:0',
  TensorShape([514, 768])),
 ('tf_roberta_model/roberta/embeddings/token_type_embeddings/embeddings:0',
  TensorShape([1, 768])),
 ('tf_roberta_model/roberta/embeddings/LayerNorm/gamma:0', TensorShape([768])),
 ('tf_roberta_model/roberta/embeddings/LayerNorm/beta:0', TensorShape([768])),
 ('down_stream/fc/kernel:0', TensorShape([768, 300])),
 ('down_stream/fc/bias:0', TensorShape([300])),
 ('down_stream/out/kernel:0', TensorShape([300, 3])),
 ('down_stream/out/bias:0', TensorShape([3]))]
Reading ../data/train.txt
INFO:tensorflow:Step 0 | Loss: 1.1535 | Spent: 76.9 secs | LR: 0.000010
INFO:tensorflow:Step 100 | Loss: 0.8667 | Spent: 36.5 secs | LR: 0.000010
INFO:tensorflow:Step 200 | Loss: 0.8522 | Spent: 37.3 secs | LR: 0.000010
INFO:tensorflow:Step 300 | Loss: 0.7046 | Spent: 36.7 secs | LR: 0.000010
INFO:tensorflow:Step 400 | Loss: 0.9072 | Spent: 37.1 secs | LR: 0.000010
INFO:tensorflow:Step 500 | Loss: 0.6932 | Spent: 36.4 secs | LR: 0.000010
INFO:tensorflow:Step 600 | Loss: 0.6520 | Spent: 36.6 secs | LR: 0.000010
INFO:tensorflow:Step 700 | Loss: 0.7399 | Spent: 36.3 secs | LR: 0.000010
INFO:tensorflow:Step 800 | Loss: 0.7020 | Spent: 37.0 secs | LR: 0.000010
INFO:tensorflow:Step 900 | Loss: 0.7783 | Spent: 36.5 secs | LR: 0.000011
INFO:tensorflow:Step 1000 | Loss: 0.7514 | Spent: 36.3 secs | LR: 0.000011
INFO:tensorflow:Step 1100 | Loss: 0.6908 | Spent: 37.4 secs | LR: 0.000011
INFO:tensorflow:Step 1200 | Loss: 0.7328 | Spent: 36.3 secs | LR: 0.000011
INFO:tensorflow:Step 1300 | Loss: 0.7299 | Spent: 36.7 secs | LR: 0.000011
INFO:tensorflow:Step 1400 | Loss: 0.7026 | Spent: 36.5 secs | LR: 0.000011
INFO:tensorflow:Step 1500 | Loss: 0.7290 | Spent: 36.4 secs | LR: 0.000011
INFO:tensorflow:Step 1600 | Loss: 0.7935 | Spent: 36.6 secs | LR: 0.000011
INFO:tensorflow:Step 1700 | Loss: 0.7325 | Spent: 36.4 secs | LR: 0.000011
INFO:tensorflow:Step 1800 | Loss: 0.6795 | Spent: 36.8 secs | LR: 0.000011
INFO:tensorflow:Step 1900 | Loss: 0.6293 | Spent: 36.8 secs | LR: 0.000011
INFO:tensorflow:Step 2000 | Loss: 0.6093 | Spent: 36.4 secs | LR: 0.000011
INFO:tensorflow:Step 2100 | Loss: 0.6671 | Spent: 36.1 secs | LR: 0.000011
INFO:tensorflow:Step 2200 | Loss: 0.7561 | Spent: 36.4 secs | LR: 0.000011
INFO:tensorflow:Step 2300 | Loss: 0.7881 | Spent: 36.3 secs | LR: 0.000011
INFO:tensorflow:Step 2400 | Loss: 0.6097 | Spent: 36.1 secs | LR: 0.000011
INFO:tensorflow:Step 2500 | Loss: 0.6882 | Spent: 36.1 secs | LR: 0.000011
INFO:tensorflow:Step 2600 | Loss: 0.6718 | Spent: 36.5 secs | LR: 0.000012
INFO:tensorflow:Step 2700 | Loss: 0.6784 | Spent: 36.2 secs | LR: 0.000012
INFO:tensorflow:Step 2800 | Loss: 0.7170 | Spent: 36.7 secs | LR: 0.000012
INFO:tensorflow:Step 2900 | Loss: 0.6025 | Spent: 36.1 secs | LR: 0.000012
INFO:tensorflow:Step 3000 | Loss: 0.6875 | Spent: 36.0 secs | LR: 0.000012
INFO:tensorflow:Step 3100 | Loss: 0.7873 | Spent: 35.8 secs | LR: 0.000012
INFO:tensorflow:Step 3200 | Loss: 0.7145 | Spent: 36.0 secs | LR: 0.000012
INFO:tensorflow:Step 3300 | Loss: 0.6526 | Spent: 36.0 secs | LR: 0.000012
INFO:tensorflow:Step 3400 | Loss: 0.6526 | Spent: 36.2 secs | LR: 0.000012
INFO:tensorflow:Step 3500 | Loss: 0.6748 | Spent: 36.3 secs | LR: 0.000012
INFO:tensorflow:Step 3600 | Loss: 0.6353 | Spent: 36.1 secs | LR: 0.000012
INFO:tensorflow:Step 3700 | Loss: 0.7318 | Spent: 36.2 secs | LR: 0.000012
INFO:tensorflow:Step 3800 | Loss: 0.6722 | Spent: 36.1 secs | LR: 0.000012
INFO:tensorflow:Step 3900 | Loss: 0.7570 | Spent: 36.0 secs | LR: 0.000012
INFO:tensorflow:Step 4000 | Loss: 0.6248 | Spent: 36.1 secs | LR: 0.000012
INFO:tensorflow:Step 4100 | Loss: 0.7311 | Spent: 36.2 secs | LR: 0.000012
INFO:tensorflow:Step 4200 | Loss: 0.7394 | Spent: 36.3 secs | LR: 0.000012
INFO:tensorflow:Step 4300 | Loss: 0.7618 | Spent: 35.8 secs | LR: 0.000013
INFO:tensorflow:Step 4400 | Loss: 0.5799 | Spent: 35.9 secs | LR: 0.000013
INFO:tensorflow:Step 4500 | Loss: 0.6132 | Spent: 36.6 secs | LR: 0.000013
INFO:tensorflow:Step 4600 | Loss: 0.6764 | Spent: 36.2 secs | LR: 0.000013
INFO:tensorflow:Step 4700 | Loss: 0.6403 | Spent: 36.0 secs | LR: 0.000013
INFO:tensorflow:Step 4800 | Loss: 0.7497 | Spent: 36.1 secs | LR: 0.000013
INFO:tensorflow:Step 4900 | Loss: 0.6660 | Spent: 36.1 secs | LR: 0.000013
INFO:tensorflow:Step 5000 | Loss: 0.6497 | Spent: 36.0 secs | LR: 0.000013
INFO:tensorflow:Step 5100 | Loss: 0.6785 | Spent: 35.9 secs | LR: 0.000013
INFO:tensorflow:Step 5200 | Loss: 0.6925 | Spent: 35.9 secs | LR: 0.000013
INFO:tensorflow:Step 5300 | Loss: 0.8134 | Spent: 35.9 secs | LR: 0.000013
INFO:tensorflow:Step 5400 | Loss: 0.5745 | Spent: 36.4 secs | LR: 0.000013
INFO:tensorflow:Step 5500 | Loss: 0.5405 | Spent: 36.6 secs | LR: 0.000013
INFO:tensorflow:Step 5600 | Loss: 0.5624 | Spent: 36.1 secs | LR: 0.000013
INFO:tensorflow:Step 5700 | Loss: 0.6496 | Spent: 36.1 secs | LR: 0.000013
INFO:tensorflow:Step 5800 | Loss: 0.6325 | Spent: 36.3 secs | LR: 0.000013
INFO:tensorflow:Step 5900 | Loss: 0.6818 | Spent: 35.9 secs | LR: 0.000013
INFO:tensorflow:Step 6000 | Loss: 0.5762 | Spent: 36.2 secs | LR: 0.000013
INFO:tensorflow:Step 6100 | Loss: 0.5542 | Spent: 36.1 secs | LR: 0.000014
INFO:tensorflow:Step 6200 | Loss: 0.5789 | Spent: 36.5 secs | LR: 0.000014
INFO:tensorflow:Step 6300 | Loss: 0.7495 | Spent: 36.1 secs | LR: 0.000014
INFO:tensorflow:Step 6400 | Loss: 0.7469 | Spent: 36.0 secs | LR: 0.000014
INFO:tensorflow:Step 6500 | Loss: 0.7421 | Spent: 35.7 secs | LR: 0.000014
INFO:tensorflow:Step 6600 | Loss: 0.6006 | Spent: 36.1 secs | LR: 0.000014
INFO:tensorflow:Step 6700 | Loss: 0.5888 | Spent: 36.2 secs | LR: 0.000014
INFO:tensorflow:Step 6800 | Loss: 0.6240 | Spent: 36.3 secs | LR: 0.000014
INFO:tensorflow:Step 6900 | Loss: 0.6383 | Spent: 36.0 secs | LR: 0.000014
INFO:tensorflow:Step 7000 | Loss: 0.6434 | Spent: 35.9 secs | LR: 0.000014
INFO:tensorflow:Step 7100 | Loss: 0.6866 | Spent: 36.5 secs | LR: 0.000014
INFO:tensorflow:Step 7200 | Loss: 0.8448 | Spent: 35.6 secs | LR: 0.000014
INFO:tensorflow:Step 7300 | Loss: 0.7114 | Spent: 36.2 secs | LR: 0.000014
INFO:tensorflow:Step 7400 | Loss: 0.5335 | Spent: 36.0 secs | LR: 0.000014
INFO:tensorflow:Step 7500 | Loss: 0.5897 | Spent: 36.3 secs | LR: 0.000014
INFO:tensorflow:Step 7600 | Loss: 0.7701 | Spent: 35.9 secs | LR: 0.000014
INFO:tensorflow:Step 7700 | Loss: 0.7354 | Spent: 35.9 secs | LR: 0.000014
INFO:tensorflow:Step 7800 | Loss: 0.5856 | Spent: 36.0 secs | LR: 0.000015
INFO:tensorflow:Step 7900 | Loss: 0.6013 | Spent: 35.9 secs | LR: 0.000015
INFO:tensorflow:Step 8000 | Loss: 0.6154 | Spent: 36.4 secs | LR: 0.000015
INFO:tensorflow:Step 8100 | Loss: 0.6289 | Spent: 36.3 secs | LR: 0.000015
INFO:tensorflow:Step 8200 | Loss: 0.5441 | Spent: 36.2 secs | LR: 0.000015
INFO:tensorflow:Step 8300 | Loss: 0.6194 | Spent: 35.9 secs | LR: 0.000015
INFO:tensorflow:Step 8400 | Loss: 0.5796 | Spent: 36.0 secs | LR: 0.000015
INFO:tensorflow:Step 8500 | Loss: 0.7505 | Spent: 36.4 secs | LR: 0.000015
INFO:tensorflow:Step 8600 | Loss: 0.7566 | Spent: 36.1 secs | LR: 0.000015
INFO:tensorflow:Step 8700 | Loss: 0.6347 | Spent: 35.8 secs | LR: 0.000015
INFO:tensorflow:Step 8800 | Loss: 0.5726 | Spent: 36.1 secs | LR: 0.000015
INFO:tensorflow:Step 8900 | Loss: 0.7123 | Spent: 35.6 secs | LR: 0.000015
INFO:tensorflow:Step 9000 | Loss: 0.7463 | Spent: 36.0 secs | LR: 0.000015
INFO:tensorflow:Step 9100 | Loss: 0.6527 | Spent: 35.9 secs | LR: 0.000015
INFO:tensorflow:Step 9200 | Loss: 0.6583 | Spent: 36.5 secs | LR: 0.000015
INFO:tensorflow:Step 9300 | Loss: 0.6457 | Spent: 36.0 secs | LR: 0.000015
INFO:tensorflow:Step 9400 | Loss: 0.5548 | Spent: 36.0 secs | LR: 0.000015
INFO:tensorflow:Step 9500 | Loss: 0.6643 | Spent: 36.3 secs | LR: 0.000016
INFO:tensorflow:Step 9600 | Loss: 0.6653 | Spent: 36.3 secs | LR: 0.000016
INFO:tensorflow:Step 9700 | Loss: 0.7518 | Spent: 36.5 secs | LR: 0.000016
INFO:tensorflow:Step 9800 | Loss: 0.6946 | Spent: 35.9 secs | LR: 0.000016
INFO:tensorflow:Step 9900 | Loss: 0.6245 | Spent: 36.0 secs | LR: 0.000016
INFO:tensorflow:Step 10000 | Loss: 0.6687 | Spent: 36.2 secs | LR: 0.000016
INFO:tensorflow:Step 10100 | Loss: 0.5551 | Spent: 36.4 secs | LR: 0.000016
INFO:tensorflow:Step 10200 | Loss: 0.9045 | Spent: 36.1 secs | LR: 0.000016
INFO:tensorflow:Step 10300 | Loss: 0.7007 | Spent: 36.1 secs | LR: 0.000016
INFO:tensorflow:Step 10400 | Loss: 0.7927 | Spent: 36.1 secs | LR: 0.000016
INFO:tensorflow:Step 10500 | Loss: 0.6235 | Spent: 36.6 secs | LR: 0.000016
INFO:tensorflow:Step 10600 | Loss: 0.6437 | Spent: 36.4 secs | LR: 0.000016
INFO:tensorflow:Step 10700 | Loss: 0.6400 | Spent: 35.8 secs | LR: 0.000016
INFO:tensorflow:Step 10800 | Loss: 0.5957 | Spent: 35.9 secs | LR: 0.000016
INFO:tensorflow:Step 10900 | Loss: 0.7246 | Spent: 36.4 secs | LR: 0.000016
INFO:tensorflow:Step 11000 | Loss: 0.7394 | Spent: 35.2 secs | LR: 0.000016
INFO:tensorflow:Step 11100 | Loss: 0.6990 | Spent: 35.0 secs | LR: 0.000016
INFO:tensorflow:Step 11200 | Loss: 0.6299 | Spent: 34.6 secs | LR: 0.000017
INFO:tensorflow:Step 11300 | Loss: 0.5928 | Spent: 35.1 secs | LR: 0.000017
INFO:tensorflow:Step 11400 | Loss: 0.6212 | Spent: 35.0 secs | LR: 0.000017
INFO:tensorflow:Step 11500 | Loss: 0.7383 | Spent: 35.2 secs | LR: 0.000017
INFO:tensorflow:Step 11600 | Loss: 0.6021 | Spent: 34.5 secs | LR: 0.000017
INFO:tensorflow:Step 11700 | Loss: 0.7435 | Spent: 34.8 secs | LR: 0.000017
INFO:tensorflow:Step 11800 | Loss: 0.6795 | Spent: 35.0 secs | LR: 0.000017
INFO:tensorflow:Step 11900 | Loss: 0.6893 | Spent: 34.7 secs | LR: 0.000017
INFO:tensorflow:Step 12000 | Loss: 0.7329 | Spent: 34.6 secs | LR: 0.000017
INFO:tensorflow:Step 12100 | Loss: 0.7617 | Spent: 35.2 secs | LR: 0.000017
INFO:tensorflow:Step 12200 | Loss: 0.7477 | Spent: 35.4 secs | LR: 0.000017
INFO:tensorflow:Step 12300 | Loss: 0.6724 | Spent: 35.3 secs | LR: 0.000017
INFO:tensorflow:Step 12400 | Loss: 0.5404 | Spent: 35.0 secs | LR: 0.000017
INFO:tensorflow:Step 12500 | Loss: 0.7042 | Spent: 35.2 secs | LR: 0.000017
INFO:tensorflow:Step 12600 | Loss: 0.7501 | Spent: 34.9 secs | LR: 0.000017
INFO:tensorflow:Step 12700 | Loss: 0.6766 | Spent: 35.2 secs | LR: 0.000017
INFO:tensorflow:Step 12800 | Loss: 0.6335 | Spent: 34.6 secs | LR: 0.000017
INFO:tensorflow:Step 12900 | Loss: 0.6139 | Spent: 35.0 secs | LR: 0.000018
INFO:tensorflow:Step 13000 | Loss: 0.5314 | Spent: 34.9 secs | LR: 0.000018
INFO:tensorflow:Step 13100 | Loss: 0.7458 | Spent: 34.8 secs | LR: 0.000018
INFO:tensorflow:Step 13200 | Loss: 0.5604 | Spent: 35.1 secs | LR: 0.000018
INFO:tensorflow:Step 13300 | Loss: 0.5718 | Spent: 34.6 secs | LR: 0.000018
INFO:tensorflow:Step 13400 | Loss: 0.6090 | Spent: 35.1 secs | LR: 0.000018
INFO:tensorflow:Step 13500 | Loss: 0.7421 | Spent: 35.1 secs | LR: 0.000018
INFO:tensorflow:Step 13600 | Loss: 0.6973 | Spent: 35.1 secs | LR: 0.000018
INFO:tensorflow:Step 13700 | Loss: 0.6859 | Spent: 34.8 secs | LR: 0.000018
INFO:tensorflow:Step 13800 | Loss: 0.6863 | Spent: 35.2 secs | LR: 0.000018
INFO:tensorflow:Step 13900 | Loss: 0.6891 | Spent: 35.0 secs | LR: 0.000018
INFO:tensorflow:Step 14000 | Loss: 0.7249 | Spent: 34.9 secs | LR: 0.000018
INFO:tensorflow:Step 14100 | Loss: 0.7013 | Spent: 35.2 secs | LR: 0.000018
INFO:tensorflow:Step 14200 | Loss: 0.6750 | Spent: 34.6 secs | LR: 0.000018
INFO:tensorflow:Step 14300 | Loss: 0.7675 | Spent: 34.8 secs | LR: 0.000018
INFO:tensorflow:Step 14400 | Loss: 0.6752 | Spent: 35.1 secs | LR: 0.000018
INFO:tensorflow:Step 14500 | Loss: 0.7301 | Spent: 34.7 secs | LR: 0.000018
INFO:tensorflow:Step 14600 | Loss: 0.6811 | Spent: 34.9 secs | LR: 0.000018
INFO:tensorflow:Step 14700 | Loss: 0.7990 | Spent: 34.9 secs | LR: 0.000019
INFO:tensorflow:Step 14800 | Loss: 0.7037 | Spent: 35.1 secs | LR: 0.000019
INFO:tensorflow:Step 14900 | Loss: 0.6216 | Spent: 34.8 secs | LR: 0.000019
INFO:tensorflow:Step 15000 | Loss: 0.5904 | Spent: 35.0 secs | LR: 0.000019
INFO:tensorflow:Step 15100 | Loss: 0.6760 | Spent: 34.5 secs | LR: 0.000019
INFO:tensorflow:Step 15200 | Loss: 0.7100 | Spent: 34.6 secs | LR: 0.000019
INFO:tensorflow:Step 15300 | Loss: 0.6568 | Spent: 34.3 secs | LR: 0.000019
INFO:tensorflow:Step 15400 | Loss: 0.5335 | Spent: 34.6 secs | LR: 0.000019
INFO:tensorflow:Step 15500 | Loss: 0.5187 | Spent: 34.8 secs | LR: 0.000019
INFO:tensorflow:Step 15600 | Loss: 0.6556 | Spent: 34.8 secs | LR: 0.000019
INFO:tensorflow:Step 15700 | Loss: 0.6405 | Spent: 35.0 secs | LR: 0.000019
INFO:tensorflow:Step 15800 | Loss: 0.6511 | Spent: 35.0 secs | LR: 0.000019
INFO:tensorflow:Step 15900 | Loss: 0.6566 | Spent: 35.3 secs | LR: 0.000019
INFO:tensorflow:Step 16000 | Loss: 0.7590 | Spent: 35.0 secs | LR: 0.000019
INFO:tensorflow:Step 16100 | Loss: 0.6868 | Spent: 35.3 secs | LR: 0.000019
INFO:tensorflow:Step 16200 | Loss: 0.6564 | Spent: 35.2 secs | LR: 0.000019
INFO:tensorflow:Step 16300 | Loss: 0.6256 | Spent: 35.0 secs | LR: 0.000019
INFO:tensorflow:Step 16400 | Loss: 0.5951 | Spent: 34.9 secs | LR: 0.000020
INFO:tensorflow:Step 16500 | Loss: 0.7261 | Spent: 34.8 secs | LR: 0.000020
INFO:tensorflow:Step 16600 | Loss: 0.6888 | Spent: 35.0 secs | LR: 0.000020
INFO:tensorflow:Step 16700 | Loss: 0.5659 | Spent: 35.1 secs | LR: 0.000020
INFO:tensorflow:Step 16800 | Loss: 0.6928 | Spent: 35.3 secs | LR: 0.000020
INFO:tensorflow:Step 16900 | Loss: 0.6513 | Spent: 34.9 secs | LR: 0.000020
INFO:tensorflow:Step 17000 | Loss: 0.6217 | Spent: 34.8 secs | LR: 0.000020
INFO:tensorflow:Step 17100 | Loss: 0.5782 | Spent: 35.1 secs | LR: 0.000020
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.898
INFO:tensorflow:Best Accuracy: 0.898
Reading ../data/train.txt
INFO:tensorflow:Step 17200 | Loss: 0.6284 | Spent: 148.6 secs | LR: 0.000020
INFO:tensorflow:Step 17300 | Loss: 0.5945 | Spent: 36.8 secs | LR: 0.000020
INFO:tensorflow:Step 17400 | Loss: 0.8344 | Spent: 36.6 secs | LR: 0.000020
INFO:tensorflow:Step 17500 | Loss: 0.7256 | Spent: 36.9 secs | LR: 0.000020
INFO:tensorflow:Step 17600 | Loss: 0.7531 | Spent: 36.1 secs | LR: 0.000020
INFO:tensorflow:Step 17700 | Loss: 0.6010 | Spent: 36.3 secs | LR: 0.000020
INFO:tensorflow:Step 17800 | Loss: 0.5837 | Spent: 36.5 secs | LR: 0.000020
INFO:tensorflow:Step 17900 | Loss: 0.6832 | Spent: 36.3 secs | LR: 0.000020
INFO:tensorflow:Step 18000 | Loss: 0.5856 | Spent: 36.3 secs | LR: 0.000020
INFO:tensorflow:Step 18100 | Loss: 0.7409 | Spent: 35.9 secs | LR: 0.000021
INFO:tensorflow:Step 18200 | Loss: 0.6120 | Spent: 36.8 secs | LR: 0.000021
INFO:tensorflow:Step 18300 | Loss: 0.6741 | Spent: 36.2 secs | LR: 0.000021
INFO:tensorflow:Step 18400 | Loss: 0.7685 | Spent: 36.3 secs | LR: 0.000021
INFO:tensorflow:Step 18500 | Loss: 0.5498 | Spent: 36.3 secs | LR: 0.000021
INFO:tensorflow:Step 18600 | Loss: 0.6368 | Spent: 36.6 secs | LR: 0.000021
INFO:tensorflow:Step 18700 | Loss: 0.5664 | Spent: 36.2 secs | LR: 0.000021
INFO:tensorflow:Step 18800 | Loss: 0.7140 | Spent: 36.5 secs | LR: 0.000021
INFO:tensorflow:Step 18900 | Loss: 0.5498 | Spent: 36.4 secs | LR: 0.000021
INFO:tensorflow:Step 19000 | Loss: 0.6254 | Spent: 36.7 secs | LR: 0.000021
INFO:tensorflow:Step 19100 | Loss: 0.5655 | Spent: 36.3 secs | LR: 0.000021
INFO:tensorflow:Step 19200 | Loss: 0.7957 | Spent: 36.7 secs | LR: 0.000021
INFO:tensorflow:Step 19300 | Loss: 0.7046 | Spent: 35.8 secs | LR: 0.000021
INFO:tensorflow:Step 19400 | Loss: 0.6363 | Spent: 36.4 secs | LR: 0.000021
INFO:tensorflow:Step 19500 | Loss: 0.7010 | Spent: 35.8 secs | LR: 0.000021
INFO:tensorflow:Step 19600 | Loss: 0.6935 | Spent: 35.9 secs | LR: 0.000021
INFO:tensorflow:Step 19700 | Loss: 0.8128 | Spent: 35.8 secs | LR: 0.000021
INFO:tensorflow:Step 19800 | Loss: 0.6288 | Spent: 35.9 secs | LR: 0.000022
INFO:tensorflow:Step 19900 | Loss: 0.5125 | Spent: 36.5 secs | LR: 0.000022
INFO:tensorflow:Step 20000 | Loss: 0.6121 | Spent: 36.1 secs | LR: 0.000022
INFO:tensorflow:Step 20100 | Loss: 0.5312 | Spent: 36.4 secs | LR: 0.000022
INFO:tensorflow:Step 20200 | Loss: 0.5587 | Spent: 35.9 secs | LR: 0.000022
INFO:tensorflow:Step 20300 | Loss: 0.6255 | Spent: 35.9 secs | LR: 0.000022
INFO:tensorflow:Step 20400 | Loss: 0.6362 | Spent: 36.0 secs | LR: 0.000022
INFO:tensorflow:Step 20500 | Loss: 0.6320 | Spent: 36.2 secs | LR: 0.000022
INFO:tensorflow:Step 20600 | Loss: 0.5934 | Spent: 36.1 secs | LR: 0.000022
INFO:tensorflow:Step 20700 | Loss: 0.6265 | Spent: 36.5 secs | LR: 0.000022
INFO:tensorflow:Step 20800 | Loss: 0.5749 | Spent: 36.3 secs | LR: 0.000022
INFO:tensorflow:Step 20900 | Loss: 0.6314 | Spent: 35.6 secs | LR: 0.000022
INFO:tensorflow:Step 21000 | Loss: 0.6237 | Spent: 36.0 secs | LR: 0.000022
INFO:tensorflow:Step 21100 | Loss: 0.7446 | Spent: 35.9 secs | LR: 0.000022
INFO:tensorflow:Step 21200 | Loss: 0.6136 | Spent: 35.6 secs | LR: 0.000022
INFO:tensorflow:Step 21300 | Loss: 0.6625 | Spent: 35.9 secs | LR: 0.000022
INFO:tensorflow:Step 21400 | Loss: 0.7150 | Spent: 35.8 secs | LR: 0.000022
INFO:tensorflow:Step 21500 | Loss: 0.6324 | Spent: 36.0 secs | LR: 0.000023
INFO:tensorflow:Step 21600 | Loss: 0.6273 | Spent: 36.1 secs | LR: 0.000023
INFO:tensorflow:Step 21700 | Loss: 0.6299 | Spent: 35.9 secs | LR: 0.000023
INFO:tensorflow:Step 21800 | Loss: 0.6604 | Spent: 36.2 secs | LR: 0.000023
INFO:tensorflow:Step 21900 | Loss: 0.8297 | Spent: 35.9 secs | LR: 0.000023
INFO:tensorflow:Step 22000 | Loss: 0.6024 | Spent: 35.5 secs | LR: 0.000023
INFO:tensorflow:Step 22100 | Loss: 0.8241 | Spent: 36.0 secs | LR: 0.000023
INFO:tensorflow:Step 22200 | Loss: 0.5643 | Spent: 35.9 secs | LR: 0.000023
INFO:tensorflow:Step 22300 | Loss: 0.7466 | Spent: 36.3 secs | LR: 0.000023
INFO:tensorflow:Step 22400 | Loss: 0.7112 | Spent: 36.7 secs | LR: 0.000023
INFO:tensorflow:Step 22500 | Loss: 0.5641 | Spent: 37.0 secs | LR: 0.000023
INFO:tensorflow:Step 22600 | Loss: 0.6136 | Spent: 36.3 secs | LR: 0.000023
INFO:tensorflow:Step 22700 | Loss: 0.6557 | Spent: 35.9 secs | LR: 0.000023
INFO:tensorflow:Step 22800 | Loss: 0.6434 | Spent: 36.0 secs | LR: 0.000023
INFO:tensorflow:Step 22900 | Loss: 0.5324 | Spent: 35.6 secs | LR: 0.000023
INFO:tensorflow:Step 23000 | Loss: 0.6538 | Spent: 36.0 secs | LR: 0.000023
INFO:tensorflow:Step 23100 | Loss: 0.6356 | Spent: 35.9 secs | LR: 0.000023
INFO:tensorflow:Step 23200 | Loss: 0.6736 | Spent: 35.9 secs | LR: 0.000023
INFO:tensorflow:Step 23300 | Loss: 0.5671 | Spent: 36.4 secs | LR: 0.000024
INFO:tensorflow:Step 23400 | Loss: 0.7739 | Spent: 35.7 secs | LR: 0.000024
INFO:tensorflow:Step 23500 | Loss: 0.6043 | Spent: 36.1 secs | LR: 0.000024
INFO:tensorflow:Step 23600 | Loss: 0.5007 | Spent: 36.0 secs | LR: 0.000024
INFO:tensorflow:Step 23700 | Loss: 0.6392 | Spent: 36.1 secs | LR: 0.000024
INFO:tensorflow:Step 23800 | Loss: 0.5728 | Spent: 36.3 secs | LR: 0.000024
INFO:tensorflow:Step 23900 | Loss: 0.6268 | Spent: 36.0 secs | LR: 0.000024
INFO:tensorflow:Step 24000 | Loss: 0.7100 | Spent: 36.2 secs | LR: 0.000024
INFO:tensorflow:Step 24100 | Loss: 0.5983 | Spent: 36.3 secs | LR: 0.000024
INFO:tensorflow:Step 24200 | Loss: 0.6790 | Spent: 36.3 secs | LR: 0.000024
INFO:tensorflow:Step 24300 | Loss: 0.6525 | Spent: 36.2 secs | LR: 0.000024
INFO:tensorflow:Step 24400 | Loss: 0.6062 | Spent: 36.1 secs | LR: 0.000024
INFO:tensorflow:Step 24500 | Loss: 0.6159 | Spent: 36.1 secs | LR: 0.000024
INFO:tensorflow:Step 24600 | Loss: 0.6187 | Spent: 36.2 secs | LR: 0.000024
INFO:tensorflow:Step 24700 | Loss: 0.6719 | Spent: 36.1 secs | LR: 0.000024
INFO:tensorflow:Step 24800 | Loss: 0.5821 | Spent: 36.5 secs | LR: 0.000024
INFO:tensorflow:Step 24900 | Loss: 0.6234 | Spent: 36.3 secs | LR: 0.000024
INFO:tensorflow:Step 25000 | Loss: 0.5498 | Spent: 36.7 secs | LR: 0.000025
INFO:tensorflow:Step 25100 | Loss: 0.6347 | Spent: 36.2 secs | LR: 0.000025
INFO:tensorflow:Step 25200 | Loss: 0.7919 | Spent: 36.6 secs | LR: 0.000025
INFO:tensorflow:Step 25300 | Loss: 0.6517 | Spent: 36.4 secs | LR: 0.000025
INFO:tensorflow:Step 25400 | Loss: 0.5811 | Spent: 36.9 secs | LR: 0.000025
INFO:tensorflow:Step 25500 | Loss: 0.6176 | Spent: 37.0 secs | LR: 0.000025
INFO:tensorflow:Step 25600 | Loss: 0.6125 | Spent: 36.7 secs | LR: 0.000025
INFO:tensorflow:Step 25700 | Loss: 0.5659 | Spent: 36.7 secs | LR: 0.000025
INFO:tensorflow:Step 25800 | Loss: 0.7494 | Spent: 36.6 secs | LR: 0.000025
INFO:tensorflow:Step 25900 | Loss: 0.6795 | Spent: 37.1 secs | LR: 0.000025
INFO:tensorflow:Step 26000 | Loss: 0.5834 | Spent: 36.9 secs | LR: 0.000025
INFO:tensorflow:Step 26100 | Loss: 0.6109 | Spent: 37.0 secs | LR: 0.000025
INFO:tensorflow:Step 26200 | Loss: 0.6673 | Spent: 37.0 secs | LR: 0.000025
INFO:tensorflow:Step 26300 | Loss: 0.7381 | Spent: 37.0 secs | LR: 0.000025
INFO:tensorflow:Step 26400 | Loss: 0.8108 | Spent: 37.1 secs | LR: 0.000025
INFO:tensorflow:Step 26500 | Loss: 0.5970 | Spent: 37.2 secs | LR: 0.000025
INFO:tensorflow:Step 26600 | Loss: 0.5565 | Spent: 37.2 secs | LR: 0.000025
INFO:tensorflow:Step 26700 | Loss: 0.6420 | Spent: 37.4 secs | LR: 0.000026
INFO:tensorflow:Step 26800 | Loss: 0.7401 | Spent: 37.4 secs | LR: 0.000026
INFO:tensorflow:Step 26900 | Loss: 0.5347 | Spent: 37.4 secs | LR: 0.000026
INFO:tensorflow:Step 27000 | Loss: 0.7396 | Spent: 37.1 secs | LR: 0.000026
INFO:tensorflow:Step 27100 | Loss: 0.7164 | Spent: 37.0 secs | LR: 0.000026
INFO:tensorflow:Step 27200 | Loss: 0.5584 | Spent: 37.1 secs | LR: 0.000026
INFO:tensorflow:Step 27300 | Loss: 0.5915 | Spent: 36.8 secs | LR: 0.000026
INFO:tensorflow:Step 27400 | Loss: 0.5600 | Spent: 37.1 secs | LR: 0.000026
INFO:tensorflow:Step 27500 | Loss: 0.5999 | Spent: 36.6 secs | LR: 0.000026
INFO:tensorflow:Step 27600 | Loss: 0.6034 | Spent: 37.6 secs | LR: 0.000026
INFO:tensorflow:Step 27700 | Loss: 0.6470 | Spent: 36.7 secs | LR: 0.000026
INFO:tensorflow:Step 27800 | Loss: 0.6340 | Spent: 37.1 secs | LR: 0.000026
INFO:tensorflow:Step 27900 | Loss: 0.6601 | Spent: 36.8 secs | LR: 0.000026
INFO:tensorflow:Step 28000 | Loss: 0.6226 | Spent: 37.4 secs | LR: 0.000026
INFO:tensorflow:Step 28100 | Loss: 0.5404 | Spent: 36.5 secs | LR: 0.000026
INFO:tensorflow:Step 28200 | Loss: 0.6931 | Spent: 35.4 secs | LR: 0.000026
INFO:tensorflow:Step 28300 | Loss: 0.6223 | Spent: 35.4 secs | LR: 0.000026
INFO:tensorflow:Step 28400 | Loss: 0.6191 | Spent: 35.7 secs | LR: 0.000027
INFO:tensorflow:Step 28500 | Loss: 0.6812 | Spent: 35.5 secs | LR: 0.000027
INFO:tensorflow:Step 28600 | Loss: 0.6685 | Spent: 35.5 secs | LR: 0.000027
INFO:tensorflow:Step 28700 | Loss: 0.5963 | Spent: 35.1 secs | LR: 0.000027
INFO:tensorflow:Step 28800 | Loss: 0.7692 | Spent: 35.4 secs | LR: 0.000027
INFO:tensorflow:Step 28900 | Loss: 0.5845 | Spent: 35.6 secs | LR: 0.000027
INFO:tensorflow:Step 29000 | Loss: 0.7441 | Spent: 35.5 secs | LR: 0.000027
INFO:tensorflow:Step 29100 | Loss: 0.5437 | Spent: 35.3 secs | LR: 0.000027
INFO:tensorflow:Step 29200 | Loss: 0.5916 | Spent: 35.6 secs | LR: 0.000027
INFO:tensorflow:Step 29300 | Loss: 0.6092 | Spent: 35.9 secs | LR: 0.000027
INFO:tensorflow:Step 29400 | Loss: 0.6819 | Spent: 35.7 secs | LR: 0.000027
INFO:tensorflow:Step 29500 | Loss: 0.5941 | Spent: 35.2 secs | LR: 0.000027
INFO:tensorflow:Step 29600 | Loss: 0.6314 | Spent: 35.2 secs | LR: 0.000027
INFO:tensorflow:Step 29700 | Loss: 0.5713 | Spent: 35.3 secs | LR: 0.000027
INFO:tensorflow:Step 29800 | Loss: 0.5639 | Spent: 34.8 secs | LR: 0.000027
INFO:tensorflow:Step 29900 | Loss: 0.6120 | Spent: 36.2 secs | LR: 0.000027
INFO:tensorflow:Step 30000 | Loss: 0.7558 | Spent: 35.2 secs | LR: 0.000027
INFO:tensorflow:Step 30100 | Loss: 0.7069 | Spent: 35.1 secs | LR: 0.000028
INFO:tensorflow:Step 30200 | Loss: 0.6200 | Spent: 35.3 secs | LR: 0.000028
INFO:tensorflow:Step 30300 | Loss: 0.6180 | Spent: 34.9 secs | LR: 0.000028
INFO:tensorflow:Step 30400 | Loss: 0.6648 | Spent: 35.0 secs | LR: 0.000028
INFO:tensorflow:Step 30500 | Loss: 0.6420 | Spent: 34.8 secs | LR: 0.000028
INFO:tensorflow:Step 30600 | Loss: 0.7007 | Spent: 34.5 secs | LR: 0.000028
INFO:tensorflow:Step 30700 | Loss: 0.6511 | Spent: 34.8 secs | LR: 0.000028
INFO:tensorflow:Step 30800 | Loss: 0.7487 | Spent: 34.7 secs | LR: 0.000028
INFO:tensorflow:Step 30900 | Loss: 0.6293 | Spent: 34.8 secs | LR: 0.000028
INFO:tensorflow:Step 31000 | Loss: 0.5401 | Spent: 34.9 secs | LR: 0.000028
INFO:tensorflow:Step 31100 | Loss: 0.6154 | Spent: 35.2 secs | LR: 0.000028
INFO:tensorflow:Step 31200 | Loss: 0.6067 | Spent: 35.3 secs | LR: 0.000028
INFO:tensorflow:Step 31300 | Loss: 0.6632 | Spent: 35.0 secs | LR: 0.000028
INFO:tensorflow:Step 31400 | Loss: 0.6056 | Spent: 35.3 secs | LR: 0.000028
INFO:tensorflow:Step 31500 | Loss: 0.5958 | Spent: 35.3 secs | LR: 0.000028
INFO:tensorflow:Step 31600 | Loss: 0.5916 | Spent: 34.9 secs | LR: 0.000028
INFO:tensorflow:Step 31700 | Loss: 0.6023 | Spent: 35.1 secs | LR: 0.000028
INFO:tensorflow:Step 31800 | Loss: 0.6483 | Spent: 35.0 secs | LR: 0.000028
INFO:tensorflow:Step 31900 | Loss: 0.7094 | Spent: 35.5 secs | LR: 0.000029
INFO:tensorflow:Step 32000 | Loss: 0.7499 | Spent: 35.3 secs | LR: 0.000029
INFO:tensorflow:Step 32100 | Loss: 0.6145 | Spent: 34.3 secs | LR: 0.000029
INFO:tensorflow:Step 32200 | Loss: 0.7288 | Spent: 34.8 secs | LR: 0.000029
INFO:tensorflow:Step 32300 | Loss: 0.6234 | Spent: 34.9 secs | LR: 0.000029
INFO:tensorflow:Step 32400 | Loss: 0.5480 | Spent: 35.0 secs | LR: 0.000029
INFO:tensorflow:Step 32500 | Loss: 0.5368 | Spent: 34.5 secs | LR: 0.000029
INFO:tensorflow:Step 32600 | Loss: 0.6982 | Spent: 34.9 secs | LR: 0.000029
INFO:tensorflow:Step 32700 | Loss: 0.7283 | Spent: 34.6 secs | LR: 0.000029
INFO:tensorflow:Step 32800 | Loss: 0.5769 | Spent: 34.9 secs | LR: 0.000029
INFO:tensorflow:Step 32900 | Loss: 0.6046 | Spent: 34.9 secs | LR: 0.000029
INFO:tensorflow:Step 33000 | Loss: 0.7872 | Spent: 34.6 secs | LR: 0.000029
INFO:tensorflow:Step 33100 | Loss: 0.6330 | Spent: 34.5 secs | LR: 0.000029
INFO:tensorflow:Step 33200 | Loss: 0.6938 | Spent: 34.8 secs | LR: 0.000029
INFO:tensorflow:Step 33300 | Loss: 0.6466 | Spent: 35.3 secs | LR: 0.000029
INFO:tensorflow:Step 33400 | Loss: 0.6109 | Spent: 35.2 secs | LR: 0.000029
INFO:tensorflow:Step 33500 | Loss: 0.6628 | Spent: 35.0 secs | LR: 0.000029
INFO:tensorflow:Step 33600 | Loss: 0.6471 | Spent: 35.2 secs | LR: 0.000030
INFO:tensorflow:Step 33700 | Loss: 0.5551 | Spent: 35.4 secs | LR: 0.000030
INFO:tensorflow:Step 33800 | Loss: 0.5827 | Spent: 35.0 secs | LR: 0.000030
INFO:tensorflow:Step 33900 | Loss: 0.5656 | Spent: 35.3 secs | LR: 0.000030
INFO:tensorflow:Step 34000 | Loss: 0.7119 | Spent: 35.0 secs | LR: 0.000030
INFO:tensorflow:Step 34100 | Loss: 0.6176 | Spent: 35.3 secs | LR: 0.000030
INFO:tensorflow:Step 34200 | Loss: 0.6491 | Spent: 35.3 secs | LR: 0.000030
INFO:tensorflow:Step 34300 | Loss: 0.7020 | Spent: 34.8 secs | LR: 0.000030
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.904
INFO:tensorflow:Best Accuracy: 0.904
Reading ../data/train.txt
INFO:tensorflow:Step 34400 | Loss: 0.5976 | Spent: 149.2 secs | LR: 0.000030
INFO:tensorflow:Step 34500 | Loss: 0.5492 | Spent: 36.5 secs | LR: 0.000030
INFO:tensorflow:Step 34600 | Loss: 0.5369 | Spent: 36.8 secs | LR: 0.000030
INFO:tensorflow:Step 34700 | Loss: 0.6232 | Spent: 36.5 secs | LR: 0.000030
INFO:tensorflow:Step 34800 | Loss: 0.6878 | Spent: 36.6 secs | LR: 0.000030
INFO:tensorflow:Step 34900 | Loss: 0.5969 | Spent: 36.4 secs | LR: 0.000030
INFO:tensorflow:Step 35000 | Loss: 0.6244 | Spent: 36.6 secs | LR: 0.000030
INFO:tensorflow:Step 35100 | Loss: 0.5750 | Spent: 36.8 secs | LR: 0.000030
INFO:tensorflow:Step 35200 | Loss: 0.5867 | Spent: 36.8 secs | LR: 0.000030
INFO:tensorflow:Step 35300 | Loss: 0.5789 | Spent: 36.7 secs | LR: 0.000029
INFO:tensorflow:Step 35400 | Loss: 0.6269 | Spent: 36.8 secs | LR: 0.000029
INFO:tensorflow:Step 35500 | Loss: 0.7506 | Spent: 36.7 secs | LR: 0.000029
INFO:tensorflow:Step 35600 | Loss: 0.6709 | Spent: 36.3 secs | LR: 0.000029
INFO:tensorflow:Step 35700 | Loss: 0.6287 | Spent: 36.6 secs | LR: 0.000029
INFO:tensorflow:Step 35800 | Loss: 0.6506 | Spent: 36.7 secs | LR: 0.000029
INFO:tensorflow:Step 35900 | Loss: 0.5836 | Spent: 36.8 secs | LR: 0.000029
INFO:tensorflow:Step 36000 | Loss: 0.6414 | Spent: 37.2 secs | LR: 0.000029
INFO:tensorflow:Step 36100 | Loss: 0.5993 | Spent: 37.0 secs | LR: 0.000029
INFO:tensorflow:Step 36200 | Loss: 0.7551 | Spent: 36.7 secs | LR: 0.000029
INFO:tensorflow:Step 36300 | Loss: 0.6815 | Spent: 36.6 secs | LR: 0.000029
INFO:tensorflow:Step 36400 | Loss: 0.5520 | Spent: 36.8 secs | LR: 0.000029
INFO:tensorflow:Step 36500 | Loss: 0.6050 | Spent: 36.5 secs | LR: 0.000029
INFO:tensorflow:Step 36600 | Loss: 0.5899 | Spent: 36.8 secs | LR: 0.000029
INFO:tensorflow:Step 36700 | Loss: 0.6580 | Spent: 37.0 secs | LR: 0.000029
INFO:tensorflow:Step 36800 | Loss: 0.7339 | Spent: 37.1 secs | LR: 0.000029
INFO:tensorflow:Step 36900 | Loss: 0.6207 | Spent: 36.9 secs | LR: 0.000029
INFO:tensorflow:Step 37000 | Loss: 0.6886 | Spent: 36.4 secs | LR: 0.000028
INFO:tensorflow:Step 37100 | Loss: 0.5902 | Spent: 36.9 secs | LR: 0.000028
INFO:tensorflow:Step 37200 | Loss: 0.5615 | Spent: 36.9 secs | LR: 0.000028
INFO:tensorflow:Step 37300 | Loss: 0.6941 | Spent: 36.6 secs | LR: 0.000028
INFO:tensorflow:Step 37400 | Loss: 0.6813 | Spent: 36.8 secs | LR: 0.000028
INFO:tensorflow:Step 37500 | Loss: 0.6650 | Spent: 36.8 secs | LR: 0.000028
INFO:tensorflow:Step 37600 | Loss: 0.6819 | Spent: 36.4 secs | LR: 0.000028
INFO:tensorflow:Step 37700 | Loss: 0.6080 | Spent: 37.0 secs | LR: 0.000028
INFO:tensorflow:Step 37800 | Loss: 0.6158 | Spent: 36.5 secs | LR: 0.000028
INFO:tensorflow:Step 37900 | Loss: 0.6242 | Spent: 36.7 secs | LR: 0.000028
INFO:tensorflow:Step 38000 | Loss: 0.6548 | Spent: 36.8 secs | LR: 0.000028
INFO:tensorflow:Step 38100 | Loss: 0.6154 | Spent: 36.7 secs | LR: 0.000028
INFO:tensorflow:Step 38200 | Loss: 0.5731 | Spent: 36.2 secs | LR: 0.000028
INFO:tensorflow:Step 38300 | Loss: 0.5696 | Spent: 36.8 secs | LR: 0.000028
INFO:tensorflow:Step 38400 | Loss: 0.7608 | Spent: 36.7 secs | LR: 0.000028
INFO:tensorflow:Step 38500 | Loss: 0.5682 | Spent: 36.7 secs | LR: 0.000028
INFO:tensorflow:Step 38600 | Loss: 0.5607 | Spent: 36.6 secs | LR: 0.000028
INFO:tensorflow:Step 38700 | Loss: 0.5871 | Spent: 36.7 secs | LR: 0.000027
INFO:tensorflow:Step 38800 | Loss: 0.6734 | Spent: 36.7 secs | LR: 0.000027
INFO:tensorflow:Step 38900 | Loss: 0.5946 | Spent: 36.8 secs | LR: 0.000027
INFO:tensorflow:Step 39000 | Loss: 0.6794 | Spent: 36.4 secs | LR: 0.000027
INFO:tensorflow:Step 39100 | Loss: 0.6473 | Spent: 36.4 secs | LR: 0.000027
INFO:tensorflow:Step 39200 | Loss: 0.6575 | Spent: 36.8 secs | LR: 0.000027
INFO:tensorflow:Step 39300 | Loss: 0.6710 | Spent: 36.7 secs | LR: 0.000027
INFO:tensorflow:Step 39400 | Loss: 0.5547 | Spent: 37.2 secs | LR: 0.000027
INFO:tensorflow:Step 39500 | Loss: 0.6027 | Spent: 36.3 secs | LR: 0.000027
INFO:tensorflow:Step 39600 | Loss: 0.6403 | Spent: 36.9 secs | LR: 0.000027
INFO:tensorflow:Step 39700 | Loss: 0.5571 | Spent: 37.0 secs | LR: 0.000027
INFO:tensorflow:Step 39800 | Loss: 0.5357 | Spent: 36.6 secs | LR: 0.000027
INFO:tensorflow:Step 39900 | Loss: 0.5765 | Spent: 36.7 secs | LR: 0.000027
INFO:tensorflow:Step 40000 | Loss: 0.6922 | Spent: 36.9 secs | LR: 0.000027
INFO:tensorflow:Step 40100 | Loss: 0.5616 | Spent: 36.6 secs | LR: 0.000027
INFO:tensorflow:Step 40200 | Loss: 0.6712 | Spent: 36.7 secs | LR: 0.000027
INFO:tensorflow:Step 40300 | Loss: 0.6415 | Spent: 36.8 secs | LR: 0.000027
INFO:tensorflow:Step 40400 | Loss: 0.5464 | Spent: 36.5 secs | LR: 0.000027
INFO:tensorflow:Step 40500 | Loss: 0.5819 | Spent: 36.4 secs | LR: 0.000026
INFO:tensorflow:Step 40600 | Loss: 0.5873 | Spent: 36.7 secs | LR: 0.000026
INFO:tensorflow:Step 40700 | Loss: 0.6282 | Spent: 36.5 secs | LR: 0.000026
INFO:tensorflow:Step 40800 | Loss: 0.5767 | Spent: 36.4 secs | LR: 0.000026
INFO:tensorflow:Step 40900 | Loss: 0.6853 | Spent: 36.9 secs | LR: 0.000026
INFO:tensorflow:Step 41000 | Loss: 0.5648 | Spent: 36.2 secs | LR: 0.000026
INFO:tensorflow:Step 41100 | Loss: 0.6085 | Spent: 36.9 secs | LR: 0.000026
INFO:tensorflow:Step 41200 | Loss: 0.6255 | Spent: 36.9 secs | LR: 0.000026
INFO:tensorflow:Step 41300 | Loss: 0.7226 | Spent: 36.2 secs | LR: 0.000026
INFO:tensorflow:Step 41400 | Loss: 0.6104 | Spent: 36.4 secs | LR: 0.000026
INFO:tensorflow:Step 41500 | Loss: 0.6120 | Spent: 36.7 secs | LR: 0.000026
INFO:tensorflow:Step 41600 | Loss: 0.6007 | Spent: 37.1 secs | LR: 0.000026
INFO:tensorflow:Step 41700 | Loss: 0.5832 | Spent: 36.7 secs | LR: 0.000026
INFO:tensorflow:Step 41800 | Loss: 0.6182 | Spent: 36.4 secs | LR: 0.000026
INFO:tensorflow:Step 41900 | Loss: 0.5754 | Spent: 36.7 secs | LR: 0.000026
INFO:tensorflow:Step 42000 | Loss: 0.5836 | Spent: 36.7 secs | LR: 0.000026
INFO:tensorflow:Step 42100 | Loss: 0.7144 | Spent: 36.2 secs | LR: 0.000026
INFO:tensorflow:Step 42200 | Loss: 0.5593 | Spent: 36.9 secs | LR: 0.000025
INFO:tensorflow:Step 42300 | Loss: 0.6156 | Spent: 36.7 secs | LR: 0.000025
INFO:tensorflow:Step 42400 | Loss: 0.6332 | Spent: 36.8 secs | LR: 0.000025
INFO:tensorflow:Step 42500 | Loss: 0.5497 | Spent: 36.7 secs | LR: 0.000025
INFO:tensorflow:Step 42600 | Loss: 0.5308 | Spent: 36.8 secs | LR: 0.000025
INFO:tensorflow:Step 42700 | Loss: 0.5936 | Spent: 36.7 secs | LR: 0.000025
INFO:tensorflow:Step 42800 | Loss: 0.6052 | Spent: 36.8 secs | LR: 0.000025
INFO:tensorflow:Step 42900 | Loss: 0.6242 | Spent: 36.7 secs | LR: 0.000025
INFO:tensorflow:Step 43000 | Loss: 0.5399 | Spent: 36.9 secs | LR: 0.000025
INFO:tensorflow:Step 43100 | Loss: 0.5863 | Spent: 36.6 secs | LR: 0.000025
INFO:tensorflow:Step 43200 | Loss: 0.5327 | Spent: 36.8 secs | LR: 0.000025
INFO:tensorflow:Step 43300 | Loss: 0.7443 | Spent: 36.7 secs | LR: 0.000025
INFO:tensorflow:Step 43400 | Loss: 0.6252 | Spent: 37.0 secs | LR: 0.000025
INFO:tensorflow:Step 43500 | Loss: 0.7869 | Spent: 36.6 secs | LR: 0.000025
INFO:tensorflow:Step 43600 | Loss: 0.5329 | Spent: 37.3 secs | LR: 0.000025
INFO:tensorflow:Step 43700 | Loss: 0.6152 | Spent: 36.6 secs | LR: 0.000025
INFO:tensorflow:Step 43800 | Loss: 0.5744 | Spent: 36.8 secs | LR: 0.000025
INFO:tensorflow:Step 43900 | Loss: 0.5462 | Spent: 36.7 secs | LR: 0.000024
INFO:tensorflow:Step 44000 | Loss: 0.7507 | Spent: 36.8 secs | LR: 0.000024
INFO:tensorflow:Step 44100 | Loss: 0.6617 | Spent: 36.6 secs | LR: 0.000024
INFO:tensorflow:Step 44200 | Loss: 0.5539 | Spent: 37.0 secs | LR: 0.000024
INFO:tensorflow:Step 44300 | Loss: 0.6033 | Spent: 36.8 secs | LR: 0.000024
INFO:tensorflow:Step 44400 | Loss: 0.6343 | Spent: 36.9 secs | LR: 0.000024
INFO:tensorflow:Step 44500 | Loss: 0.5229 | Spent: 37.1 secs | LR: 0.000024
INFO:tensorflow:Step 44600 | Loss: 0.5722 | Spent: 36.6 secs | LR: 0.000024
INFO:tensorflow:Step 44700 | Loss: 0.5504 | Spent: 36.8 secs | LR: 0.000024
INFO:tensorflow:Step 44800 | Loss: 0.5687 | Spent: 36.5 secs | LR: 0.000024
INFO:tensorflow:Step 44900 | Loss: 0.5508 | Spent: 36.7 secs | LR: 0.000024
INFO:tensorflow:Step 45000 | Loss: 0.6216 | Spent: 36.9 secs | LR: 0.000024
INFO:tensorflow:Step 45100 | Loss: 0.6561 | Spent: 36.7 secs | LR: 0.000024
INFO:tensorflow:Step 45200 | Loss: 0.5761 | Spent: 36.5 secs | LR: 0.000024
INFO:tensorflow:Step 45300 | Loss: 0.5604 | Spent: 36.1 secs | LR: 0.000024
INFO:tensorflow:Step 45400 | Loss: 0.5332 | Spent: 35.6 secs | LR: 0.000024
INFO:tensorflow:Step 45500 | Loss: 0.6120 | Spent: 35.2 secs | LR: 0.000024
INFO:tensorflow:Step 45600 | Loss: 0.6157 | Spent: 35.2 secs | LR: 0.000023
INFO:tensorflow:Step 45700 | Loss: 0.6329 | Spent: 35.2 secs | LR: 0.000023
INFO:tensorflow:Step 45800 | Loss: 0.6137 | Spent: 35.4 secs | LR: 0.000023
INFO:tensorflow:Step 45900 | Loss: 0.5673 | Spent: 35.4 secs | LR: 0.000023
INFO:tensorflow:Step 46000 | Loss: 0.5341 | Spent: 35.2 secs | LR: 0.000023
INFO:tensorflow:Step 46100 | Loss: 0.6583 | Spent: 35.1 secs | LR: 0.000023
INFO:tensorflow:Step 46200 | Loss: 0.5808 | Spent: 36.0 secs | LR: 0.000023
INFO:tensorflow:Step 46300 | Loss: 0.7607 | Spent: 35.7 secs | LR: 0.000023
INFO:tensorflow:Step 46400 | Loss: 0.5260 | Spent: 36.4 secs | LR: 0.000023
INFO:tensorflow:Step 46500 | Loss: 0.6460 | Spent: 36.4 secs | LR: 0.000023
INFO:tensorflow:Step 46600 | Loss: 0.6316 | Spent: 35.2 secs | LR: 0.000023
INFO:tensorflow:Step 46700 | Loss: 0.5633 | Spent: 35.9 secs | LR: 0.000023
INFO:tensorflow:Step 46800 | Loss: 0.6513 | Spent: 35.1 secs | LR: 0.000023
INFO:tensorflow:Step 46900 | Loss: 0.5716 | Spent: 35.0 secs | LR: 0.000023
INFO:tensorflow:Step 47000 | Loss: 0.6087 | Spent: 34.5 secs | LR: 0.000023
INFO:tensorflow:Step 47100 | Loss: 0.5417 | Spent: 34.7 secs | LR: 0.000023
INFO:tensorflow:Step 47200 | Loss: 0.5990 | Spent: 34.5 secs | LR: 0.000023
INFO:tensorflow:Step 47300 | Loss: 0.5730 | Spent: 34.5 secs | LR: 0.000022
INFO:tensorflow:Step 47400 | Loss: 0.6093 | Spent: 34.2 secs | LR: 0.000022
INFO:tensorflow:Step 47500 | Loss: 0.6344 | Spent: 34.7 secs | LR: 0.000022
INFO:tensorflow:Step 47600 | Loss: 0.6182 | Spent: 34.7 secs | LR: 0.000022
INFO:tensorflow:Step 47700 | Loss: 0.7179 | Spent: 34.5 secs | LR: 0.000022
INFO:tensorflow:Step 47800 | Loss: 0.6597 | Spent: 34.8 secs | LR: 0.000022
INFO:tensorflow:Step 47900 | Loss: 0.5810 | Spent: 34.6 secs | LR: 0.000022
INFO:tensorflow:Step 48000 | Loss: 0.6206 | Spent: 34.5 secs | LR: 0.000022
INFO:tensorflow:Step 48100 | Loss: 0.6121 | Spent: 34.9 secs | LR: 0.000022
INFO:tensorflow:Step 48200 | Loss: 0.5387 | Spent: 34.4 secs | LR: 0.000022
INFO:tensorflow:Step 48300 | Loss: 0.6438 | Spent: 34.5 secs | LR: 0.000022
INFO:tensorflow:Step 48400 | Loss: 0.6223 | Spent: 34.5 secs | LR: 0.000022
INFO:tensorflow:Step 48500 | Loss: 0.5683 | Spent: 34.7 secs | LR: 0.000022
INFO:tensorflow:Step 48600 | Loss: 0.6074 | Spent: 34.8 secs | LR: 0.000022
INFO:tensorflow:Step 48700 | Loss: 0.5976 | Spent: 34.5 secs | LR: 0.000022
INFO:tensorflow:Step 48800 | Loss: 0.5131 | Spent: 34.9 secs | LR: 0.000022
INFO:tensorflow:Step 48900 | Loss: 0.6973 | Spent: 34.4 secs | LR: 0.000022
INFO:tensorflow:Step 49000 | Loss: 0.5523 | Spent: 34.4 secs | LR: 0.000021
INFO:tensorflow:Step 49100 | Loss: 0.6546 | Spent: 34.5 secs | LR: 0.000021
INFO:tensorflow:Step 49200 | Loss: 0.6658 | Spent: 34.7 secs | LR: 0.000021
INFO:tensorflow:Step 49300 | Loss: 0.8393 | Spent: 34.8 secs | LR: 0.000021
INFO:tensorflow:Step 49400 | Loss: 0.7723 | Spent: 34.7 secs | LR: 0.000021
INFO:tensorflow:Step 49500 | Loss: 0.5597 | Spent: 34.7 secs | LR: 0.000021
INFO:tensorflow:Step 49600 | Loss: 0.6871 | Spent: 34.4 secs | LR: 0.000021
INFO:tensorflow:Step 49700 | Loss: 0.7172 | Spent: 35.1 secs | LR: 0.000021
INFO:tensorflow:Step 49800 | Loss: 0.6470 | Spent: 35.0 secs | LR: 0.000021
INFO:tensorflow:Step 49900 | Loss: 0.5455 | Spent: 34.6 secs | LR: 0.000021
INFO:tensorflow:Step 50000 | Loss: 0.6035 | Spent: 34.9 secs | LR: 0.000021
INFO:tensorflow:Step 50100 | Loss: 0.7743 | Spent: 34.7 secs | LR: 0.000021
INFO:tensorflow:Step 50200 | Loss: 0.5767 | Spent: 34.5 secs | LR: 0.000021
INFO:tensorflow:Step 50300 | Loss: 0.6054 | Spent: 35.1 secs | LR: 0.000021
INFO:tensorflow:Step 50400 | Loss: 0.6018 | Spent: 34.9 secs | LR: 0.000021
INFO:tensorflow:Step 50500 | Loss: 0.5571 | Spent: 34.9 secs | LR: 0.000021
INFO:tensorflow:Step 50600 | Loss: 0.5851 | Spent: 35.4 secs | LR: 0.000021
INFO:tensorflow:Step 50700 | Loss: 0.5433 | Spent: 34.6 secs | LR: 0.000021
INFO:tensorflow:Step 50800 | Loss: 0.7359 | Spent: 34.9 secs | LR: 0.000020
INFO:tensorflow:Step 50900 | Loss: 0.5486 | Spent: 34.8 secs | LR: 0.000020
INFO:tensorflow:Step 51000 | Loss: 0.6791 | Spent: 34.6 secs | LR: 0.000020
INFO:tensorflow:Step 51100 | Loss: 0.5689 | Spent: 34.8 secs | LR: 0.000020
INFO:tensorflow:Step 51200 | Loss: 0.5775 | Spent: 34.8 secs | LR: 0.000020
INFO:tensorflow:Step 51300 | Loss: 0.7624 | Spent: 34.1 secs | LR: 0.000020
INFO:tensorflow:Step 51400 | Loss: 0.5461 | Spent: 35.0 secs | LR: 0.000020
INFO:tensorflow:Step 51500 | Loss: 0.5842 | Spent: 35.3 secs | LR: 0.000020
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.909
INFO:tensorflow:Best Accuracy: 0.909
Reading ../data/train.txt
INFO:tensorflow:Step 51600 | Loss: 0.6850 | Spent: 148.3 secs | LR: 0.000020
INFO:tensorflow:Step 51700 | Loss: 0.5856 | Spent: 36.9 secs | LR: 0.000020
INFO:tensorflow:Step 51800 | Loss: 0.6094 | Spent: 36.5 secs | LR: 0.000020
INFO:tensorflow:Step 51900 | Loss: 0.6256 | Spent: 36.7 secs | LR: 0.000020
INFO:tensorflow:Step 52000 | Loss: 0.6816 | Spent: 36.7 secs | LR: 0.000020
INFO:tensorflow:Step 52100 | Loss: 0.5392 | Spent: 36.6 secs | LR: 0.000020
INFO:tensorflow:Step 52200 | Loss: 0.6242 | Spent: 36.4 secs | LR: 0.000020
INFO:tensorflow:Step 52300 | Loss: 0.6083 | Spent: 36.1 secs | LR: 0.000020
INFO:tensorflow:Step 52400 | Loss: 0.6074 | Spent: 36.1 secs | LR: 0.000020
INFO:tensorflow:Step 52500 | Loss: 0.5607 | Spent: 36.5 secs | LR: 0.000019
INFO:tensorflow:Step 52600 | Loss: 0.6246 | Spent: 36.3 secs | LR: 0.000019
INFO:tensorflow:Step 52700 | Loss: 0.5864 | Spent: 36.4 secs | LR: 0.000019
INFO:tensorflow:Step 52800 | Loss: 0.5880 | Spent: 36.3 secs | LR: 0.000019
INFO:tensorflow:Step 52900 | Loss: 0.5348 | Spent: 36.7 secs | LR: 0.000019
INFO:tensorflow:Step 53000 | Loss: 0.5267 | Spent: 35.9 secs | LR: 0.000019
INFO:tensorflow:Step 53100 | Loss: 0.6296 | Spent: 35.8 secs | LR: 0.000019
INFO:tensorflow:Step 53200 | Loss: 0.5856 | Spent: 35.9 secs | LR: 0.000019
INFO:tensorflow:Step 53300 | Loss: 0.6028 | Spent: 36.5 secs | LR: 0.000019
INFO:tensorflow:Step 53400 | Loss: 0.5817 | Spent: 36.3 secs | LR: 0.000019
INFO:tensorflow:Step 53500 | Loss: 0.5684 | Spent: 36.2 secs | LR: 0.000019
INFO:tensorflow:Step 53600 | Loss: 0.6095 | Spent: 36.2 secs | LR: 0.000019
INFO:tensorflow:Step 53700 | Loss: 0.6279 | Spent: 36.3 secs | LR: 0.000019
INFO:tensorflow:Step 53800 | Loss: 0.5652 | Spent: 36.7 secs | LR: 0.000019
INFO:tensorflow:Step 53900 | Loss: 0.6205 | Spent: 36.4 secs | LR: 0.000019
INFO:tensorflow:Step 54000 | Loss: 0.5109 | Spent: 36.0 secs | LR: 0.000019
INFO:tensorflow:Step 54100 | Loss: 0.5385 | Spent: 35.9 secs | LR: 0.000019
INFO:tensorflow:Step 54200 | Loss: 0.5944 | Spent: 36.5 secs | LR: 0.000018
INFO:tensorflow:Step 54300 | Loss: 0.6556 | Spent: 36.4 secs | LR: 0.000018
INFO:tensorflow:Step 54400 | Loss: 0.6163 | Spent: 36.3 secs | LR: 0.000018
INFO:tensorflow:Step 54500 | Loss: 0.6608 | Spent: 36.2 secs | LR: 0.000018
INFO:tensorflow:Step 54600 | Loss: 0.6235 | Spent: 36.6 secs | LR: 0.000018
INFO:tensorflow:Step 54700 | Loss: 0.6393 | Spent: 36.8 secs | LR: 0.000018
INFO:tensorflow:Step 54800 | Loss: 0.5870 | Spent: 36.2 secs | LR: 0.000018
INFO:tensorflow:Step 54900 | Loss: 0.6238 | Spent: 35.8 secs | LR: 0.000018
INFO:tensorflow:Step 55000 | Loss: 0.5404 | Spent: 36.8 secs | LR: 0.000018
INFO:tensorflow:Step 55100 | Loss: 0.5113 | Spent: 36.6 secs | LR: 0.000018
INFO:tensorflow:Step 55200 | Loss: 0.6076 | Spent: 36.2 secs | LR: 0.000018
INFO:tensorflow:Step 55300 | Loss: 0.5849 | Spent: 36.6 secs | LR: 0.000018
INFO:tensorflow:Step 55400 | Loss: 0.5851 | Spent: 36.3 secs | LR: 0.000018
INFO:tensorflow:Step 55500 | Loss: 0.5106 | Spent: 36.7 secs | LR: 0.000018
INFO:tensorflow:Step 55600 | Loss: 0.6047 | Spent: 36.6 secs | LR: 0.000018
INFO:tensorflow:Step 55700 | Loss: 0.6550 | Spent: 36.8 secs | LR: 0.000018
INFO:tensorflow:Step 55800 | Loss: 0.5936 | Spent: 36.2 secs | LR: 0.000018
INFO:tensorflow:Step 55900 | Loss: 0.5967 | Spent: 36.5 secs | LR: 0.000017
INFO:tensorflow:Step 56000 | Loss: 0.7103 | Spent: 36.5 secs | LR: 0.000017
INFO:tensorflow:Step 56100 | Loss: 0.6506 | Spent: 36.4 secs | LR: 0.000017
INFO:tensorflow:Step 56200 | Loss: 0.6064 | Spent: 36.5 secs | LR: 0.000017
INFO:tensorflow:Step 56300 | Loss: 0.5882 | Spent: 36.5 secs | LR: 0.000017
INFO:tensorflow:Step 56400 | Loss: 0.5321 | Spent: 37.1 secs | LR: 0.000017
INFO:tensorflow:Step 56500 | Loss: 0.7599 | Spent: 36.3 secs | LR: 0.000017
INFO:tensorflow:Step 56600 | Loss: 0.6502 | Spent: 36.5 secs | LR: 0.000017
INFO:tensorflow:Step 56700 | Loss: 0.6288 | Spent: 36.5 secs | LR: 0.000017
INFO:tensorflow:Step 56800 | Loss: 0.6282 | Spent: 36.6 secs | LR: 0.000017
INFO:tensorflow:Step 56900 | Loss: 0.6368 | Spent: 36.4 secs | LR: 0.000017
INFO:tensorflow:Step 57000 | Loss: 0.6086 | Spent: 36.3 secs | LR: 0.000017
INFO:tensorflow:Step 57100 | Loss: 0.5721 | Spent: 36.2 secs | LR: 0.000017
INFO:tensorflow:Step 57200 | Loss: 0.5660 | Spent: 36.8 secs | LR: 0.000017
INFO:tensorflow:Step 57300 | Loss: 0.5714 | Spent: 36.6 secs | LR: 0.000017
INFO:tensorflow:Step 57400 | Loss: 0.6880 | Spent: 36.0 secs | LR: 0.000017
INFO:tensorflow:Step 57500 | Loss: 0.6785 | Spent: 36.7 secs | LR: 0.000017
INFO:tensorflow:Step 57600 | Loss: 0.6589 | Spent: 36.4 secs | LR: 0.000016
INFO:tensorflow:Step 57700 | Loss: 0.5382 | Spent: 36.2 secs | LR: 0.000016
INFO:tensorflow:Step 57800 | Loss: 0.5306 | Spent: 36.2 secs | LR: 0.000016
INFO:tensorflow:Step 57900 | Loss: 0.5039 | Spent: 36.3 secs | LR: 0.000016
INFO:tensorflow:Step 58000 | Loss: 0.5562 | Spent: 36.4 secs | LR: 0.000016
INFO:tensorflow:Step 58100 | Loss: 0.5843 | Spent: 36.5 secs | LR: 0.000016
INFO:tensorflow:Step 58200 | Loss: 0.6921 | Spent: 36.3 secs | LR: 0.000016
INFO:tensorflow:Step 58300 | Loss: 0.6289 | Spent: 36.4 secs | LR: 0.000016
INFO:tensorflow:Step 58400 | Loss: 0.5502 | Spent: 36.5 secs | LR: 0.000016
INFO:tensorflow:Step 58500 | Loss: 0.5782 | Spent: 36.3 secs | LR: 0.000016
INFO:tensorflow:Step 58600 | Loss: 0.5807 | Spent: 36.7 secs | LR: 0.000016
INFO:tensorflow:Step 58700 | Loss: 0.6473 | Spent: 36.5 secs | LR: 0.000016
INFO:tensorflow:Step 58800 | Loss: 0.5667 | Spent: 36.5 secs | LR: 0.000016
INFO:tensorflow:Step 58900 | Loss: 0.6409 | Spent: 37.2 secs | LR: 0.000016
INFO:tensorflow:Step 59000 | Loss: 0.5997 | Spent: 36.6 secs | LR: 0.000016
INFO:tensorflow:Step 59100 | Loss: 0.5564 | Spent: 36.4 secs | LR: 0.000016
INFO:tensorflow:Step 59200 | Loss: 0.6732 | Spent: 36.8 secs | LR: 0.000016
INFO:tensorflow:Step 59300 | Loss: 0.5614 | Spent: 36.7 secs | LR: 0.000016
INFO:tensorflow:Step 59400 | Loss: 0.5309 | Spent: 36.5 secs | LR: 0.000015
INFO:tensorflow:Step 59500 | Loss: 0.5797 | Spent: 36.5 secs | LR: 0.000015
INFO:tensorflow:Step 59600 | Loss: 0.5929 | Spent: 36.4 secs | LR: 0.000015
INFO:tensorflow:Step 59700 | Loss: 0.6236 | Spent: 36.2 secs | LR: 0.000015
INFO:tensorflow:Step 59800 | Loss: 0.5123 | Spent: 36.7 secs | LR: 0.000015
INFO:tensorflow:Step 59900 | Loss: 0.5071 | Spent: 36.5 secs | LR: 0.000015
INFO:tensorflow:Step 60000 | Loss: 0.5273 | Spent: 36.3 secs | LR: 0.000015
INFO:tensorflow:Step 60100 | Loss: 0.5534 | Spent: 36.7 secs | LR: 0.000015
INFO:tensorflow:Step 60200 | Loss: 0.5278 | Spent: 36.1 secs | LR: 0.000015
INFO:tensorflow:Step 60300 | Loss: 0.5376 | Spent: 36.8 secs | LR: 0.000015
INFO:tensorflow:Step 60400 | Loss: 0.5559 | Spent: 36.2 secs | LR: 0.000015
INFO:tensorflow:Step 60500 | Loss: 0.6411 | Spent: 36.5 secs | LR: 0.000015
INFO:tensorflow:Step 60600 | Loss: 0.5538 | Spent: 36.7 secs | LR: 0.000015
INFO:tensorflow:Step 60700 | Loss: 0.5696 | Spent: 36.0 secs | LR: 0.000015
INFO:tensorflow:Step 60800 | Loss: 0.5737 | Spent: 36.1 secs | LR: 0.000015
INFO:tensorflow:Step 60900 | Loss: 0.5466 | Spent: 36.4 secs | LR: 0.000015
INFO:tensorflow:Step 61000 | Loss: 0.6113 | Spent: 36.3 secs | LR: 0.000015
INFO:tensorflow:Step 61100 | Loss: 0.6604 | Spent: 36.2 secs | LR: 0.000014
INFO:tensorflow:Step 61200 | Loss: 0.5633 | Spent: 36.2 secs | LR: 0.000014
INFO:tensorflow:Step 61300 | Loss: 0.6548 | Spent: 35.9 secs | LR: 0.000014
INFO:tensorflow:Step 61400 | Loss: 0.6174 | Spent: 36.3 secs | LR: 0.000014
INFO:tensorflow:Step 61500 | Loss: 0.5719 | Spent: 36.5 secs | LR: 0.000014
INFO:tensorflow:Step 61600 | Loss: 0.5405 | Spent: 36.2 secs | LR: 0.000014
INFO:tensorflow:Step 61700 | Loss: 0.7432 | Spent: 36.5 secs | LR: 0.000014
INFO:tensorflow:Step 61800 | Loss: 0.6597 | Spent: 36.4 secs | LR: 0.000014
INFO:tensorflow:Step 61900 | Loss: 0.5023 | Spent: 36.1 secs | LR: 0.000014
INFO:tensorflow:Step 62000 | Loss: 0.7216 | Spent: 36.0 secs | LR: 0.000014
INFO:tensorflow:Step 62100 | Loss: 0.6240 | Spent: 36.2 secs | LR: 0.000014
INFO:tensorflow:Step 62200 | Loss: 0.5786 | Spent: 36.3 secs | LR: 0.000014
INFO:tensorflow:Step 62300 | Loss: 0.5173 | Spent: 36.6 secs | LR: 0.000014
INFO:tensorflow:Step 62400 | Loss: 0.6337 | Spent: 36.1 secs | LR: 0.000014
INFO:tensorflow:Step 62500 | Loss: 0.5260 | Spent: 35.2 secs | LR: 0.000014
INFO:tensorflow:Step 62600 | Loss: 0.5458 | Spent: 34.5 secs | LR: 0.000014
INFO:tensorflow:Step 62700 | Loss: 0.5066 | Spent: 34.6 secs | LR: 0.000014
INFO:tensorflow:Step 62800 | Loss: 0.4963 | Spent: 34.9 secs | LR: 0.000013
INFO:tensorflow:Step 62900 | Loss: 0.5071 | Spent: 34.9 secs | LR: 0.000013
INFO:tensorflow:Step 63000 | Loss: 0.6698 | Spent: 34.7 secs | LR: 0.000013
INFO:tensorflow:Step 63100 | Loss: 0.6082 | Spent: 34.9 secs | LR: 0.000013
INFO:tensorflow:Step 63200 | Loss: 0.5312 | Spent: 35.6 secs | LR: 0.000013
INFO:tensorflow:Step 63300 | Loss: 0.6037 | Spent: 34.8 secs | LR: 0.000013
INFO:tensorflow:Step 63400 | Loss: 0.5024 | Spent: 34.8 secs | LR: 0.000013
INFO:tensorflow:Step 63500 | Loss: 0.5747 | Spent: 35.0 secs | LR: 0.000013
INFO:tensorflow:Step 63600 | Loss: 0.5530 | Spent: 34.5 secs | LR: 0.000013
INFO:tensorflow:Step 63700 | Loss: 0.6464 | Spent: 34.6 secs | LR: 0.000013
INFO:tensorflow:Step 63800 | Loss: 0.5060 | Spent: 34.4 secs | LR: 0.000013
INFO:tensorflow:Step 63900 | Loss: 0.5788 | Spent: 34.7 secs | LR: 0.000013
INFO:tensorflow:Step 64000 | Loss: 0.6121 | Spent: 35.0 secs | LR: 0.000013
INFO:tensorflow:Step 64100 | Loss: 0.6609 | Spent: 35.0 secs | LR: 0.000013
INFO:tensorflow:Step 64200 | Loss: 0.5616 | Spent: 35.1 secs | LR: 0.000013
INFO:tensorflow:Step 64300 | Loss: 0.5876 | Spent: 34.7 secs | LR: 0.000013
INFO:tensorflow:Step 64400 | Loss: 0.5920 | Spent: 34.6 secs | LR: 0.000013
INFO:tensorflow:Step 64500 | Loss: 0.5822 | Spent: 34.8 secs | LR: 0.000012
INFO:tensorflow:Step 64600 | Loss: 0.5638 | Spent: 34.7 secs | LR: 0.000012
INFO:tensorflow:Step 64700 | Loss: 0.6244 | Spent: 34.7 secs | LR: 0.000012
INFO:tensorflow:Step 64800 | Loss: 0.6006 | Spent: 34.8 secs | LR: 0.000012
INFO:tensorflow:Step 64900 | Loss: 0.5687 | Spent: 35.0 secs | LR: 0.000012
INFO:tensorflow:Step 65000 | Loss: 0.5526 | Spent: 35.3 secs | LR: 0.000012
INFO:tensorflow:Step 65100 | Loss: 0.5369 | Spent: 35.2 secs | LR: 0.000012
INFO:tensorflow:Step 65200 | Loss: 0.6170 | Spent: 35.0 secs | LR: 0.000012
INFO:tensorflow:Step 65300 | Loss: 0.7168 | Spent: 35.3 secs | LR: 0.000012
INFO:tensorflow:Step 65400 | Loss: 0.5458 | Spent: 35.3 secs | LR: 0.000012
INFO:tensorflow:Step 65500 | Loss: 0.5814 | Spent: 35.0 secs | LR: 0.000012
INFO:tensorflow:Step 65600 | Loss: 0.5836 | Spent: 35.5 secs | LR: 0.000012
INFO:tensorflow:Step 65700 | Loss: 0.5394 | Spent: 35.1 secs | LR: 0.000012
INFO:tensorflow:Step 65800 | Loss: 0.6054 | Spent: 34.9 secs | LR: 0.000012
INFO:tensorflow:Step 65900 | Loss: 0.5543 | Spent: 35.2 secs | LR: 0.000012
INFO:tensorflow:Step 66000 | Loss: 0.5946 | Spent: 34.9 secs | LR: 0.000012
INFO:tensorflow:Step 66100 | Loss: 0.5605 | Spent: 34.7 secs | LR: 0.000012
INFO:tensorflow:Step 66200 | Loss: 0.5694 | Spent: 34.6 secs | LR: 0.000011
INFO:tensorflow:Step 66300 | Loss: 0.5492 | Spent: 34.3 secs | LR: 0.000011
INFO:tensorflow:Step 66400 | Loss: 0.6009 | Spent: 34.6 secs | LR: 0.000011
INFO:tensorflow:Step 66500 | Loss: 0.6071 | Spent: 34.5 secs | LR: 0.000011
INFO:tensorflow:Step 66600 | Loss: 0.5769 | Spent: 34.4 secs | LR: 0.000011
INFO:tensorflow:Step 66700 | Loss: 0.6170 | Spent: 34.7 secs | LR: 0.000011
INFO:tensorflow:Step 66800 | Loss: 0.5462 | Spent: 35.1 secs | LR: 0.000011
INFO:tensorflow:Step 66900 | Loss: 0.5578 | Spent: 34.7 secs | LR: 0.000011
INFO:tensorflow:Step 67000 | Loss: 0.6640 | Spent: 35.1 secs | LR: 0.000011
INFO:tensorflow:Step 67100 | Loss: 0.5454 | Spent: 34.9 secs | LR: 0.000011
INFO:tensorflow:Step 67200 | Loss: 0.5617 | Spent: 35.1 secs | LR: 0.000011
INFO:tensorflow:Step 67300 | Loss: 0.5064 | Spent: 34.9 secs | LR: 0.000011
INFO:tensorflow:Step 67400 | Loss: 0.5832 | Spent: 34.6 secs | LR: 0.000011
INFO:tensorflow:Step 67500 | Loss: 0.6218 | Spent: 34.8 secs | LR: 0.000011
INFO:tensorflow:Step 67600 | Loss: 0.5690 | Spent: 34.5 secs | LR: 0.000011
INFO:tensorflow:Step 67700 | Loss: 0.5444 | Spent: 34.8 secs | LR: 0.000011
INFO:tensorflow:Step 67800 | Loss: 0.5766 | Spent: 34.7 secs | LR: 0.000011
INFO:tensorflow:Step 67900 | Loss: 0.6285 | Spent: 34.8 secs | LR: 0.000011
INFO:tensorflow:Step 68000 | Loss: 0.5062 | Spent: 34.7 secs | LR: 0.000010
INFO:tensorflow:Step 68100 | Loss: 0.6384 | Spent: 34.9 secs | LR: 0.000010
INFO:tensorflow:Step 68200 | Loss: 0.6263 | Spent: 34.8 secs | LR: 0.000010
INFO:tensorflow:Step 68300 | Loss: 0.6249 | Spent: 34.4 secs | LR: 0.000010
INFO:tensorflow:Step 68400 | Loss: 0.5151 | Spent: 34.5 secs | LR: 0.000010
INFO:tensorflow:Step 68500 | Loss: 0.5898 | Spent: 34.8 secs | LR: 0.000010
INFO:tensorflow:Step 68600 | Loss: 0.5927 | Spent: 35.0 secs | LR: 0.000010
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.911
INFO:tensorflow:Best Accuracy: 0.911
Reading ../data/train.txt