In [1]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_classification/imdb/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
%tensorflow_version 2.x
!pip install tensorflow-addons
Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (0.8.3)
Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons) (2.7.1)
In [3]:
import tensorflow as tf
import numpy as np
import pprint
import logging
import time

from tensorflow_addons.optimizers.cyclical_learning_rate import ExponentialCyclicalLearningRate

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.2.0
WARNING:tensorflow:From <ipython-input-3-f21124be3d8d>:10: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU Enabled: True
In [4]:
def get_vocab(f_path):
  word2idx = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx


def get_idx(symbol2idx, symbol):
  return symbol2idx.get(symbol, len(symbol2idx))
In [5]:
def data_generator(f_paths, params):
  for f_path in f_paths:
    
    with open(f_path) as f:
      print('Reading', f_path)
      for line in f:
        line = line.rstrip()
        label, text = line.split('\t')
        text = text.split(' ')

        words = [get_idx(params['word2idx'], w) for w in text]
        if len(words) >= params['max_word_len']:
          words = words[:params['max_word_len']]
        else:
          words += [0] * (params['max_word_len'] - len(words))

        chars = []
        for w in text:
          temp = []
          for c in list(w):
            temp.append(get_idx(params['char2idx'], c))
          if len(temp) < params['max_char_len']:
            temp += [0] * (params['max_char_len'] - len(temp))
          else:
            temp = temp[:params['max_char_len']]
          chars.append(temp)
        if len(chars) >= params['max_word_len']:
          chars = chars[:params['max_word_len']]
        else:
          chars += [[0] * params['max_char_len']] * (params['max_word_len'] - len(chars))

        y = int(label)

        yield words, chars, y


def dataset(is_training, params):
  _shapes = ([params['max_word_len']], [params['max_word_len'], params['max_char_len']], ())
  _types = (tf.int32, tf.int32, tf.int32)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_paths'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['buffer_size'])
    ds = ds.batch(params['batch_size'])
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_paths'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.batch(params['batch_size'])
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [6]:
class AttentivePooling(tf.keras.Model):
  def __init__(self, params):
    super().__init__()

    self.dropout_1 = tf.keras.layers.Dropout(params['dropout_rate'])

    self.dropout_2 = tf.keras.layers.Dropout(params['dropout_rate'])

    self.kernel_1 = tf.keras.layers.Dense(params['rnn_units'], activation=tf.tanh,)
    
    self.kernel_2 = tf.keras.layers.Dense(1, use_bias=False)

  
  def call(self, inputs, training=False):
    inputs, masks = inputs
    x = self.dropout_1(inputs, training=training)
    # alignment
    align = tf.squeeze(self.kernel_2(self.dropout_2(self.kernel_1(x), training=training)), -1)
    # masking
    paddings = tf.fill(tf.shape(align), float('-inf'))
    align = tf.where(tf.equal(masks, 0), paddings, align)
    # probability
    align = tf.nn.softmax(align)
    align = tf.expand_dims(align, -1)
    # weighted sum
    return tf.squeeze(tf.matmul(inputs, align, transpose_a=True), -1)
In [7]:
class Model(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    
    self.char_embedding = tf.keras.layers.Embedding(len(params['char2idx'])+1, params['char_embed_size'])
    self.word_embedding = tf.Variable(np.load('../vocab/word.npy'),
                                      dtype=tf.float32,
                                      name='pretrained_glove',
                                      trainable=False,)
    
    self.embed_drop = tf.keras.layers.Dropout(params['dropout_rate'])
    self.embed_fc = tf.keras.layers.Dense(params['cnn_filters'], tf.nn.elu, name='embed_fc')
    
    self.drop1 = tf.keras.layers.Dropout(params['dropout_rate'])
    self.drop2 = tf.keras.layers.Dropout(params['dropout_rate'])
    
    self.char_cnn = tf.keras.layers.Conv1D(filters=params['cnn_filters'], kernel_size=params['cnn_kernel_size'], padding='same')
    self.word_rnn1 = tf.keras.layers.Bidirectional(tf.keras.layers.RNN(tf.keras.layers.LSTMCell(params['rnn_units']), return_sequences=True))
    self.word_rnn2 = tf.keras.layers.Bidirectional(tf.keras.layers.RNN(tf.keras.layers.LSTMCell(params['rnn_units']), return_sequences=True))

    self.attentive_pooling = AttentivePooling(params)
    
    self.fc_drop = tf.keras.layers.Dropout(params['dropout_rate'])
    self.fc = tf.keras.layers.Dense(params['rnn_units'], tf.nn.elu)
    
    self.out_linear = tf.keras.layers.Dense(2)
  
  
  def call(self, inputs, training=False):
    words, chars = inputs
    if words.dtype != tf.int32:
      words = tf.cast(words, tf.int32)
    
    batch_sz = tf.shape(words)[0]
    rnn_units = 2 * params['rnn_units']
    rnn_mask = tf.sign(words)
    #cnn_mask = tf.expand_dims(tf.cast(tf.sign(chars), tf.float32), -1)
    
    chars = self.char_embedding(chars)
    chars = tf.reshape(chars, (batch_sz*params['max_word_len'], params['max_char_len'], params['char_embed_size']))
    chars = self.char_cnn(chars)
    #chars *= cnn_mask
    chars = tf.reduce_max(chars, 1)
    chars = tf.reshape(chars, (batch_sz, params['max_word_len'], params['cnn_filters']))
    
    words = tf.nn.embedding_lookup(self.word_embedding, words)
    
    x = tf.concat((words, chars), axis=-1)
    x = self.embed_drop(x, training=training)
    x = self.embed_fc(x)
    
    x = tf.reshape(x, (batch_sz*5*10, 20, params['cnn_filters']))
    x = self.drop1(x, training=training)
    rnn_mask = tf.reshape(rnn_mask, (batch_sz*5*10, 20))
    x = self.word_rnn1(x, mask=tf.cast(rnn_mask, tf.bool))
    x = tf.reduce_max(x, 1)
    
    x = tf.reshape(x, (batch_sz*5, 10, rnn_units))
    x = self.drop2(x, training=training)
    rnn_mask = tf.reshape(tf.sign(tf.reduce_sum(rnn_mask, 1)), (batch_sz*5, 10))
    x = self.word_rnn2(x, mask=tf.cast(rnn_mask, tf.bool))
    x = tf.reduce_max(x, 1)
    
    x = tf.reshape(x, (batch_sz, 5, rnn_units))
    rnn_mask = tf.reshape(tf.sign(tf.reduce_sum(rnn_mask, 1)), (batch_sz, 5))
    x = self.attentive_pooling((x, rnn_mask), training=training)
    
    x = self.fc_drop(x, training=training)
    x = self.fc(x)
    
    x = self.out_linear(x)
    
    return x
In [8]:
params = {
  'train_paths': [
    '../data/train_bt_part1.txt',
    '../data/train_bt_part2.txt',
    '../data/train_bt_part3.txt',
    '../data/train_bt_part4.txt',
    '../data/train_bt_part5.txt',
    '../data/train_bt_part6.txt',
  ],
  'test_paths': [
    '../data/test.txt',
  ],
  'buffer_size': 25000*2,
  'num_labels': 2,
  'batch_size': 32,
  'max_word_len': 1000,
  'max_char_len': 10,
  'char_embed_size': 100,
  'rnn_units': 200,
  'cnn_filters': 300,
  'cnn_kernel_size': 5,
  'dropout_rate': .2,
  'clip_norm': .1,
  'num_patience': 7,
  'init_lr': 1e-4,
  'max_lr': 8e-4,
}
In [ ]:
params['char2idx'] = get_vocab('../vocab/char.txt')
params['word2idx'] = get_vocab('../vocab/word.txt')

model = Model(params)
model.build(input_shape=[[None, params['max_word_len']], [None, params['max_word_len'], params['max_char_len']]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

step_size = 8 * params['buffer_size'] // params['batch_size']
decay_lr = ExponentialCyclicalLearningRate(
  initial_learning_rate = params['init_lr'],
  maximal_learning_rate = params['max_lr'],
  step_size = step_size,)
optim = tf.optimizers.Adam(params['init_lr'])
global_step = 0

best_acc = .0
count = 0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)


while True:
  # TRAINING
  for words, chars, labels in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      logits = model((words, chars), training=True)
      loss = tf.reduce_mean(tf.losses.categorical_crossentropy(y_true = tf.one_hot(labels, 2),
                                                               y_pred = logits,
                                                               from_logits = True,
                                                               label_smoothing = .2,))
  
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1

  # EVALUATION
  m = tf.keras.metrics.Accuracy()

  for words, chars, labels in dataset(is_training=False, params=params):
    logits = model((words, chars), training=False)
    y_pred = tf.argmax(logits, axis=-1)
    m.update_state(y_true=labels, y_pred=y_pred)
    
  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))
  
  if acc > best_acc:
    best_acc = acc
    model.save_weights('../model/sliced_rnn_char_cnn')
    count = 0
  else:
    count += 1
  logger.info("Best Accuracy: {:.3f}".format(best_acc))
  
  if count == params['num_patience']:
    logger.info("Testing Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
[('embedding/embeddings:0', TensorShape([80, 100])),
 ('embed_fc/kernel:0', TensorShape([600, 300])),
 ('embed_fc/bias:0', TensorShape([300])),
 ('conv1d/kernel:0', TensorShape([5, 100, 300])),
 ('conv1d/bias:0', TensorShape([300])),
 ('bidirectional/forward_rnn/lstm_cell/kernel:0', TensorShape([300, 800])),
 ('bidirectional/forward_rnn/lstm_cell/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional/forward_rnn/lstm_cell/bias:0', TensorShape([800])),
 ('bidirectional/backward_rnn/lstm_cell/kernel:0', TensorShape([300, 800])),
 ('bidirectional/backward_rnn/lstm_cell/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional/backward_rnn/lstm_cell/bias:0', TensorShape([800])),
 ('bidirectional_1/forward_rnn_1/lstm_cell_1/kernel:0',
  TensorShape([400, 800])),
 ('bidirectional_1/forward_rnn_1/lstm_cell_1/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional_1/forward_rnn_1/lstm_cell_1/bias:0', TensorShape([800])),
 ('bidirectional_1/backward_rnn_1/lstm_cell_1/kernel:0',
  TensorShape([400, 800])),
 ('bidirectional_1/backward_rnn_1/lstm_cell_1/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional_1/backward_rnn_1/lstm_cell_1/bias:0', TensorShape([800])),
 ('attentive_pooling/dense/kernel:0', TensorShape([400, 200])),
 ('attentive_pooling/dense/bias:0', TensorShape([200])),
 ('attentive_pooling/dense_1/kernel:0', TensorShape([200, 1])),
 ('dense_2/kernel:0', TensorShape([400, 200])),
 ('dense_2/bias:0', TensorShape([200])),
 ('dense_3/kernel:0', TensorShape([200, 2])),
 ('dense_3/bias:0', TensorShape([2]))]
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 0 | Loss: 0.7000 | Spent: 98.3 secs | LR: 0.000100
INFO:tensorflow:Step 50 | Loss: 0.6774 | Spent: 28.5 secs | LR: 0.000103
INFO:tensorflow:Step 100 | Loss: 0.6125 | Spent: 28.3 secs | LR: 0.000106
INFO:tensorflow:Step 150 | Loss: 0.5822 | Spent: 28.4 secs | LR: 0.000108
INFO:tensorflow:Step 200 | Loss: 0.5467 | Spent: 28.4 secs | LR: 0.000111
INFO:tensorflow:Step 250 | Loss: 0.5582 | Spent: 28.5 secs | LR: 0.000114
INFO:tensorflow:Step 300 | Loss: 0.5438 | Spent: 28.2 secs | LR: 0.000117
INFO:tensorflow:Step 350 | Loss: 0.4846 | Spent: 28.7 secs | LR: 0.000120
INFO:tensorflow:Step 400 | Loss: 0.5384 | Spent: 28.1 secs | LR: 0.000122
INFO:tensorflow:Step 450 | Loss: 0.5290 | Spent: 28.6 secs | LR: 0.000125
INFO:tensorflow:Step 500 | Loss: 0.5467 | Spent: 28.4 secs | LR: 0.000128
INFO:tensorflow:Step 550 | Loss: 0.4820 | Spent: 28.7 secs | LR: 0.000131
INFO:tensorflow:Step 600 | Loss: 0.4314 | Spent: 28.5 secs | LR: 0.000134
INFO:tensorflow:Step 650 | Loss: 0.4185 | Spent: 28.5 secs | LR: 0.000136
INFO:tensorflow:Step 700 | Loss: 0.5147 | Spent: 28.6 secs | LR: 0.000139
INFO:tensorflow:Step 750 | Loss: 0.4576 | Spent: 28.5 secs | LR: 0.000142
INFO:tensorflow:Step 800 | Loss: 0.4762 | Spent: 28.8 secs | LR: 0.000145
INFO:tensorflow:Step 850 | Loss: 0.5399 | Spent: 28.7 secs | LR: 0.000148
INFO:tensorflow:Step 900 | Loss: 0.4634 | Spent: 28.7 secs | LR: 0.000150
INFO:tensorflow:Step 950 | Loss: 0.4480 | Spent: 28.9 secs | LR: 0.000153
INFO:tensorflow:Step 1000 | Loss: 0.5081 | Spent: 28.5 secs | LR: 0.000156
INFO:tensorflow:Step 1050 | Loss: 0.4625 | Spent: 28.7 secs | LR: 0.000159
INFO:tensorflow:Step 1100 | Loss: 0.5780 | Spent: 28.9 secs | LR: 0.000162
INFO:tensorflow:Step 1150 | Loss: 0.4784 | Spent: 28.6 secs | LR: 0.000164
INFO:tensorflow:Step 1200 | Loss: 0.5698 | Spent: 28.6 secs | LR: 0.000167
INFO:tensorflow:Step 1250 | Loss: 0.4349 | Spent: 28.8 secs | LR: 0.000170
INFO:tensorflow:Step 1300 | Loss: 0.4878 | Spent: 28.8 secs | LR: 0.000173
INFO:tensorflow:Step 1350 | Loss: 0.4752 | Spent: 28.7 secs | LR: 0.000176
INFO:tensorflow:Step 1400 | Loss: 0.4651 | Spent: 28.6 secs | LR: 0.000178
INFO:tensorflow:Step 1450 | Loss: 0.4458 | Spent: 28.8 secs | LR: 0.000181
INFO:tensorflow:Step 1500 | Loss: 0.4663 | Spent: 28.7 secs | LR: 0.000184
INFO:tensorflow:Step 1550 | Loss: 0.5406 | Spent: 28.6 secs | LR: 0.000187
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.877
INFO:tensorflow:Best Accuracy: 0.877
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 1600 | Loss: 0.5629 | Spent: 292.4 secs | LR: 0.000190
INFO:tensorflow:Step 1650 | Loss: 0.4835 | Spent: 28.8 secs | LR: 0.000192
INFO:tensorflow:Step 1700 | Loss: 0.3801 | Spent: 28.8 secs | LR: 0.000195
INFO:tensorflow:Step 1750 | Loss: 0.4362 | Spent: 28.6 secs | LR: 0.000198
INFO:tensorflow:Step 1800 | Loss: 0.4458 | Spent: 28.3 secs | LR: 0.000201
INFO:tensorflow:Step 1850 | Loss: 0.4160 | Spent: 28.4 secs | LR: 0.000204
INFO:tensorflow:Step 1900 | Loss: 0.5501 | Spent: 28.6 secs | LR: 0.000206
INFO:tensorflow:Step 1950 | Loss: 0.4967 | Spent: 28.6 secs | LR: 0.000209
INFO:tensorflow:Step 2000 | Loss: 0.4726 | Spent: 28.7 secs | LR: 0.000212
INFO:tensorflow:Step 2050 | Loss: 0.5788 | Spent: 28.5 secs | LR: 0.000215
INFO:tensorflow:Step 2100 | Loss: 0.5582 | Spent: 28.7 secs | LR: 0.000218
INFO:tensorflow:Step 2150 | Loss: 0.5993 | Spent: 28.6 secs | LR: 0.000220
INFO:tensorflow:Step 2200 | Loss: 0.5141 | Spent: 28.9 secs | LR: 0.000223
INFO:tensorflow:Step 2250 | Loss: 0.4052 | Spent: 28.7 secs | LR: 0.000226
INFO:tensorflow:Step 2300 | Loss: 0.4746 | Spent: 28.6 secs | LR: 0.000229
INFO:tensorflow:Step 2350 | Loss: 0.4895 | Spent: 28.4 secs | LR: 0.000232
INFO:tensorflow:Step 2400 | Loss: 0.4400 | Spent: 28.6 secs | LR: 0.000234
INFO:tensorflow:Step 2450 | Loss: 0.5115 | Spent: 28.5 secs | LR: 0.000237
INFO:tensorflow:Step 2500 | Loss: 0.4481 | Spent: 28.5 secs | LR: 0.000240
INFO:tensorflow:Step 2550 | Loss: 0.4211 | Spent: 28.7 secs | LR: 0.000243
INFO:tensorflow:Step 2600 | Loss: 0.4799 | Spent: 28.7 secs | LR: 0.000246
INFO:tensorflow:Step 2650 | Loss: 0.5622 | Spent: 28.5 secs | LR: 0.000248
INFO:tensorflow:Step 2700 | Loss: 0.4387 | Spent: 28.6 secs | LR: 0.000251
INFO:tensorflow:Step 2750 | Loss: 0.5964 | Spent: 28.4 secs | LR: 0.000254
INFO:tensorflow:Step 2800 | Loss: 0.5568 | Spent: 28.5 secs | LR: 0.000257
INFO:tensorflow:Step 2850 | Loss: 0.4750 | Spent: 28.4 secs | LR: 0.000260
INFO:tensorflow:Step 2900 | Loss: 0.4116 | Spent: 28.2 secs | LR: 0.000262
INFO:tensorflow:Step 2950 | Loss: 0.6100 | Spent: 28.7 secs | LR: 0.000265
INFO:tensorflow:Step 3000 | Loss: 0.5057 | Spent: 28.2 secs | LR: 0.000268
INFO:tensorflow:Step 3050 | Loss: 0.3723 | Spent: 28.3 secs | LR: 0.000271
INFO:tensorflow:Step 3100 | Loss: 0.4453 | Spent: 28.3 secs | LR: 0.000274
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.873
INFO:tensorflow:Best Accuracy: 0.877
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 3150 | Loss: 0.4977 | Spent: 283.3 secs | LR: 0.000276
INFO:tensorflow:Step 3200 | Loss: 0.4588 | Spent: 28.5 secs | LR: 0.000279
INFO:tensorflow:Step 3250 | Loss: 0.3852 | Spent: 28.1 secs | LR: 0.000282
INFO:tensorflow:Step 3300 | Loss: 0.4084 | Spent: 28.5 secs | LR: 0.000285
INFO:tensorflow:Step 3350 | Loss: 0.3894 | Spent: 28.0 secs | LR: 0.000288
INFO:tensorflow:Step 3400 | Loss: 0.5035 | Spent: 27.9 secs | LR: 0.000290
INFO:tensorflow:Step 3450 | Loss: 0.4652 | Spent: 27.9 secs | LR: 0.000293
INFO:tensorflow:Step 3500 | Loss: 0.4383 | Spent: 28.1 secs | LR: 0.000296
INFO:tensorflow:Step 3550 | Loss: 0.5017 | Spent: 28.0 secs | LR: 0.000299
INFO:tensorflow:Step 3600 | Loss: 0.4219 | Spent: 27.9 secs | LR: 0.000302
INFO:tensorflow:Step 3650 | Loss: 0.4479 | Spent: 28.2 secs | LR: 0.000304
INFO:tensorflow:Step 3700 | Loss: 0.4202 | Spent: 28.1 secs | LR: 0.000307
INFO:tensorflow:Step 3750 | Loss: 0.4489 | Spent: 27.8 secs | LR: 0.000310
INFO:tensorflow:Step 3800 | Loss: 0.4866 | Spent: 27.9 secs | LR: 0.000313
INFO:tensorflow:Step 3850 | Loss: 0.4839 | Spent: 28.1 secs | LR: 0.000316
INFO:tensorflow:Step 3900 | Loss: 0.3884 | Spent: 27.9 secs | LR: 0.000318
INFO:tensorflow:Step 3950 | Loss: 0.4727 | Spent: 27.9 secs | LR: 0.000321
INFO:tensorflow:Step 4000 | Loss: 0.3938 | Spent: 27.8 secs | LR: 0.000324
INFO:tensorflow:Step 4050 | Loss: 0.4978 | Spent: 28.3 secs | LR: 0.000327
INFO:tensorflow:Step 4100 | Loss: 0.4883 | Spent: 27.8 secs | LR: 0.000330
INFO:tensorflow:Step 4150 | Loss: 0.4106 | Spent: 28.0 secs | LR: 0.000332
INFO:tensorflow:Step 4200 | Loss: 0.4595 | Spent: 28.2 secs | LR: 0.000335
INFO:tensorflow:Step 4250 | Loss: 0.3895 | Spent: 27.7 secs | LR: 0.000338
INFO:tensorflow:Step 4300 | Loss: 0.4003 | Spent: 27.7 secs | LR: 0.000341
INFO:tensorflow:Step 4350 | Loss: 0.5231 | Spent: 28.2 secs | LR: 0.000344
INFO:tensorflow:Step 4400 | Loss: 0.4436 | Spent: 28.0 secs | LR: 0.000346
INFO:tensorflow:Step 4450 | Loss: 0.4177 | Spent: 28.2 secs | LR: 0.000349
INFO:tensorflow:Step 4500 | Loss: 0.4082 | Spent: 27.8 secs | LR: 0.000352
INFO:tensorflow:Step 4550 | Loss: 0.4124 | Spent: 27.7 secs | LR: 0.000355
INFO:tensorflow:Step 4600 | Loss: 0.4272 | Spent: 27.7 secs | LR: 0.000358
INFO:tensorflow:Step 4650 | Loss: 0.4455 | Spent: 27.8 secs | LR: 0.000360
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.911
INFO:tensorflow:Best Accuracy: 0.911
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 4700 | Loss: 0.5045 | Spent: 278.6 secs | LR: 0.000363
INFO:tensorflow:Step 4750 | Loss: 0.5369 | Spent: 27.9 secs | LR: 0.000366
INFO:tensorflow:Step 4800 | Loss: 0.4280 | Spent: 27.5 secs | LR: 0.000369
INFO:tensorflow:Step 4850 | Loss: 0.4434 | Spent: 27.9 secs | LR: 0.000372
INFO:tensorflow:Step 4900 | Loss: 0.4300 | Spent: 27.9 secs | LR: 0.000374
INFO:tensorflow:Step 4950 | Loss: 0.4065 | Spent: 27.8 secs | LR: 0.000377
INFO:tensorflow:Step 5000 | Loss: 0.3871 | Spent: 27.7 secs | LR: 0.000380
INFO:tensorflow:Step 5050 | Loss: 0.3979 | Spent: 28.4 secs | LR: 0.000383
INFO:tensorflow:Step 5100 | Loss: 0.3845 | Spent: 28.2 secs | LR: 0.000386
INFO:tensorflow:Step 5150 | Loss: 0.4062 | Spent: 28.2 secs | LR: 0.000388
INFO:tensorflow:Step 5200 | Loss: 0.4334 | Spent: 27.8 secs | LR: 0.000391
INFO:tensorflow:Step 5250 | Loss: 0.4712 | Spent: 28.2 secs | LR: 0.000394
INFO:tensorflow:Step 5300 | Loss: 0.5205 | Spent: 28.0 secs | LR: 0.000397
INFO:tensorflow:Step 5350 | Loss: 0.4102 | Spent: 28.1 secs | LR: 0.000400
INFO:tensorflow:Step 5400 | Loss: 0.4548 | Spent: 28.1 secs | LR: 0.000402
INFO:tensorflow:Step 5450 | Loss: 0.4704 | Spent: 28.2 secs | LR: 0.000405
INFO:tensorflow:Step 5500 | Loss: 0.4103 | Spent: 28.0 secs | LR: 0.000408
INFO:tensorflow:Step 5550 | Loss: 0.4477 | Spent: 27.9 secs | LR: 0.000411
INFO:tensorflow:Step 5600 | Loss: 0.4122 | Spent: 28.1 secs | LR: 0.000414
INFO:tensorflow:Step 5650 | Loss: 0.4697 | Spent: 28.3 secs | LR: 0.000416
INFO:tensorflow:Step 5700 | Loss: 0.3973 | Spent: 28.2 secs | LR: 0.000419
INFO:tensorflow:Step 5750 | Loss: 0.4069 | Spent: 28.2 secs | LR: 0.000422
INFO:tensorflow:Step 5800 | Loss: 0.3901 | Spent: 28.0 secs | LR: 0.000425
INFO:tensorflow:Step 5850 | Loss: 0.5813 | Spent: 28.5 secs | LR: 0.000428
INFO:tensorflow:Step 5900 | Loss: 0.4726 | Spent: 28.7 secs | LR: 0.000430
INFO:tensorflow:Step 5950 | Loss: 0.4035 | Spent: 28.2 secs | LR: 0.000433
INFO:tensorflow:Step 6000 | Loss: 0.4349 | Spent: 28.2 secs | LR: 0.000436
INFO:tensorflow:Step 6050 | Loss: 0.4264 | Spent: 28.8 secs | LR: 0.000439
INFO:tensorflow:Step 6100 | Loss: 0.3864 | Spent: 28.5 secs | LR: 0.000442
INFO:tensorflow:Step 6150 | Loss: 0.4551 | Spent: 28.6 secs | LR: 0.000444
INFO:tensorflow:Step 6200 | Loss: 0.4874 | Spent: 28.2 secs | LR: 0.000447
INFO:tensorflow:Step 6250 | Loss: 0.4391 | Spent: 28.3 secs | LR: 0.000450
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.906
INFO:tensorflow:Best Accuracy: 0.911
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 6300 | Loss: 0.3867 | Spent: 287.7 secs | LR: 0.000453
INFO:tensorflow:Step 6350 | Loss: 0.3930 | Spent: 27.9 secs | LR: 0.000456
INFO:tensorflow:Step 6400 | Loss: 0.4357 | Spent: 28.3 secs | LR: 0.000458
INFO:tensorflow:Step 6450 | Loss: 0.3593 | Spent: 28.2 secs | LR: 0.000461
INFO:tensorflow:Step 6500 | Loss: 0.4115 | Spent: 28.1 secs | LR: 0.000464
INFO:tensorflow:Step 6550 | Loss: 0.4912 | Spent: 28.1 secs | LR: 0.000467
INFO:tensorflow:Step 6600 | Loss: 0.4562 | Spent: 28.5 secs | LR: 0.000470
INFO:tensorflow:Step 6650 | Loss: 0.6390 | Spent: 28.4 secs | LR: 0.000472
INFO:tensorflow:Step 6700 | Loss: 0.4881 | Spent: 28.5 secs | LR: 0.000475
INFO:tensorflow:Step 6750 | Loss: 0.4823 | Spent: 28.6 secs | LR: 0.000478
INFO:tensorflow:Step 6800 | Loss: 0.3777 | Spent: 28.4 secs | LR: 0.000481
INFO:tensorflow:Step 6850 | Loss: 0.4298 | Spent: 28.3 secs | LR: 0.000484
INFO:tensorflow:Step 6900 | Loss: 0.4022 | Spent: 28.7 secs | LR: 0.000486
INFO:tensorflow:Step 6950 | Loss: 0.4245 | Spent: 28.3 secs | LR: 0.000489
INFO:tensorflow:Step 7000 | Loss: 0.5003 | Spent: 28.5 secs | LR: 0.000492
INFO:tensorflow:Step 7050 | Loss: 0.5874 | Spent: 28.4 secs | LR: 0.000495
INFO:tensorflow:Step 7100 | Loss: 0.3980 | Spent: 28.5 secs | LR: 0.000498
INFO:tensorflow:Step 7150 | Loss: 0.4795 | Spent: 28.4 secs | LR: 0.000500
INFO:tensorflow:Step 7200 | Loss: 0.4265 | Spent: 28.3 secs | LR: 0.000503
INFO:tensorflow:Step 7250 | Loss: 0.3851 | Spent: 28.4 secs | LR: 0.000506
INFO:tensorflow:Step 7300 | Loss: 0.4219 | Spent: 28.7 secs | LR: 0.000509
INFO:tensorflow:Step 7350 | Loss: 0.4689 | Spent: 28.3 secs | LR: 0.000512
INFO:tensorflow:Step 7400 | Loss: 0.3762 | Spent: 28.2 secs | LR: 0.000514
INFO:tensorflow:Step 7450 | Loss: 0.3792 | Spent: 27.9 secs | LR: 0.000517
INFO:tensorflow:Step 7500 | Loss: 0.4915 | Spent: 27.9 secs | LR: 0.000520
INFO:tensorflow:Step 7550 | Loss: 0.3773 | Spent: 27.8 secs | LR: 0.000523
INFO:tensorflow:Step 7600 | Loss: 0.5528 | Spent: 27.8 secs | LR: 0.000526
INFO:tensorflow:Step 7650 | Loss: 0.3688 | Spent: 27.9 secs | LR: 0.000528
INFO:tensorflow:Step 7700 | Loss: 0.4555 | Spent: 28.0 secs | LR: 0.000531
INFO:tensorflow:Step 7750 | Loss: 0.3938 | Spent: 27.9 secs | LR: 0.000534
INFO:tensorflow:Step 7800 | Loss: 0.4709 | Spent: 27.9 secs | LR: 0.000537
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.918
INFO:tensorflow:Best Accuracy: 0.918
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 7850 | Loss: 0.4368 | Spent: 279.0 secs | LR: 0.000540
INFO:tensorflow:Step 7900 | Loss: 0.4492 | Spent: 27.8 secs | LR: 0.000542
INFO:tensorflow:Step 7950 | Loss: 0.3902 | Spent: 27.6 secs | LR: 0.000545
INFO:tensorflow:Step 8000 | Loss: 0.4853 | Spent: 28.1 secs | LR: 0.000548
INFO:tensorflow:Step 8050 | Loss: 0.4905 | Spent: 27.8 secs | LR: 0.000551
INFO:tensorflow:Step 8100 | Loss: 0.5081 | Spent: 28.0 secs | LR: 0.000554
INFO:tensorflow:Step 8150 | Loss: 0.4537 | Spent: 27.9 secs | LR: 0.000556
INFO:tensorflow:Step 8200 | Loss: 0.4139 | Spent: 28.0 secs | LR: 0.000559
INFO:tensorflow:Step 8250 | Loss: 0.4239 | Spent: 28.0 secs | LR: 0.000562
INFO:tensorflow:Step 8300 | Loss: 0.4082 | Spent: 27.9 secs | LR: 0.000565
INFO:tensorflow:Step 8350 | Loss: 0.4677 | Spent: 28.0 secs | LR: 0.000568
INFO:tensorflow:Step 8400 | Loss: 0.4212 | Spent: 27.8 secs | LR: 0.000570
INFO:tensorflow:Step 8450 | Loss: 0.4620 | Spent: 27.8 secs | LR: 0.000573
INFO:tensorflow:Step 8500 | Loss: 0.3796 | Spent: 27.9 secs | LR: 0.000576
INFO:tensorflow:Step 8550 | Loss: 0.4504 | Spent: 27.8 secs | LR: 0.000579
INFO:tensorflow:Step 8600 | Loss: 0.4991 | Spent: 28.0 secs | LR: 0.000582
INFO:tensorflow:Step 8650 | Loss: 0.4276 | Spent: 27.8 secs | LR: 0.000584
INFO:tensorflow:Step 8700 | Loss: 0.3599 | Spent: 28.1 secs | LR: 0.000587
INFO:tensorflow:Step 8750 | Loss: 0.3849 | Spent: 27.7 secs | LR: 0.000590
INFO:tensorflow:Step 8800 | Loss: 0.3517 | Spent: 27.8 secs | LR: 0.000593
INFO:tensorflow:Step 8850 | Loss: 0.3635 | Spent: 27.7 secs | LR: 0.000596
INFO:tensorflow:Step 8900 | Loss: 0.3594 | Spent: 27.9 secs | LR: 0.000598
INFO:tensorflow:Step 8950 | Loss: 0.4141 | Spent: 27.8 secs | LR: 0.000601
INFO:tensorflow:Step 9000 | Loss: 0.4631 | Spent: 27.9 secs | LR: 0.000604
INFO:tensorflow:Step 9050 | Loss: 0.4436 | Spent: 27.7 secs | LR: 0.000607
INFO:tensorflow:Step 9100 | Loss: 0.4145 | Spent: 27.8 secs | LR: 0.000610
INFO:tensorflow:Step 9150 | Loss: 0.3465 | Spent: 28.1 secs | LR: 0.000612
INFO:tensorflow:Step 9200 | Loss: 0.3651 | Spent: 27.8 secs | LR: 0.000615
INFO:tensorflow:Step 9250 | Loss: 0.3462 | Spent: 27.8 secs | LR: 0.000618
INFO:tensorflow:Step 9300 | Loss: 0.4240 | Spent: 27.9 secs | LR: 0.000621
INFO:tensorflow:Step 9350 | Loss: 0.4269 | Spent: 27.8 secs | LR: 0.000624
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.914
INFO:tensorflow:Best Accuracy: 0.918
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 9400 | Loss: 0.3717 | Spent: 278.6 secs | LR: 0.000626
INFO:tensorflow:Step 9450 | Loss: 0.3813 | Spent: 27.9 secs | LR: 0.000629
INFO:tensorflow:Step 9500 | Loss: 0.4651 | Spent: 27.9 secs | LR: 0.000632
INFO:tensorflow:Step 9550 | Loss: 0.3714 | Spent: 28.0 secs | LR: 0.000635
INFO:tensorflow:Step 9600 | Loss: 0.4042 | Spent: 27.9 secs | LR: 0.000638
INFO:tensorflow:Step 9650 | Loss: 0.4008 | Spent: 28.0 secs | LR: 0.000640
INFO:tensorflow:Step 9700 | Loss: 0.4202 | Spent: 27.9 secs | LR: 0.000643
INFO:tensorflow:Step 9750 | Loss: 0.3629 | Spent: 27.9 secs | LR: 0.000646
INFO:tensorflow:Step 9800 | Loss: 0.4674 | Spent: 28.0 secs | LR: 0.000649
INFO:tensorflow:Step 9850 | Loss: 0.4711 | Spent: 28.0 secs | LR: 0.000652
INFO:tensorflow:Step 9900 | Loss: 0.4032 | Spent: 27.6 secs | LR: 0.000654
INFO:tensorflow:Step 9950 | Loss: 0.5209 | Spent: 27.9 secs | LR: 0.000657
INFO:tensorflow:Step 10000 | Loss: 0.4793 | Spent: 28.0 secs | LR: 0.000660
INFO:tensorflow:Step 10050 | Loss: 0.3611 | Spent: 27.9 secs | LR: 0.000663
INFO:tensorflow:Step 10100 | Loss: 0.3604 | Spent: 27.8 secs | LR: 0.000666
INFO:tensorflow:Step 10150 | Loss: 0.4691 | Spent: 28.0 secs | LR: 0.000668
INFO:tensorflow:Step 10200 | Loss: 0.4049 | Spent: 27.9 secs | LR: 0.000671
INFO:tensorflow:Step 10250 | Loss: 0.3527 | Spent: 27.8 secs | LR: 0.000674
INFO:tensorflow:Step 10300 | Loss: 0.3412 | Spent: 27.8 secs | LR: 0.000677
INFO:tensorflow:Step 10350 | Loss: 0.4511 | Spent: 28.1 secs | LR: 0.000680
INFO:tensorflow:Step 10400 | Loss: 0.3668 | Spent: 27.9 secs | LR: 0.000682
INFO:tensorflow:Step 10450 | Loss: 0.4413 | Spent: 27.9 secs | LR: 0.000685
INFO:tensorflow:Step 10500 | Loss: 0.3824 | Spent: 27.9 secs | LR: 0.000688
INFO:tensorflow:Step 10550 | Loss: 0.4305 | Spent: 28.2 secs | LR: 0.000691
INFO:tensorflow:Step 10600 | Loss: 0.5338 | Spent: 28.2 secs | LR: 0.000694
INFO:tensorflow:Step 10650 | Loss: 0.4278 | Spent: 28.0 secs | LR: 0.000696
INFO:tensorflow:Step 10700 | Loss: 0.3350 | Spent: 28.0 secs | LR: 0.000699
INFO:tensorflow:Step 10750 | Loss: 0.4536 | Spent: 27.9 secs | LR: 0.000702
INFO:tensorflow:Step 10800 | Loss: 0.5316 | Spent: 28.1 secs | LR: 0.000705
INFO:tensorflow:Step 10850 | Loss: 0.3707 | Spent: 27.9 secs | LR: 0.000708
INFO:tensorflow:Step 10900 | Loss: 0.4578 | Spent: 28.3 secs | LR: 0.000710
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.912
INFO:tensorflow:Best Accuracy: 0.918
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 10950 | Loss: 0.4732 | Spent: 280.9 secs | LR: 0.000713
INFO:tensorflow:Step 11000 | Loss: 0.4611 | Spent: 27.9 secs | LR: 0.000716
INFO:tensorflow:Step 11050 | Loss: 0.4229 | Spent: 27.8 secs | LR: 0.000719
INFO:tensorflow:Step 11100 | Loss: 0.5667 | Spent: 28.0 secs | LR: 0.000722
INFO:tensorflow:Step 11150 | Loss: 0.4370 | Spent: 28.0 secs | LR: 0.000724
INFO:tensorflow:Step 11200 | Loss: 0.4696 | Spent: 28.0 secs | LR: 0.000727
INFO:tensorflow:Step 11250 | Loss: 0.3546 | Spent: 27.7 secs | LR: 0.000730
INFO:tensorflow:Step 11300 | Loss: 0.4027 | Spent: 27.9 secs | LR: 0.000733
INFO:tensorflow:Step 11350 | Loss: 0.3693 | Spent: 28.0 secs | LR: 0.000736
INFO:tensorflow:Step 11400 | Loss: 0.4506 | Spent: 27.8 secs | LR: 0.000738
INFO:tensorflow:Step 11450 | Loss: 0.4729 | Spent: 27.9 secs | LR: 0.000741
INFO:tensorflow:Step 11500 | Loss: 0.4308 | Spent: 28.2 secs | LR: 0.000744
INFO:tensorflow:Step 11550 | Loss: 0.3601 | Spent: 27.9 secs | LR: 0.000747
INFO:tensorflow:Step 11600 | Loss: 0.3500 | Spent: 27.8 secs | LR: 0.000750
INFO:tensorflow:Step 11650 | Loss: 0.4226 | Spent: 28.1 secs | LR: 0.000752
INFO:tensorflow:Step 11700 | Loss: 0.3849 | Spent: 27.8 secs | LR: 0.000755
INFO:tensorflow:Step 11750 | Loss: 0.4203 | Spent: 27.8 secs | LR: 0.000758
INFO:tensorflow:Step 11800 | Loss: 0.4800 | Spent: 28.0 secs | LR: 0.000761
INFO:tensorflow:Step 11850 | Loss: 0.3584 | Spent: 27.9 secs | LR: 0.000764
INFO:tensorflow:Step 11900 | Loss: 0.4059 | Spent: 27.6 secs | LR: 0.000766
INFO:tensorflow:Step 11950 | Loss: 0.3717 | Spent: 27.8 secs | LR: 0.000769
INFO:tensorflow:Step 12000 | Loss: 0.3385 | Spent: 28.2 secs | LR: 0.000772
INFO:tensorflow:Step 12050 | Loss: 0.4892 | Spent: 27.8 secs | LR: 0.000775
INFO:tensorflow:Step 12100 | Loss: 0.4279 | Spent: 27.9 secs | LR: 0.000778
INFO:tensorflow:Step 12150 | Loss: 0.4969 | Spent: 27.7 secs | LR: 0.000780
INFO:tensorflow:Step 12200 | Loss: 0.3580 | Spent: 27.7 secs | LR: 0.000783
INFO:tensorflow:Step 12250 | Loss: 0.5453 | Spent: 28.0 secs | LR: 0.000786
INFO:tensorflow:Step 12300 | Loss: 0.4181 | Spent: 28.0 secs | LR: 0.000789
INFO:tensorflow:Step 12350 | Loss: 0.3687 | Spent: 27.8 secs | LR: 0.000792
INFO:tensorflow:Step 12400 | Loss: 0.4727 | Spent: 27.9 secs | LR: 0.000794
INFO:tensorflow:Step 12450 | Loss: 0.3472 | Spent: 28.0 secs | LR: 0.000797
INFO:tensorflow:Step 12500 | Loss: 0.3664 | Spent: 28.1 secs | LR: 0.000800
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.914
INFO:tensorflow:Best Accuracy: 0.918
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 12550 | Loss: 0.3543 | Spent: 280.0 secs | LR: 0.000797
INFO:tensorflow:Step 12600 | Loss: 0.3405 | Spent: 27.8 secs | LR: 0.000794
INFO:tensorflow:Step 12650 | Loss: 0.4488 | Spent: 27.9 secs | LR: 0.000792
INFO:tensorflow:Step 12700 | Loss: 0.3304 | Spent: 27.8 secs | LR: 0.000789
INFO:tensorflow:Step 12750 | Loss: 0.4304 | Spent: 28.1 secs | LR: 0.000786
INFO:tensorflow:Step 12800 | Loss: 0.4145 | Spent: 28.3 secs | LR: 0.000783
INFO:tensorflow:Step 12850 | Loss: 0.4149 | Spent: 28.0 secs | LR: 0.000780
INFO:tensorflow:Step 12900 | Loss: 0.3416 | Spent: 27.8 secs | LR: 0.000778
INFO:tensorflow:Step 12950 | Loss: 0.4161 | Spent: 28.2 secs | LR: 0.000775
INFO:tensorflow:Step 13000 | Loss: 0.3966 | Spent: 28.1 secs | LR: 0.000772
INFO:tensorflow:Step 13050 | Loss: 0.3772 | Spent: 28.1 secs | LR: 0.000769
INFO:tensorflow:Step 13100 | Loss: 0.4504 | Spent: 28.1 secs | LR: 0.000766
INFO:tensorflow:Step 13150 | Loss: 0.3500 | Spent: 28.2 secs | LR: 0.000764
INFO:tensorflow:Step 13200 | Loss: 0.4135 | Spent: 28.0 secs | LR: 0.000761
INFO:tensorflow:Step 13250 | Loss: 0.3503 | Spent: 28.1 secs | LR: 0.000758
INFO:tensorflow:Step 13300 | Loss: 0.5135 | Spent: 28.0 secs | LR: 0.000755
INFO:tensorflow:Step 13350 | Loss: 0.3673 | Spent: 28.2 secs | LR: 0.000752
INFO:tensorflow:Step 13400 | Loss: 0.3432 | Spent: 28.0 secs | LR: 0.000750
INFO:tensorflow:Step 13450 | Loss: 0.3574 | Spent: 27.8 secs | LR: 0.000747
INFO:tensorflow:Step 13500 | Loss: 0.4089 | Spent: 27.8 secs | LR: 0.000744
INFO:tensorflow:Step 13550 | Loss: 0.4004 | Spent: 27.9 secs | LR: 0.000741
INFO:tensorflow:Step 13600 | Loss: 0.3830 | Spent: 27.9 secs | LR: 0.000738
INFO:tensorflow:Step 13650 | Loss: 0.3664 | Spent: 27.8 secs | LR: 0.000736
INFO:tensorflow:Step 13700 | Loss: 0.4182 | Spent: 28.0 secs | LR: 0.000733
INFO:tensorflow:Step 13750 | Loss: 0.3830 | Spent: 27.8 secs | LR: 0.000730
INFO:tensorflow:Step 13800 | Loss: 0.3753 | Spent: 27.9 secs | LR: 0.000727
INFO:tensorflow:Step 13850 | Loss: 0.4003 | Spent: 27.8 secs | LR: 0.000724
INFO:tensorflow:Step 13900 | Loss: 0.4691 | Spent: 28.0 secs | LR: 0.000722
INFO:tensorflow:Step 13950 | Loss: 0.3669 | Spent: 28.0 secs | LR: 0.000719
INFO:tensorflow:Step 14000 | Loss: 0.4280 | Spent: 28.3 secs | LR: 0.000716
INFO:tensorflow:Step 14050 | Loss: 0.4100 | Spent: 27.7 secs | LR: 0.000713
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.923
INFO:tensorflow:Best Accuracy: 0.923
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 14100 | Loss: 0.4738 | Spent: 280.3 secs | LR: 0.000710
INFO:tensorflow:Step 14150 | Loss: 0.3375 | Spent: 28.1 secs | LR: 0.000708
INFO:tensorflow:Step 14200 | Loss: 0.3518 | Spent: 27.9 secs | LR: 0.000705
INFO:tensorflow:Step 14250 | Loss: 0.3984 | Spent: 27.9 secs | LR: 0.000702
INFO:tensorflow:Step 14300 | Loss: 0.4110 | Spent: 27.9 secs | LR: 0.000699
INFO:tensorflow:Step 14350 | Loss: 0.3819 | Spent: 27.9 secs | LR: 0.000696
INFO:tensorflow:Step 14400 | Loss: 0.3370 | Spent: 28.0 secs | LR: 0.000694
INFO:tensorflow:Step 14450 | Loss: 0.4128 | Spent: 28.2 secs | LR: 0.000691
INFO:tensorflow:Step 14500 | Loss: 0.4062 | Spent: 27.9 secs | LR: 0.000688
INFO:tensorflow:Step 14550 | Loss: 0.3725 | Spent: 28.2 secs | LR: 0.000685
INFO:tensorflow:Step 14600 | Loss: 0.4953 | Spent: 28.1 secs | LR: 0.000682
INFO:tensorflow:Step 14650 | Loss: 0.3814 | Spent: 28.0 secs | LR: 0.000680
INFO:tensorflow:Step 14700 | Loss: 0.4623 | Spent: 28.0 secs | LR: 0.000677
INFO:tensorflow:Step 14750 | Loss: 0.4020 | Spent: 28.2 secs | LR: 0.000674
INFO:tensorflow:Step 14800 | Loss: 0.3417 | Spent: 28.0 secs | LR: 0.000671
INFO:tensorflow:Step 14850 | Loss: 0.4269 | Spent: 27.7 secs | LR: 0.000668
INFO:tensorflow:Step 14900 | Loss: 0.4818 | Spent: 28.1 secs | LR: 0.000666
INFO:tensorflow:Step 14950 | Loss: 0.4853 | Spent: 27.8 secs | LR: 0.000663
INFO:tensorflow:Step 15000 | Loss: 0.4036 | Spent: 28.2 secs | LR: 0.000660
INFO:tensorflow:Step 15050 | Loss: 0.3699 | Spent: 28.0 secs | LR: 0.000657
INFO:tensorflow:Step 15100 | Loss: 0.4262 | Spent: 28.1 secs | LR: 0.000654
INFO:tensorflow:Step 15150 | Loss: 0.4916 | Spent: 27.8 secs | LR: 0.000652
INFO:tensorflow:Step 15200 | Loss: 0.3811 | Spent: 27.8 secs | LR: 0.000649
INFO:tensorflow:Step 15250 | Loss: 0.4438 | Spent: 28.1 secs | LR: 0.000646
INFO:tensorflow:Step 15300 | Loss: 0.4080 | Spent: 28.1 secs | LR: 0.000643
INFO:tensorflow:Step 15350 | Loss: 0.3412 | Spent: 28.0 secs | LR: 0.000640
INFO:tensorflow:Step 15400 | Loss: 0.4377 | Spent: 27.9 secs | LR: 0.000638
INFO:tensorflow:Step 15450 | Loss: 0.3595 | Spent: 28.0 secs | LR: 0.000635
INFO:tensorflow:Step 15500 | Loss: 0.4513 | Spent: 28.1 secs | LR: 0.000632
INFO:tensorflow:Step 15550 | Loss: 0.4011 | Spent: 28.0 secs | LR: 0.000629
INFO:tensorflow:Step 15600 | Loss: 0.5344 | Spent: 28.0 secs | LR: 0.000626
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.913
INFO:tensorflow:Best Accuracy: 0.923
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 15650 | Loss: 0.4053 | Spent: 282.0 secs | LR: 0.000624
INFO:tensorflow:Step 15700 | Loss: 0.4232 | Spent: 28.0 secs | LR: 0.000621
INFO:tensorflow:Step 15750 | Loss: 0.3828 | Spent: 28.0 secs | LR: 0.000618
INFO:tensorflow:Step 15800 | Loss: 0.4196 | Spent: 28.2 secs | LR: 0.000615
INFO:tensorflow:Step 15850 | Loss: 0.3423 | Spent: 28.1 secs | LR: 0.000612
INFO:tensorflow:Step 15900 | Loss: 0.3348 | Spent: 28.0 secs | LR: 0.000610
INFO:tensorflow:Step 15950 | Loss: 0.3609 | Spent: 28.0 secs | LR: 0.000607
INFO:tensorflow:Step 16000 | Loss: 0.3800 | Spent: 28.1 secs | LR: 0.000604
INFO:tensorflow:Step 16050 | Loss: 0.3437 | Spent: 28.0 secs | LR: 0.000601
INFO:tensorflow:Step 16100 | Loss: 0.4213 | Spent: 28.0 secs | LR: 0.000598
INFO:tensorflow:Step 16150 | Loss: 0.4279 | Spent: 28.0 secs | LR: 0.000596
INFO:tensorflow:Step 16200 | Loss: 0.3522 | Spent: 27.9 secs | LR: 0.000593
INFO:tensorflow:Step 16250 | Loss: 0.3686 | Spent: 27.9 secs | LR: 0.000590
INFO:tensorflow:Step 16300 | Loss: 0.4100 | Spent: 28.0 secs | LR: 0.000587
INFO:tensorflow:Step 16350 | Loss: 0.4193 | Spent: 28.0 secs | LR: 0.000584
INFO:tensorflow:Step 16400 | Loss: 0.4010 | Spent: 28.1 secs | LR: 0.000582
INFO:tensorflow:Step 16450 | Loss: 0.3952 | Spent: 27.9 secs | LR: 0.000579
INFO:tensorflow:Step 16500 | Loss: 0.3377 | Spent: 27.9 secs | LR: 0.000576
INFO:tensorflow:Step 16550 | Loss: 0.3739 | Spent: 28.0 secs | LR: 0.000573
INFO:tensorflow:Step 16600 | Loss: 0.4031 | Spent: 27.9 secs | LR: 0.000570
INFO:tensorflow:Step 16650 | Loss: 0.3745 | Spent: 28.0 secs | LR: 0.000568
INFO:tensorflow:Step 16700 | Loss: 0.4103 | Spent: 28.2 secs | LR: 0.000565
INFO:tensorflow:Step 16750 | Loss: 0.4012 | Spent: 27.9 secs | LR: 0.000562
INFO:tensorflow:Step 16800 | Loss: 0.4188 | Spent: 27.7 secs | LR: 0.000559
INFO:tensorflow:Step 16850 | Loss: 0.3569 | Spent: 28.2 secs | LR: 0.000556
INFO:tensorflow:Step 16900 | Loss: 0.3383 | Spent: 28.0 secs | LR: 0.000554
INFO:tensorflow:Step 16950 | Loss: 0.3963 | Spent: 28.3 secs | LR: 0.000551
INFO:tensorflow:Step 17000 | Loss: 0.4316 | Spent: 28.4 secs | LR: 0.000548
INFO:tensorflow:Step 17050 | Loss: 0.4520 | Spent: 28.4 secs | LR: 0.000545
INFO:tensorflow:Step 17100 | Loss: 0.3608 | Spent: 28.3 secs | LR: 0.000542
INFO:tensorflow:Step 17150 | Loss: 0.3579 | Spent: 28.0 secs | LR: 0.000540
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.923
INFO:tensorflow:Best Accuracy: 0.923
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 17200 | Loss: 0.3611 | Spent: 282.0 secs | LR: 0.000537
INFO:tensorflow:Step 17250 | Loss: 0.4380 | Spent: 28.1 secs | LR: 0.000534
INFO:tensorflow:Step 17300 | Loss: 0.4099 | Spent: 27.8 secs | LR: 0.000531
INFO:tensorflow:Step 17350 | Loss: 0.3808 | Spent: 27.8 secs | LR: 0.000528
INFO:tensorflow:Step 17400 | Loss: 0.4178 | Spent: 28.1 secs | LR: 0.000526
INFO:tensorflow:Step 17450 | Loss: 0.3979 | Spent: 28.1 secs | LR: 0.000523
INFO:tensorflow:Step 17500 | Loss: 0.4211 | Spent: 28.1 secs | LR: 0.000520
INFO:tensorflow:Step 17550 | Loss: 0.3892 | Spent: 28.1 secs | LR: 0.000517
INFO:tensorflow:Step 17600 | Loss: 0.3768 | Spent: 28.0 secs | LR: 0.000514
INFO:tensorflow:Step 17650 | Loss: 0.3388 | Spent: 28.0 secs | LR: 0.000512
INFO:tensorflow:Step 17700 | Loss: 0.3758 | Spent: 27.8 secs | LR: 0.000509
INFO:tensorflow:Step 17750 | Loss: 0.3461 | Spent: 28.1 secs | LR: 0.000506
INFO:tensorflow:Step 17800 | Loss: 0.3623 | Spent: 28.0 secs | LR: 0.000503
INFO:tensorflow:Step 17850 | Loss: 0.3636 | Spent: 28.0 secs | LR: 0.000500
INFO:tensorflow:Step 17900 | Loss: 0.4187 | Spent: 27.8 secs | LR: 0.000498
INFO:tensorflow:Step 17950 | Loss: 0.3303 | Spent: 28.1 secs | LR: 0.000495
INFO:tensorflow:Step 18000 | Loss: 0.3652 | Spent: 28.0 secs | LR: 0.000492
INFO:tensorflow:Step 18050 | Loss: 0.3751 | Spent: 28.1 secs | LR: 0.000489
INFO:tensorflow:Step 18100 | Loss: 0.3350 | Spent: 28.1 secs | LR: 0.000486
INFO:tensorflow:Step 18150 | Loss: 0.3433 | Spent: 28.1 secs | LR: 0.000484
INFO:tensorflow:Step 18200 | Loss: 0.3409 | Spent: 27.9 secs | LR: 0.000481
INFO:tensorflow:Step 18250 | Loss: 0.4300 | Spent: 28.1 secs | LR: 0.000478
INFO:tensorflow:Step 18300 | Loss: 0.4169 | Spent: 27.9 secs | LR: 0.000475
INFO:tensorflow:Step 18350 | Loss: 0.3726 | Spent: 27.8 secs | LR: 0.000472
INFO:tensorflow:Step 18400 | Loss: 0.3443 | Spent: 28.0 secs | LR: 0.000470
INFO:tensorflow:Step 18450 | Loss: 0.3269 | Spent: 28.0 secs | LR: 0.000467
INFO:tensorflow:Step 18500 | Loss: 0.3884 | Spent: 27.8 secs | LR: 0.000464
INFO:tensorflow:Step 18550 | Loss: 0.3705 | Spent: 28.0 secs | LR: 0.000461
INFO:tensorflow:Step 18600 | Loss: 0.3358 | Spent: 28.0 secs | LR: 0.000458
INFO:tensorflow:Step 18650 | Loss: 0.3330 | Spent: 28.2 secs | LR: 0.000456
INFO:tensorflow:Step 18700 | Loss: 0.3298 | Spent: 27.9 secs | LR: 0.000453
INFO:tensorflow:Step 18750 | Loss: 0.4032 | Spent: 27.8 secs | LR: 0.000450
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.921
INFO:tensorflow:Best Accuracy: 0.923
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 18800 | Loss: 0.3502 | Spent: 280.6 secs | LR: 0.000447
INFO:tensorflow:Step 18850 | Loss: 0.4296 | Spent: 27.8 secs | LR: 0.000444
INFO:tensorflow:Step 18900 | Loss: 0.3270 | Spent: 28.0 secs | LR: 0.000442
INFO:tensorflow:Step 18950 | Loss: 0.3975 | Spent: 27.9 secs | LR: 0.000439
INFO:tensorflow:Step 19000 | Loss: 0.3941 | Spent: 27.7 secs | LR: 0.000436
INFO:tensorflow:Step 19050 | Loss: 0.3780 | Spent: 27.9 secs | LR: 0.000433
INFO:tensorflow:Step 19100 | Loss: 0.3788 | Spent: 28.2 secs | LR: 0.000430
INFO:tensorflow:Step 19150 | Loss: 0.3874 | Spent: 27.9 secs | LR: 0.000428
INFO:tensorflow:Step 19200 | Loss: 0.3319 | Spent: 27.8 secs | LR: 0.000425
INFO:tensorflow:Step 19250 | Loss: 0.3541 | Spent: 27.6 secs | LR: 0.000422
INFO:tensorflow:Step 19300 | Loss: 0.3585 | Spent: 28.1 secs | LR: 0.000419
INFO:tensorflow:Step 19350 | Loss: 0.3365 | Spent: 28.1 secs | LR: 0.000416
INFO:tensorflow:Step 19400 | Loss: 0.4028 | Spent: 28.2 secs | LR: 0.000414
INFO:tensorflow:Step 19450 | Loss: 0.3278 | Spent: 28.0 secs | LR: 0.000411
INFO:tensorflow:Step 19500 | Loss: 0.3905 | Spent: 27.9 secs | LR: 0.000408
INFO:tensorflow:Step 19550 | Loss: 0.3313 | Spent: 27.8 secs | LR: 0.000405
INFO:tensorflow:Step 19600 | Loss: 0.4131 | Spent: 28.0 secs | LR: 0.000402
INFO:tensorflow:Step 19650 | Loss: 0.4598 | Spent: 27.9 secs | LR: 0.000400
INFO:tensorflow:Step 19700 | Loss: 0.3579 | Spent: 28.0 secs | LR: 0.000397
INFO:tensorflow:Step 19750 | Loss: 0.3452 | Spent: 27.8 secs | LR: 0.000394
INFO:tensorflow:Step 19800 | Loss: 0.3465 | Spent: 28.0 secs | LR: 0.000391
INFO:tensorflow:Step 19850 | Loss: 0.3327 | Spent: 28.0 secs | LR: 0.000388
INFO:tensorflow:Step 19900 | Loss: 0.3597 | Spent: 27.9 secs | LR: 0.000386
INFO:tensorflow:Step 19950 | Loss: 0.3506 | Spent: 28.1 secs | LR: 0.000383
INFO:tensorflow:Step 20000 | Loss: 0.3572 | Spent: 27.9 secs | LR: 0.000380
INFO:tensorflow:Step 20050 | Loss: 0.3661 | Spent: 27.8 secs | LR: 0.000377
INFO:tensorflow:Step 20100 | Loss: 0.3946 | Spent: 27.9 secs | LR: 0.000374
INFO:tensorflow:Step 20150 | Loss: 0.3732 | Spent: 27.9 secs | LR: 0.000372
INFO:tensorflow:Step 20200 | Loss: 0.4134 | Spent: 27.9 secs | LR: 0.000369
INFO:tensorflow:Step 20250 | Loss: 0.3403 | Spent: 28.2 secs | LR: 0.000366
INFO:tensorflow:Step 20300 | Loss: 0.3310 | Spent: 27.9 secs | LR: 0.000363
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.920
INFO:tensorflow:Best Accuracy: 0.923
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 20350 | Loss: 0.3268 | Spent: 279.8 secs | LR: 0.000360
INFO:tensorflow:Step 20400 | Loss: 0.3554 | Spent: 27.8 secs | LR: 0.000358
INFO:tensorflow:Step 20450 | Loss: 0.3398 | Spent: 28.2 secs | LR: 0.000355
INFO:tensorflow:Step 20500 | Loss: 0.3313 | Spent: 28.1 secs | LR: 0.000352
INFO:tensorflow:Step 20550 | Loss: 0.3287 | Spent: 27.9 secs | LR: 0.000349
INFO:tensorflow:Step 20600 | Loss: 0.3746 | Spent: 28.0 secs | LR: 0.000346
INFO:tensorflow:Step 20650 | Loss: 0.3748 | Spent: 27.9 secs | LR: 0.000344
INFO:tensorflow:Step 20700 | Loss: 0.3721 | Spent: 27.9 secs | LR: 0.000341
INFO:tensorflow:Step 20750 | Loss: 0.3535 | Spent: 28.1 secs | LR: 0.000338
INFO:tensorflow:Step 20800 | Loss: 0.3826 | Spent: 28.1 secs | LR: 0.000335
INFO:tensorflow:Step 20850 | Loss: 0.4521 | Spent: 28.0 secs | LR: 0.000332
INFO:tensorflow:Step 20900 | Loss: 0.3287 | Spent: 28.0 secs | LR: 0.000330
INFO:tensorflow:Step 20950 | Loss: 0.3274 | Spent: 28.0 secs | LR: 0.000327
INFO:tensorflow:Step 21000 | Loss: 0.4008 | Spent: 27.9 secs | LR: 0.000324
INFO:tensorflow:Step 21050 | Loss: 0.4316 | Spent: 28.1 secs | LR: 0.000321
INFO:tensorflow:Step 21100 | Loss: 0.3880 | Spent: 28.0 secs | LR: 0.000318
INFO:tensorflow:Step 21150 | Loss: 0.3394 | Spent: 27.9 secs | LR: 0.000316
INFO:tensorflow:Step 21200 | Loss: 0.3552 | Spent: 27.9 secs | LR: 0.000313
INFO:tensorflow:Step 21250 | Loss: 0.3306 | Spent: 28.2 secs | LR: 0.000310
INFO:tensorflow:Step 21300 | Loss: 0.3501 | Spent: 27.7 secs | LR: 0.000307
INFO:tensorflow:Step 21350 | Loss: 0.4486 | Spent: 27.8 secs | LR: 0.000304
INFO:tensorflow:Step 21400 | Loss: 0.3539 | Spent: 27.8 secs | LR: 0.000302
INFO:tensorflow:Step 21450 | Loss: 0.3755 | Spent: 27.7 secs | LR: 0.000299
INFO:tensorflow:Step 21500 | Loss: 0.3477 | Spent: 27.9 secs | LR: 0.000296
INFO:tensorflow:Step 21550 | Loss: 0.3856 | Spent: 27.7 secs | LR: 0.000293
INFO:tensorflow:Step 21600 | Loss: 0.3521 | Spent: 27.9 secs | LR: 0.000290
INFO:tensorflow:Step 21650 | Loss: 0.4414 | Spent: 28.1 secs | LR: 0.000288
INFO:tensorflow:Step 21700 | Loss: 0.3786 | Spent: 28.1 secs | LR: 0.000285
INFO:tensorflow:Step 21750 | Loss: 0.3663 | Spent: 27.8 secs | LR: 0.000282
INFO:tensorflow:Step 21800 | Loss: 0.3283 | Spent: 28.0 secs | LR: 0.000279
INFO:tensorflow:Step 21850 | Loss: 0.3325 | Spent: 28.0 secs | LR: 0.000276
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.919
INFO:tensorflow:Best Accuracy: 0.923
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 21900 | Loss: 0.3542 | Spent: 281.3 secs | LR: 0.000274
INFO:tensorflow:Step 21950 | Loss: 0.3975 | Spent: 27.9 secs | LR: 0.000271
INFO:tensorflow:Step 22000 | Loss: 0.3266 | Spent: 27.9 secs | LR: 0.000268
INFO:tensorflow:Step 22050 | Loss: 0.4251 | Spent: 27.8 secs | LR: 0.000265
INFO:tensorflow:Step 22100 | Loss: 0.3318 | Spent: 27.9 secs | LR: 0.000262
INFO:tensorflow:Step 22150 | Loss: 0.3965 | Spent: 28.0 secs | LR: 0.000260
INFO:tensorflow:Step 22200 | Loss: 0.3626 | Spent: 27.8 secs | LR: 0.000257
INFO:tensorflow:Step 22250 | Loss: 0.4474 | Spent: 28.1 secs | LR: 0.000254
INFO:tensorflow:Step 22300 | Loss: 0.3326 | Spent: 28.0 secs | LR: 0.000251
INFO:tensorflow:Step 22350 | Loss: 0.3375 | Spent: 27.7 secs | LR: 0.000248
INFO:tensorflow:Step 22400 | Loss: 0.3275 | Spent: 28.0 secs | LR: 0.000246
INFO:tensorflow:Step 22450 | Loss: 0.3324 | Spent: 27.9 secs | LR: 0.000243
INFO:tensorflow:Step 22500 | Loss: 0.3464 | Spent: 28.0 secs | LR: 0.000240
INFO:tensorflow:Step 22550 | Loss: 0.3701 | Spent: 28.0 secs | LR: 0.000237
INFO:tensorflow:Step 22600 | Loss: 0.3329 | Spent: 27.7 secs | LR: 0.000234
INFO:tensorflow:Step 22650 | Loss: 0.3313 | Spent: 27.8 secs | LR: 0.000232
INFO:tensorflow:Step 22700 | Loss: 0.3819 | Spent: 27.9 secs | LR: 0.000229
INFO:tensorflow:Step 22750 | Loss: 0.3507 | Spent: 28.1 secs | LR: 0.000226
INFO:tensorflow:Step 22800 | Loss: 0.3305 | Spent: 28.0 secs | LR: 0.000223
INFO:tensorflow:Step 22850 | Loss: 0.3299 | Spent: 28.0 secs | LR: 0.000220
INFO:tensorflow:Step 22900 | Loss: 0.3559 | Spent: 27.7 secs | LR: 0.000218
INFO:tensorflow:Step 22950 | Loss: 0.3341 | Spent: 27.9 secs | LR: 0.000215
INFO:tensorflow:Step 23000 | Loss: 0.3789 | Spent: 28.1 secs | LR: 0.000212
INFO:tensorflow:Step 23050 | Loss: 0.3797 | Spent: 28.2 secs | LR: 0.000209
INFO:tensorflow:Step 23100 | Loss: 0.3260 | Spent: 28.0 secs | LR: 0.000206
INFO:tensorflow:Step 23150 | Loss: 0.3983 | Spent: 28.0 secs | LR: 0.000204
INFO:tensorflow:Step 23200 | Loss: 0.3677 | Spent: 28.0 secs | LR: 0.000201
INFO:tensorflow:Step 23250 | Loss: 0.3326 | Spent: 27.9 secs | LR: 0.000198
INFO:tensorflow:Step 23300 | Loss: 0.3314 | Spent: 27.9 secs | LR: 0.000195
INFO:tensorflow:Step 23350 | Loss: 0.3278 | Spent: 28.1 secs | LR: 0.000192
INFO:tensorflow:Step 23400 | Loss: 0.3263 | Spent: 28.1 secs | LR: 0.000190
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.925
INFO:tensorflow:Best Accuracy: 0.925
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 23450 | Loss: 0.3870 | Spent: 282.8 secs | LR: 0.000187
INFO:tensorflow:Step 23500 | Loss: 0.3277 | Spent: 28.1 secs | LR: 0.000184
INFO:tensorflow:Step 23550 | Loss: 0.3348 | Spent: 27.9 secs | LR: 0.000181
INFO:tensorflow:Step 23600 | Loss: 0.3427 | Spent: 28.1 secs | LR: 0.000178
INFO:tensorflow:Step 23650 | Loss: 0.3316 | Spent: 28.0 secs | LR: 0.000176
INFO:tensorflow:Step 23700 | Loss: 0.4041 | Spent: 28.0 secs | LR: 0.000173
INFO:tensorflow:Step 23750 | Loss: 0.3356 | Spent: 28.0 secs | LR: 0.000170
INFO:tensorflow:Step 23800 | Loss: 0.3663 | Spent: 28.0 secs | LR: 0.000167
INFO:tensorflow:Step 23850 | Loss: 0.3460 | Spent: 27.7 secs | LR: 0.000164
INFO:tensorflow:Step 23900 | Loss: 0.3363 | Spent: 27.9 secs | LR: 0.000162
INFO:tensorflow:Step 23950 | Loss: 0.3848 | Spent: 27.7 secs | LR: 0.000159
INFO:tensorflow:Step 24000 | Loss: 0.3804 | Spent: 27.9 secs | LR: 0.000156
INFO:tensorflow:Step 24050 | Loss: 0.3417 | Spent: 28.2 secs | LR: 0.000153
INFO:tensorflow:Step 24100 | Loss: 0.3275 | Spent: 27.9 secs | LR: 0.000150
INFO:tensorflow:Step 24150 | Loss: 0.3770 | Spent: 27.9 secs | LR: 0.000148
INFO:tensorflow:Step 24200 | Loss: 0.3264 | Spent: 28.2 secs | LR: 0.000145
INFO:tensorflow:Step 24250 | Loss: 0.3338 | Spent: 27.9 secs | LR: 0.000142
INFO:tensorflow:Step 24300 | Loss: 0.3902 | Spent: 27.9 secs | LR: 0.000139
INFO:tensorflow:Step 24350 | Loss: 0.3920 | Spent: 28.0 secs | LR: 0.000136
INFO:tensorflow:Step 24400 | Loss: 0.3268 | Spent: 28.0 secs | LR: 0.000134
INFO:tensorflow:Step 24450 | Loss: 0.3322 | Spent: 27.9 secs | LR: 0.000131
INFO:tensorflow:Step 24500 | Loss: 0.3759 | Spent: 28.1 secs | LR: 0.000128
INFO:tensorflow:Step 24550 | Loss: 0.3294 | Spent: 28.0 secs | LR: 0.000125
INFO:tensorflow:Step 24600 | Loss: 0.3587 | Spent: 28.3 secs | LR: 0.000122
INFO:tensorflow:Step 24650 | Loss: 0.3499 | Spent: 27.9 secs | LR: 0.000120
INFO:tensorflow:Step 24700 | Loss: 0.3267 | Spent: 27.9 secs | LR: 0.000117
INFO:tensorflow:Step 24750 | Loss: 0.3289 | Spent: 28.1 secs | LR: 0.000114
INFO:tensorflow:Step 24800 | Loss: 0.3287 | Spent: 28.0 secs | LR: 0.000111
INFO:tensorflow:Step 24850 | Loss: 0.3730 | Spent: 27.9 secs | LR: 0.000108
INFO:tensorflow:Step 24900 | Loss: 0.3447 | Spent: 27.8 secs | LR: 0.000106
INFO:tensorflow:Step 24950 | Loss: 0.3730 | Spent: 28.0 secs | LR: 0.000103
INFO:tensorflow:Step 25000 | Loss: 0.3615 | Spent: 27.9 secs | LR: 0.000100
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.924
INFO:tensorflow:Best Accuracy: 0.925
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 25050 | Loss: 0.3389 | Spent: 279.9 secs | LR: 0.000103
INFO:tensorflow:Step 25100 | Loss: 0.3262 | Spent: 27.8 secs | LR: 0.000106
INFO:tensorflow:Step 25150 | Loss: 0.3301 | Spent: 28.0 secs | LR: 0.000108
INFO:tensorflow:Step 25200 | Loss: 0.3380 | Spent: 28.0 secs | LR: 0.000111
INFO:tensorflow:Step 25250 | Loss: 0.3803 | Spent: 28.2 secs | LR: 0.000114
INFO:tensorflow:Step 25300 | Loss: 0.3267 | Spent: 28.2 secs | LR: 0.000117
INFO:tensorflow:Step 25350 | Loss: 0.3273 | Spent: 28.3 secs | LR: 0.000120
INFO:tensorflow:Step 25400 | Loss: 0.3259 | Spent: 28.1 secs | LR: 0.000122
INFO:tensorflow:Step 25450 | Loss: 0.3266 | Spent: 28.2 secs | LR: 0.000125
INFO:tensorflow:Step 25500 | Loss: 0.3297 | Spent: 28.5 secs | LR: 0.000128
INFO:tensorflow:Step 25550 | Loss: 0.3276 | Spent: 28.0 secs | LR: 0.000131
INFO:tensorflow:Step 25600 | Loss: 0.3315 | Spent: 28.0 secs | LR: 0.000134
INFO:tensorflow:Step 25650 | Loss: 0.3432 | Spent: 28.1 secs | LR: 0.000136
INFO:tensorflow:Step 25700 | Loss: 0.3258 | Spent: 28.4 secs | LR: 0.000139
INFO:tensorflow:Step 25750 | Loss: 0.3758 | Spent: 28.0 secs | LR: 0.000142
INFO:tensorflow:Step 25800 | Loss: 0.3491 | Spent: 28.4 secs | LR: 0.000145
INFO:tensorflow:Step 25850 | Loss: 0.3307 | Spent: 28.0 secs | LR: 0.000148
INFO:tensorflow:Step 25900 | Loss: 0.4159 | Spent: 28.4 secs | LR: 0.000150
INFO:tensorflow:Step 25950 | Loss: 0.3334 | Spent: 28.2 secs | LR: 0.000153
INFO:tensorflow:Step 26000 | Loss: 0.3515 | Spent: 28.3 secs | LR: 0.000156
INFO:tensorflow:Step 26050 | Loss: 0.3257 | Spent: 28.5 secs | LR: 0.000159
INFO:tensorflow:Step 26100 | Loss: 0.3376 | Spent: 28.4 secs | LR: 0.000162
INFO:tensorflow:Step 26150 | Loss: 0.3572 | Spent: 28.4 secs | LR: 0.000164
INFO:tensorflow:Step 26200 | Loss: 0.3273 | Spent: 28.4 secs | LR: 0.000167
INFO:tensorflow:Step 26250 | Loss: 0.3306 | Spent: 28.2 secs | LR: 0.000170
INFO:tensorflow:Step 26300 | Loss: 0.3526 | Spent: 28.7 secs | LR: 0.000173
INFO:tensorflow:Step 26350 | Loss: 0.3824 | Spent: 28.8 secs | LR: 0.000176
INFO:tensorflow:Step 26400 | Loss: 0.3260 | Spent: 28.6 secs | LR: 0.000178
INFO:tensorflow:Step 26450 | Loss: 0.4229 | Spent: 28.5 secs | LR: 0.000181
INFO:tensorflow:Step 26500 | Loss: 0.3279 | Spent: 28.3 secs | LR: 0.000184
INFO:tensorflow:Step 26550 | Loss: 0.3644 | Spent: 28.2 secs | LR: 0.000187
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.926
INFO:tensorflow:Best Accuracy: 0.926
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 26600 | Loss: 0.3449 | Spent: 285.0 secs | LR: 0.000190
INFO:tensorflow:Step 26650 | Loss: 0.3748 | Spent: 28.2 secs | LR: 0.000192
INFO:tensorflow:Step 26700 | Loss: 0.3295 | Spent: 28.0 secs | LR: 0.000195
INFO:tensorflow:Step 26750 | Loss: 0.3266 | Spent: 28.2 secs | LR: 0.000198
INFO:tensorflow:Step 26800 | Loss: 0.3354 | Spent: 28.3 secs | LR: 0.000201
INFO:tensorflow:Step 26850 | Loss: 0.3407 | Spent: 28.6 secs | LR: 0.000204
INFO:tensorflow:Step 26900 | Loss: 0.3282 | Spent: 28.9 secs | LR: 0.000206
INFO:tensorflow:Step 26950 | Loss: 0.3343 | Spent: 28.4 secs | LR: 0.000209
INFO:tensorflow:Step 27000 | Loss: 0.3260 | Spent: 28.4 secs | LR: 0.000212
INFO:tensorflow:Step 27050 | Loss: 0.3259 | Spent: 28.4 secs | LR: 0.000215
INFO:tensorflow:Step 27100 | Loss: 0.3676 | Spent: 28.1 secs | LR: 0.000218
INFO:tensorflow:Step 27150 | Loss: 0.3796 | Spent: 28.4 secs | LR: 0.000220
INFO:tensorflow:Step 27200 | Loss: 0.3808 | Spent: 27.8 secs | LR: 0.000223
INFO:tensorflow:Step 27250 | Loss: 0.3853 | Spent: 28.0 secs | LR: 0.000226
INFO:tensorflow:Step 27300 | Loss: 0.3264 | Spent: 27.8 secs | LR: 0.000229
INFO:tensorflow:Step 27350 | Loss: 0.3523 | Spent: 27.8 secs | LR: 0.000232
INFO:tensorflow:Step 27400 | Loss: 0.3364 | Spent: 27.9 secs | LR: 0.000234
INFO:tensorflow:Step 27450 | Loss: 0.3287 | Spent: 28.1 secs | LR: 0.000237
INFO:tensorflow:Step 27500 | Loss: 0.3397 | Spent: 27.8 secs | LR: 0.000240
INFO:tensorflow:Step 27550 | Loss: 0.3832 | Spent: 28.2 secs | LR: 0.000243
INFO:tensorflow:Step 27600 | Loss: 0.4359 | Spent: 27.8 secs | LR: 0.000246
INFO:tensorflow:Step 27650 | Loss: 0.3783 | Spent: 27.9 secs | LR: 0.000248
INFO:tensorflow:Step 27700 | Loss: 0.3748 | Spent: 28.0 secs | LR: 0.000251
INFO:tensorflow:Step 27750 | Loss: 0.3705 | Spent: 28.0 secs | LR: 0.000254
INFO:tensorflow:Step 27800 | Loss: 0.4056 | Spent: 27.9 secs | LR: 0.000257
INFO:tensorflow:Step 27850 | Loss: 0.3275 | Spent: 27.9 secs | LR: 0.000260
INFO:tensorflow:Step 27900 | Loss: 0.3956 | Spent: 28.2 secs | LR: 0.000262
INFO:tensorflow:Step 27950 | Loss: 0.3314 | Spent: 27.8 secs | LR: 0.000265
INFO:tensorflow:Step 28000 | Loss: 0.4093 | Spent: 28.0 secs | LR: 0.000268
INFO:tensorflow:Step 28050 | Loss: 0.3792 | Spent: 28.1 secs | LR: 0.000271
INFO:tensorflow:Step 28100 | Loss: 0.3706 | Spent: 28.2 secs | LR: 0.000274
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.925
INFO:tensorflow:Best Accuracy: 0.926
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 28150 | Loss: 0.3780 | Spent: 280.0 secs | LR: 0.000276
INFO:tensorflow:Step 28200 | Loss: 0.3309 | Spent: 27.9 secs | LR: 0.000279
INFO:tensorflow:Step 28250 | Loss: 0.3297 | Spent: 28.2 secs | LR: 0.000282
INFO:tensorflow:Step 28300 | Loss: 0.3311 | Spent: 27.9 secs | LR: 0.000285
INFO:tensorflow:Step 28350 | Loss: 0.3338 | Spent: 27.9 secs | LR: 0.000288
INFO:tensorflow:Step 28400 | Loss: 0.3424 | Spent: 27.9 secs | LR: 0.000290
INFO:tensorflow:Step 28450 | Loss: 0.3842 | Spent: 28.1 secs | LR: 0.000293
INFO:tensorflow:Step 28500 | Loss: 0.3877 | Spent: 28.0 secs | LR: 0.000296
INFO:tensorflow:Step 28550 | Loss: 0.3312 | Spent: 27.8 secs | LR: 0.000299
INFO:tensorflow:Step 28600 | Loss: 0.3890 | Spent: 27.9 secs | LR: 0.000302
INFO:tensorflow:Step 28650 | Loss: 0.3312 | Spent: 27.8 secs | LR: 0.000304
INFO:tensorflow:Step 28700 | Loss: 0.3262 | Spent: 28.1 secs | LR: 0.000307
INFO:tensorflow:Step 28750 | Loss: 0.3282 | Spent: 28.2 secs | LR: 0.000310
INFO:tensorflow:Step 28800 | Loss: 0.3511 | Spent: 28.2 secs | LR: 0.000313
INFO:tensorflow:Step 28850 | Loss: 0.3867 | Spent: 27.9 secs | LR: 0.000316
INFO:tensorflow:Step 28900 | Loss: 0.3287 | Spent: 28.2 secs | LR: 0.000318
INFO:tensorflow:Step 28950 | Loss: 0.3659 | Spent: 28.1 secs | LR: 0.000321
INFO:tensorflow:Step 29000 | Loss: 0.3644 | Spent: 28.0 secs | LR: 0.000324
INFO:tensorflow:Step 29050 | Loss: 0.3771 | Spent: 27.9 secs | LR: 0.000327
INFO:tensorflow:Step 29100 | Loss: 0.3303 | Spent: 27.8 secs | LR: 0.000330
INFO:tensorflow:Step 29150 | Loss: 0.3605 | Spent: 28.0 secs | LR: 0.000332
INFO:tensorflow:Step 29200 | Loss: 0.3386 | Spent: 28.2 secs | LR: 0.000335
INFO:tensorflow:Step 29250 | Loss: 0.3267 | Spent: 28.0 secs | LR: 0.000338
INFO:tensorflow:Step 29300 | Loss: 0.3335 | Spent: 27.9 secs | LR: 0.000341
INFO:tensorflow:Step 29350 | Loss: 0.4153 | Spent: 28.0 secs | LR: 0.000344
INFO:tensorflow:Step 29400 | Loss: 0.3859 | Spent: 28.1 secs | LR: 0.000346
INFO:tensorflow:Step 29450 | Loss: 0.3270 | Spent: 28.4 secs | LR: 0.000349
INFO:tensorflow:Step 29500 | Loss: 0.3387 | Spent: 27.9 secs | LR: 0.000352
INFO:tensorflow:Step 29550 | Loss: 0.3526 | Spent: 27.9 secs | LR: 0.000355
INFO:tensorflow:Step 29600 | Loss: 0.3259 | Spent: 27.9 secs | LR: 0.000358
INFO:tensorflow:Step 29650 | Loss: 0.3267 | Spent: 27.8 secs | LR: 0.000360
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.923
INFO:tensorflow:Best Accuracy: 0.926
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 29700 | Loss: 0.3270 | Spent: 280.0 secs | LR: 0.000363
INFO:tensorflow:Step 29750 | Loss: 0.3268 | Spent: 27.9 secs | LR: 0.000366
INFO:tensorflow:Step 29800 | Loss: 0.3267 | Spent: 27.9 secs | LR: 0.000369
INFO:tensorflow:Step 29850 | Loss: 0.3417 | Spent: 28.1 secs | LR: 0.000372