In [ ]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/free_chat/chinese_lccc/main')
In [ ]:
%tensorflow_version 1.x
!pip install texar
In [ ]:
import tensorflow as tf
import texar.tf as tx

import numpy as np
import pprint
import logging
import copy

from pathlib import Path
from texar.tf.modules import TransformerEncoder

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 1.15.2
GPU Enabled: True
In [ ]:
# stream data from text files
def data_generator(f_path, params):
  char2idx = params['char2idx']
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = line.rstrip()
      source, target = line.split('<SEP>')
      source = [char2idx.get(c, len(char2idx)) for c in list(source)]
      target = [char2idx.get(c, len(char2idx)) for c in list(target)]
      if len(source) > params['max_len']:
        source = source[:params['max_len']]
      if len(target) > params['max_len']:
        target = target[:params['max_len']]
      target_in = [1] + target
      target_out = target + [2]
      yield (source, (target_in, target_out))
In [ ]:
def dataset(is_training, params):
  _shapes = ([None], ([None], [None]))
  _types = (tf.int32, (tf.int32, tf.int32))
  _pads = (0, (0, 0))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['buffer_size'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [ ]:
def clip_grads(loss):
    variables = tf.trainable_variables()
    pprint.pprint(variables)
    grads = tf.gradients(loss, variables)
    clipped_grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    return zip(clipped_grads, variables)


def rnn_cell():
    def cell_fn():
        cell = tf.nn.rnn_cell.LSTMCell(params['rnn_units'],
                                       initializer=tf.orthogonal_initializer())
        return cell
    if params['dec_layers'] > 1:
      cells = []
      for i in range(params['dec_layers']):
        if i == params['dec_layers'] - 1:
          cells.append(cell_fn())
        else:
          cells.append(tf.nn.rnn_cell.ResidualWrapper(cell_fn(), residual_fn=lambda i,o: tf.concat((i,o), -1)))
      return tf.nn.rnn_cell.MultiRNNCell(cells)
    else:
      return cell_fn()

  
def dec_cell(enc_out, enc_seq_len):
    attn = tf.contrib.seq2seq.BahdanauAttention(
        num_units = params['rnn_units'],
        memory = enc_out,
        memory_sequence_length = enc_seq_len)
    
    return tf.contrib.seq2seq.AttentionWrapper(
        cell = rnn_cell(),
        attention_mechanism = attn,
        attention_layer_size = params['rnn_units'])
In [ ]:
class TiedDense(tf.layers.Layer):
  def __init__(self, tied_embed, out_dim):
    super().__init__()
    self.tied_embed = tied_embed
    self.out_dim = out_dim
  
  def build(self, input_shape):
    self.bias = self.add_weight(name='bias',
                                shape=[self.out_dim],
                                trainable=True)
    if params['rnn_units'] != 300:
      self.proj_W = self.add_weight(name='proj_W',
                                    shape=[params['rnn_units'], params['embed_dim']],
                                    trainable=True)
      self.proj_b = self.add_weight(name='proj_b',
                                    shape=[params['embed_dim']],
                                    trainable=True)
    super().build(input_shape)
  
  def call(self, inputs):
    if params['rnn_units'] != 300:
      inputs = params['activation'](tf.nn.bias_add(tf.matmul(inputs, self.proj_W), self.proj_b))
    x = tf.matmul(inputs, self.tied_embed, transpose_b=True)
    x = tf.nn.bias_add(x, self.bias)
    return x
  
  def compute_output_shape(self, input_shape):
    return input_shape[:-1].concatenate(self.out_dim)
In [ ]:
def forward(words, labels, mode):
    words_len = tf.count_nonzero(words, 1, dtype=tf.int32)
    
    is_training = (mode == tf.estimator.ModeKeys.TRAIN)
    batch_sz = tf.shape(words)[0]
    
  
    with tf.variable_scope('Embedding'):
        embedding = tf.Variable(np.load('../vocab/char.npy'),
                                dtype=tf.float32,
                                name='fasttext_vectors')
        embedding = tf.concat([tf.zeros(shape=[1, params['embed_dim']]), embedding[1:, :]], axis=0)
        x = tf.nn.embedding_lookup(embedding, words)
        pos_embedder = tx.modules.SinusoidsPositionEmbedder(
            position_size = params['max_len'] + 1,
            hparams = config_model.position_embedder_hparams)
        x = (x * config_model.hidden_dim ** 0.5) + pos_embedder(sequence_length=words_len)


    with tf.variable_scope('Encoder'):
        encoder = TransformerEncoder(hparams=config_model.encoder)
        enc_out = encoder(inputs=x,
                          sequence_length=words_len,
                          mode=mode,)
        enc_state = tf.reduce_max(enc_out, axis=1)
        enc_state = tf.nn.rnn_cell.LSTMStateTuple(c=enc_state, h=enc_state)
    
    
    with tf.variable_scope('Decoder'):
        output_proj = TiedDense(embedding, len(params['char2idx'])+1)
        
        if is_training or (mode == tf.estimator.ModeKeys.EVAL):
            dec_inputs, dec_outputs = labels
            dec_seq_len = tf.count_nonzero(dec_inputs, 1, dtype=tf.int32)
            dec_inputs = tf.nn.embedding_lookup(embedding, dec_inputs)
            dec_inputs = tf.layers.dropout(dec_inputs, params['dropout_rate'], training=is_training)
            cell = dec_cell(enc_out, words_len)
            
            init_state = cell.zero_state(batch_sz, tf.float32).clone(
                cell_state=enc_state)
            
            helper = tf.contrib.seq2seq.TrainingHelper(
                inputs = dec_inputs,
                sequence_length = dec_seq_len,)
            decoder = tf.contrib.seq2seq.BasicDecoder(
                cell = cell,
                helper = helper,
                initial_state = init_state,
                output_layer = output_proj)
            decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
                decoder = decoder,
                maximum_iterations = tf.reduce_max(dec_seq_len))
            
            return decoder_output.rnn_output
        else:
            enc_out_t = tf.contrib.seq2seq.tile_batch(enc_out, params['beam_width'])
            enc_state_t = tf.contrib.seq2seq.tile_batch(enc_state, params['beam_width'])
            enc_seq_len_t = tf.contrib.seq2seq.tile_batch(words_len, params['beam_width'])
            
            cell = dec_cell(enc_out_t, enc_seq_len_t)
            
            init_state = cell.zero_state(batch_sz*params['beam_width'], tf.float32).clone(
                cell_state=enc_state_t)
            
            decoder = tf.contrib.seq2seq.BeamSearchDecoder(
                cell = cell,
                embedding = embedding,
                start_tokens = tf.tile(tf.constant([1], tf.int32), [batch_sz]),
                end_token = 2,
                initial_state = init_state,
                beam_width = params['beam_width'],
                output_layer = output_proj,
                length_penalty_weight = params['length_penalty'],)
            decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
                decoder = decoder,
                maximum_iterations = params['max_len'],)
            
            return decoder_output.predicted_ids[:, :, :params['top_k']]
In [ ]:
def clr(step,
        initial_learning_rate,
        maximal_learning_rate,
        step_size,
        scale_fn,
        scale_mode,):
  step = tf.cast(step, tf.float32)
  
  initial_learning_rate = tf.convert_to_tensor(
    initial_learning_rate, name='initial_learning_rate')
  dtype = initial_learning_rate.dtype
  maximal_learning_rate = tf.cast(maximal_learning_rate, dtype)
  step_size = tf.cast(step_size, dtype)
  cycle = tf.floor(1 + step / (2 * step_size))
  x = tf.abs(step / step_size - 2 * cycle + 1)

  mode_step = cycle if scale_mode == 'cycle' else step

  return initial_learning_rate + (
    maximal_learning_rate - initial_learning_rate) * tf.maximum(
      tf.cast(0, dtype), (1 - x)) * scale_fn(mode_step)
In [ ]:
def model_fn(features, labels, mode, params):
    logits_or_ids = forward(features, labels, mode)
    
    if mode == tf.estimator.ModeKeys.PREDICT:
        return tf.estimator.EstimatorSpec(mode, predictions=logits_or_ids)
    
    dec_inputs, dec_outputs = labels
    if (params['label_smoothing'] <= .0) or (mode == tf.estimator.ModeKeys.EVAL):
      loss_op = tf.contrib.seq2seq.sequence_loss(logits = logits_or_ids,
                                                 targets = dec_outputs,
                                                 weights = tf.to_float(tf.sign(dec_outputs)))
    else:
      loss_op = tf.losses.softmax_cross_entropy(onehot_labels = tf.one_hot(dec_outputs, len(params['char2idx'])+1),
                                                logits = logits_or_ids,
                                                weights = tf.to_float(tf.sign(dec_outputs)),
                                                label_smoothing = params['label_smoothing'],)
      
    if mode == tf.estimator.ModeKeys.TRAIN:
        global_step=tf.train.get_or_create_global_step()
        
        decay_lr = clr(
          step = global_step,
          initial_learning_rate = 1e-4,
          maximal_learning_rate = 8e-4,
          step_size = params['num_samples'] / params['batch_size'] // 2,
          scale_fn=lambda x: 1 / (2.0 ** (x - 1)),
          scale_mode = 'cycle',)
        
        train_op = tf.train.AdamOptimizer(decay_lr).apply_gradients(
            clip_grads(loss_op), global_step=global_step)
        
        hook = tf.train.LoggingTensorHook({'lr': decay_lr}, every_n_iter=100)
        
        return tf.estimator.EstimatorSpec(
            mode=mode, loss=loss_op, train_op=train_op, training_hooks=[hook],)
      
    if mode == tf.estimator.ModeKeys.EVAL:
      return tf.estimator.EstimatorSpec(mode=mode, loss=loss_op)
In [ ]:
def get_vocab(f_path):
  word2idx = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip('\n')
      word2idx[line] = i
  return word2idx
In [ ]:
def pad(test_strs):
  max_len = max([len(test_str) for test_str in test_strs])
  for test_str in test_strs:
    if len(test_str) < max_len:
      test_str += ['<pad>'] * (max_len - len(test_str))


def unit_test(estimator):
  test_strs = [
    '你好',
    '早上好',
    '晚上好',
    '再见',
    '好久不见',
    '想死你了',
    '谢谢你',
    '爱你',
    '你好厉害啊',
    '你叫什么',
    '你几岁了',
    '现在几点',
    '今天天气怎么样',
    '你今天心情好吗',
    '我们现在在哪里',
    '讲个笑话',
    '你会几种语言呀',
    '你觉得我帅吗',
    '讨厌的周一',
    '好烦啊',
    '天气真好',
    '今天好冷',
    '今天好热',
    '下雨了',
    '风好大',
    '终于周五了',
    '我想去唱歌',
  ]
  test_strs = [list(test_str) for test_str in test_strs]
  pad(test_strs)
  test_arrs = [[params['char2idx'].get(c, len(params['char2idx'])) for c in test_str] for test_str in test_strs]
  predicted = list(estimator.predict(tf.estimator.inputs.numpy_input_fn(
    x = np.asarray(test_arrs), shuffle = False)))
  predicted = np.asarray(predicted)
  print('-'*12)
  print('unit test')
  for i, test_str in enumerate(test_strs):
    print('Q:', ' '.join([c for c in test_str if c != '<pad>']))
    for j in range(params['top_k']):
      sent = ' '.join([params['idx2char'].get(idx, '<unk>') for idx in predicted[i, :, j] if (idx != 0 and idx != 2)])
      print('A{}:'.format(j+1), sent)
    print()
  print('-'*12)
In [ ]:
class config_model:
    hidden_dim = 300
    num_heads = 8
    dropout_rate = .2
    num_blocks = 3

    position_embedder_hparams = {
        'dim': hidden_dim
    }

    encoder = {
        'dim': hidden_dim,
        'embedding_dropout': dropout_rate,
        'residual_dropout': dropout_rate,
        'num_blocks': num_blocks,
        'initializer': {
            'type': 'variance_scaling_initializer',
            'kwargs': {
                'scale': 1.0,
                'mode': 'fan_avg',
                'distribution': 'uniform',
            },
        },
        'multihead_attention': {
            'dropout_rate': dropout_rate,
            'num_heads': num_heads,
            'output_dim': hidden_dim,
            'use_bias': True,
        },
        'poswise_feedforward': {
          'name': 'fnn',
          'layers': [
              {
                  'type': 'Dense',
                  'kwargs': {
                      'name': 'conv1',
                      'units': hidden_dim * 4,
                      'activation': 'gelu',
                      'use_bias': True,
                  },
              },
              {
                  'type': 'Dropout',
                  'kwargs': {
                      'rate': dropout_rate,
                  }
              },
              {
                  'type': 'Dense',
                  'kwargs': {
                      'name': 'conv2',
                      'units': hidden_dim,
                      'use_bias': True,
                  }
              }
          ],
        },
    }


params = {
    'model_dir': '../model/transformer_rnn',
    'train_path': '../data/train.txt',
    'test_path': '../data/test.txt',
    'vocab_path': '../vocab/char.txt',
    'max_len': 30,
    'embed_dim': config_model.hidden_dim,
    'rnn_units': 300,
    'dec_layers': 1,
    'dropout_rate': .2,
    'beam_width': 10,
    'top_k': 3,
    'length_penalty': .6,
    'label_smoothing': .2,
    'clip_norm': .1,
    'num_samples': 5000000,
    'buffer_size': 500000,
    'batch_size': 64,
    'num_patience': 5,
}
In [ ]:
params['char2idx'] = get_vocab(params['vocab_path'])
params['idx2char'] = {idx: char for char, idx in params['char2idx'].items()}
print(len(params['char2idx']), 'Chars')

# Create directory if not exist
Path(params['model_dir']).mkdir(exist_ok=True, parents=True)

# Logging
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

# Create an estimator
estimator = tf.estimator.Estimator(
  model_fn=model_fn,
  model_dir=params['model_dir'],
  config=tf.estimator.RunConfig(
    save_checkpoints_steps = params['num_samples'] // params['batch_size'] + 1,
    keep_checkpoint_max = 2),
  params=params)

best_ppl = 10000.
count = 0
tf.enable_eager_execution()

while True:
  estimator.train(input_fn=lambda: dataset(is_training=True, params=params))

  unit_test(estimator)
  
  loss = estimator.evaluate(input_fn=lambda: dataset(is_training=False, params=params))['loss']
  ppl = np.exp(loss)
  print("Perplexity: {:.3f}".format(ppl))

  if ppl < best_ppl:
    best_ppl = ppl
    count = 0
  else:
    count += 1
  print("Best Perplexity: {:.3f}".format(best_ppl))

  if count == params['num_patience']:
    print(params['num_patience'], "times not improve the best result, therefore stop training")
    break
3042 Chars
INFO:tensorflow:Using config: {'_model_dir': '../model/transformer_rnn', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 78126, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true
graph_options {
  rewrite_options {
    meta_optimizer_iterations: ONE
  }
}
, '_keep_checkpoint_max': 2, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f6e9eacef60>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.
INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.
Instructions for updating:
reduction_indices is deprecated, use axis instead
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/module_base.py:83: The name tf.make_template is deprecated. Please use tf.compat.v1.make_template instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/modules/embedders/position_embedders.py:338: The name tf.mod is deprecated. Please use tf.math.mod instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/modules/encoders/transformer_encoders.py:142: The name tf.get_variable_scope is deprecated. Please use tf.compat.v1.get_variable_scope instead.

WARNING:tensorflow:From /usr/lib/python3.6/pydoc.py:1595: The name tf.layers.Dense is deprecated. Please use tf.compat.v1.layers.Dense instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/core/layers.py:608: The name tf.layers.Layer is deprecated. Please use tf.compat.v1.layers.Layer instead.

WARNING:tensorflow:From /usr/lib/python3.6/pydoc.py:1595: The name tf.layers.Dropout is deprecated. Please use tf.compat.v1.layers.Dropout instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/modules/encoders/transformer_encoders.py:317: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dropout instead.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/layers/core.py:271: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.__call__` method instead.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/utils/transformer_utils.py:73: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/module_base.py:140: The name tf.get_collection is deprecated. Please use tf.compat.v1.get_collection instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/module_base.py:141: The name tf.GraphKeys is deprecated. Please use tf.compat.v1.GraphKeys instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/modules/networks/network_base.py:125: The name tf.layers.BatchNormalization is deprecated. Please use tf.compat.v1.layers.BatchNormalization instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/texar/tf/core/layers.py:1270: The name tf.erf is deprecated. Please use tf.math.erf instead.

WARNING:tensorflow:From <ipython-input-6-33c7bca471bf>:12: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn_cell_impl.py:958: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.add_weight` method instead.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn_cell_impl.py:962: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
WARNING:tensorflow:Entity <bound method TiedDense.call of <__main__.TiedDense object at 0x7f6e903914a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING: Entity <bound method TiedDense.call of <__main__.TiedDense object at 0x7f6e903914a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:From <ipython-input-10-1d70a7bd47f9>:15: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.cast` instead.
[<tf.Variable 'Embedding/fasttext_vectors:0' shape=(3043, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/query/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/query/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/key/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/key/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/value/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/value/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/output/kernel:0' shape=(512, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/output/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/output/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/output/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/fnn/conv1/kernel:0' shape=(300, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/fnn/conv1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/fnn/conv2/kernel:0' shape=(1200, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/fnn/conv2/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/query/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/query/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/key/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/key/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/value/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/value/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/output/kernel:0' shape=(512, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/output/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/output/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/output/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/fnn/conv1/kernel:0' shape=(300, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/fnn/conv1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/fnn/conv2/kernel:0' shape=(1200, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/fnn/conv2/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/query/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/query/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/key/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/key/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/value/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/value/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/output/kernel:0' shape=(512, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/output/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/output/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/output/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/fnn/conv1/kernel:0' shape=(300, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/fnn/conv1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/fnn/conv2/kernel:0' shape=(1200, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/fnn/conv2/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer/kernel:0' shape=(300, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel:0' shape=(300, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/attention_v:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/bias:0' shape=(3043,) dtype=float32_ref>]
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 0 into ../model/transformer_rnn/model.ckpt.
Reading ../data/train.txt
INFO:tensorflow:loss = 8.937515, step = 0
INFO:tensorflow:lr = 1e-04
INFO:tensorflow:global_step/sec: 8.71145
INFO:tensorflow:loss = 6.517472, step = 100 (11.480 sec)
INFO:tensorflow:lr = 0.00010179201 (11.481 sec)
INFO:tensorflow:global_step/sec: 9.61915
INFO:tensorflow:loss = 6.4360666, step = 200 (10.397 sec)
INFO:tensorflow:lr = 0.000103584025 (10.397 sec)
INFO:tensorflow:global_step/sec: 9.47261
INFO:tensorflow:loss = 6.410827, step = 300 (10.555 sec)
INFO:tensorflow:lr = 0.00010537608 (10.556 sec)
INFO:tensorflow:global_step/sec: 9.76318
INFO:tensorflow:loss = 6.240726, step = 400 (10.243 sec)
INFO:tensorflow:lr = 0.000107168096 (10.244 sec)
INFO:tensorflow:global_step/sec: 9.38388
INFO:tensorflow:loss = 6.329311, step = 500 (10.656 sec)
INFO:tensorflow:lr = 0.00010896011 (10.656 sec)
INFO:tensorflow:global_step/sec: 9.68245
INFO:tensorflow:loss = 6.216666, step = 600 (10.330 sec)
INFO:tensorflow:lr = 0.000110752124 (10.331 sec)
INFO:tensorflow:global_step/sec: 9.41577
INFO:tensorflow:loss = 6.192756, step = 700 (10.618 sec)
INFO:tensorflow:lr = 0.00011254418 (10.616 sec)
INFO:tensorflow:global_step/sec: 9.78002
INFO:tensorflow:loss = 6.1685157, step = 800 (10.227 sec)
INFO:tensorflow:lr = 0.00011433619 (10.226 sec)
INFO:tensorflow:global_step/sec: 9.49643
INFO:tensorflow:loss = 6.175221, step = 900 (10.531 sec)
INFO:tensorflow:lr = 0.000116128205 (10.531 sec)
INFO:tensorflow:global_step/sec: 9.46536
INFO:tensorflow:loss = 6.014949, step = 1000 (10.564 sec)
INFO:tensorflow:lr = 0.000117920215 (10.564 sec)
INFO:tensorflow:global_step/sec: 9.70079
INFO:tensorflow:loss = 6.086758, step = 1100 (10.310 sec)
INFO:tensorflow:lr = 0.00011971223 (10.310 sec)
INFO:tensorflow:global_step/sec: 9.64372
INFO:tensorflow:loss = 5.9653287, step = 1200 (10.368 sec)
INFO:tensorflow:lr = 0.00012150429 (10.368 sec)
INFO:tensorflow:global_step/sec: 9.50996
INFO:tensorflow:loss = 5.899817, step = 1300 (10.516 sec)
INFO:tensorflow:lr = 0.0001232963 (10.517 sec)
INFO:tensorflow:global_step/sec: 9.67609
INFO:tensorflow:loss = 5.9957976, step = 1400 (10.334 sec)
INFO:tensorflow:lr = 0.0001250883 (10.335 sec)
INFO:tensorflow:global_step/sec: 9.51666
INFO:tensorflow:loss = 5.886746, step = 1500 (10.509 sec)
INFO:tensorflow:lr = 0.00012688033 (10.508 sec)
INFO:tensorflow:global_step/sec: 9.42603
INFO:tensorflow:loss = 5.846593, step = 1600 (10.610 sec)
INFO:tensorflow:lr = 0.00012867239 (10.610 sec)
INFO:tensorflow:global_step/sec: 9.63493
INFO:tensorflow:loss = 5.91529, step = 1700 (10.375 sec)
INFO:tensorflow:lr = 0.0001304644 (10.377 sec)
INFO:tensorflow:global_step/sec: 9.46334
INFO:tensorflow:loss = 5.8557997, step = 1800 (10.570 sec)
INFO:tensorflow:lr = 0.0001322564 (10.569 sec)
INFO:tensorflow:global_step/sec: 9.35606
INFO:tensorflow:loss = 5.9050403, step = 1900 (10.687 sec)
INFO:tensorflow:lr = 0.00013404843 (10.688 sec)
INFO:tensorflow:global_step/sec: 9.54904
INFO:tensorflow:loss = 5.8140674, step = 2000 (10.474 sec)
INFO:tensorflow:lr = 0.00013584044 (10.473 sec)
INFO:tensorflow:global_step/sec: 9.35126
INFO:tensorflow:loss = 5.8583922, step = 2100 (10.692 sec)
INFO:tensorflow:lr = 0.0001376325 (10.692 sec)
INFO:tensorflow:global_step/sec: 9.63218
INFO:tensorflow:loss = 5.758723, step = 2200 (10.383 sec)
INFO:tensorflow:lr = 0.0001394245 (10.382 sec)
INFO:tensorflow:global_step/sec: 9.42249
INFO:tensorflow:loss = 5.5649285, step = 2300 (10.610 sec)
INFO:tensorflow:lr = 0.00014121651 (10.611 sec)
INFO:tensorflow:global_step/sec: 9.43176
INFO:tensorflow:loss = 5.705367, step = 2400 (10.606 sec)
INFO:tensorflow:lr = 0.00014300854 (10.605 sec)
INFO:tensorflow:global_step/sec: 9.35512
INFO:tensorflow:loss = 5.627061, step = 2500 (10.688 sec)
INFO:tensorflow:lr = 0.00014480058 (10.688 sec)
INFO:tensorflow:global_step/sec: 9.30433
INFO:tensorflow:loss = 5.874296, step = 2600 (10.746 sec)
INFO:tensorflow:lr = 0.0001465926 (10.746 sec)
INFO:tensorflow:global_step/sec: 9.62686
INFO:tensorflow:loss = 5.717611, step = 2700 (10.389 sec)
INFO:tensorflow:lr = 0.00014838461 (10.388 sec)
INFO:tensorflow:global_step/sec: 9.42155
INFO:tensorflow:loss = 5.7843623, step = 2800 (10.615 sec)
INFO:tensorflow:lr = 0.00015017662 (10.616 sec)
INFO:tensorflow:global_step/sec: 9.37644
INFO:tensorflow:loss = 5.6821647, step = 2900 (10.663 sec)
INFO:tensorflow:lr = 0.00015196865 (10.662 sec)
INFO:tensorflow:global_step/sec: 9.7579
INFO:tensorflow:loss = 5.754989, step = 3000 (10.250 sec)
INFO:tensorflow:lr = 0.0001537607 (10.251 sec)
INFO:tensorflow:global_step/sec: 9.52185
INFO:tensorflow:loss = 5.6745124, step = 3100 (10.502 sec)
INFO:tensorflow:lr = 0.00015555271 (10.499 sec)
INFO:tensorflow:global_step/sec: 9.37107
INFO:tensorflow:loss = 5.59257, step = 3200 (10.672 sec)
INFO:tensorflow:lr = 0.00015734472 (10.673 sec)
INFO:tensorflow:global_step/sec: 9.61674
INFO:tensorflow:loss = 5.575217, step = 3300 (10.399 sec)
INFO:tensorflow:lr = 0.00015913673 (10.399 sec)
INFO:tensorflow:global_step/sec: 9.17963
INFO:tensorflow:loss = 5.610762, step = 3400 (10.894 sec)
INFO:tensorflow:lr = 0.00016092879 (10.896 sec)
INFO:tensorflow:global_step/sec: 9.4759
INFO:tensorflow:loss = 5.8031707, step = 3500 (10.554 sec)
INFO:tensorflow:lr = 0.0001627208 (10.555 sec)
INFO:tensorflow:global_step/sec: 9.22768
INFO:tensorflow:loss = 5.616017, step = 3600 (10.835 sec)
INFO:tensorflow:lr = 0.00016451282 (10.833 sec)
INFO:tensorflow:global_step/sec: 9.45921
INFO:tensorflow:loss = 5.503318, step = 3700 (10.575 sec)
INFO:tensorflow:lr = 0.00016630483 (10.574 sec)
INFO:tensorflow:global_step/sec: 9.65632
INFO:tensorflow:loss = 5.469181, step = 3800 (10.354 sec)
INFO:tensorflow:lr = 0.00016809686 (10.353 sec)
INFO:tensorflow:global_step/sec: 9.63209
INFO:tensorflow:loss = 5.605735, step = 3900 (10.381 sec)
INFO:tensorflow:lr = 0.0001698889 (10.380 sec)
INFO:tensorflow:global_step/sec: 9.78939
INFO:tensorflow:loss = 5.615002, step = 4000 (10.218 sec)
INFO:tensorflow:lr = 0.0001716809 (10.218 sec)
INFO:tensorflow:global_step/sec: 9.52705
INFO:tensorflow:loss = 5.3957057, step = 4100 (10.494 sec)
INFO:tensorflow:lr = 0.00017347293 (10.496 sec)
INFO:tensorflow:global_step/sec: 9.82795
INFO:tensorflow:loss = 5.737772, step = 4200 (10.173 sec)
INFO:tensorflow:lr = 0.00017526494 (10.173 sec)
INFO:tensorflow:global_step/sec: 9.77213
INFO:tensorflow:loss = 5.41721, step = 4300 (10.233 sec)
INFO:tensorflow:lr = 0.000177057 (10.232 sec)
INFO:tensorflow:global_step/sec: 9.5106
INFO:tensorflow:loss = 5.3590026, step = 4400 (10.517 sec)
INFO:tensorflow:lr = 0.000178849 (10.518 sec)
INFO:tensorflow:global_step/sec: 9.76941
INFO:tensorflow:loss = 5.5810437, step = 4500 (10.236 sec)
INFO:tensorflow:lr = 0.00018064103 (10.235 sec)
INFO:tensorflow:global_step/sec: 9.41166
INFO:tensorflow:loss = 5.559284, step = 4600 (10.627 sec)
INFO:tensorflow:lr = 0.00018243304 (10.626 sec)
INFO:tensorflow:global_step/sec: 9.56269
INFO:tensorflow:loss = 5.490581, step = 4700 (10.455 sec)
INFO:tensorflow:lr = 0.00018422505 (10.455 sec)
INFO:tensorflow:global_step/sec: 9.64462
INFO:tensorflow:loss = 5.5523834, step = 4800 (10.370 sec)
INFO:tensorflow:lr = 0.0001860171 (10.370 sec)
INFO:tensorflow:global_step/sec: 9.65217
INFO:tensorflow:loss = 5.3931236, step = 4900 (10.359 sec)
INFO:tensorflow:lr = 0.00018780911 (10.360 sec)
INFO:tensorflow:global_step/sec: 9.29587
INFO:tensorflow:loss = 5.4739566, step = 5000 (10.757 sec)
INFO:tensorflow:lr = 0.00018960114 (10.759 sec)
INFO:tensorflow:global_step/sec: 9.58949
INFO:tensorflow:loss = 5.5099225, step = 5100 (10.428 sec)
INFO:tensorflow:lr = 0.00019139315 (10.426 sec)
INFO:tensorflow:global_step/sec: 9.34559
INFO:tensorflow:loss = 5.263151, step = 5200 (10.700 sec)
INFO:tensorflow:lr = 0.0001931852 (10.699 sec)
INFO:tensorflow:global_step/sec: 9.36656
INFO:tensorflow:loss = 5.316375, step = 5300 (10.677 sec)
INFO:tensorflow:lr = 0.00019497721 (10.678 sec)
INFO:tensorflow:global_step/sec: 9.36721
INFO:tensorflow:loss = 5.5502787, step = 5400 (10.676 sec)
INFO:tensorflow:lr = 0.00019676922 (10.675 sec)
INFO:tensorflow:global_step/sec: 9.64089
INFO:tensorflow:loss = 5.6246333, step = 5500 (10.372 sec)
INFO:tensorflow:lr = 0.00019856125 (10.372 sec)
INFO:tensorflow:global_step/sec: 9.69617
INFO:tensorflow:loss = 5.522617, step = 5600 (10.314 sec)
INFO:tensorflow:lr = 0.00020035326 (10.314 sec)
INFO:tensorflow:global_step/sec: 9.22217
INFO:tensorflow:loss = 5.4793525, step = 5700 (10.843 sec)
INFO:tensorflow:lr = 0.00020214531 (10.843 sec)
INFO:tensorflow:global_step/sec: 9.59383
INFO:tensorflow:loss = 5.429895, step = 5800 (10.425 sec)
INFO:tensorflow:lr = 0.00020393732 (10.424 sec)
INFO:tensorflow:global_step/sec: 9.47609
INFO:tensorflow:loss = 5.4219847, step = 5900 (10.552 sec)
INFO:tensorflow:lr = 0.00020572933 (10.553 sec)
INFO:tensorflow:global_step/sec: 9.64629
INFO:tensorflow:loss = 5.4989614, step = 6000 (10.364 sec)
INFO:tensorflow:lr = 0.00020752136 (10.366 sec)
INFO:tensorflow:global_step/sec: 9.47022
INFO:tensorflow:loss = 5.4147143, step = 6100 (10.561 sec)
INFO:tensorflow:lr = 0.00020931341 (10.560 sec)
INFO:tensorflow:global_step/sec: 9.55096
INFO:tensorflow:loss = 5.580407, step = 6200 (10.472 sec)
INFO:tensorflow:lr = 0.00021110542 (10.472 sec)
INFO:tensorflow:global_step/sec: 9.10851
INFO:tensorflow:loss = 5.465265, step = 6300 (10.976 sec)
INFO:tensorflow:lr = 0.00021289743 (10.977 sec)
INFO:tensorflow:global_step/sec: 9.62073
INFO:tensorflow:loss = 5.5079503, step = 6400 (10.396 sec)
INFO:tensorflow:lr = 0.00021468944 (10.395 sec)
INFO:tensorflow:global_step/sec: 9.4835
INFO:tensorflow:loss = 5.3989377, step = 6500 (10.543 sec)
INFO:tensorflow:lr = 0.00021648147 (10.544 sec)
INFO:tensorflow:global_step/sec: 9.23903
INFO:tensorflow:loss = 5.455261, step = 6600 (10.824 sec)
INFO:tensorflow:lr = 0.00021827352 (10.824 sec)
INFO:tensorflow:global_step/sec: 9.52242
INFO:tensorflow:loss = 5.354864, step = 6700 (10.499 sec)
INFO:tensorflow:lr = 0.00022006553 (10.499 sec)
INFO:tensorflow:global_step/sec: 9.50308
INFO:tensorflow:loss = 5.3888826, step = 6800 (10.525 sec)
INFO:tensorflow:lr = 0.00022185754 (10.525 sec)
INFO:tensorflow:global_step/sec: 9.42782
INFO:tensorflow:loss = 5.4703174, step = 6900 (10.612 sec)
INFO:tensorflow:lr = 0.00022364955 (10.610 sec)
INFO:tensorflow:global_step/sec: 9.46152
INFO:tensorflow:loss = 5.257609, step = 7000 (10.565 sec)
INFO:tensorflow:lr = 0.00022544162 (10.566 sec)
INFO:tensorflow:global_step/sec: 9.71523
INFO:tensorflow:loss = 5.425678, step = 7100 (10.293 sec)
INFO:tensorflow:lr = 0.00022723363 (10.293 sec)
INFO:tensorflow:global_step/sec: 9.41054
INFO:tensorflow:loss = 5.468957, step = 7200 (10.625 sec)
INFO:tensorflow:lr = 0.00022902564 (10.625 sec)
INFO:tensorflow:global_step/sec: 9.5384
INFO:tensorflow:loss = 5.490213, step = 7300 (10.484 sec)
INFO:tensorflow:lr = 0.00023081765 (10.485 sec)
INFO:tensorflow:global_step/sec: 9.38962
INFO:tensorflow:loss = 5.4084835, step = 7400 (10.650 sec)
INFO:tensorflow:lr = 0.00023260966 (10.650 sec)
INFO:tensorflow:global_step/sec: 9.44578
INFO:tensorflow:loss = 5.2608895, step = 7500 (10.586 sec)
INFO:tensorflow:lr = 0.00023440173 (10.586 sec)
INFO:tensorflow:global_step/sec: 9.36414
INFO:tensorflow:loss = 5.4410133, step = 7600 (10.681 sec)
INFO:tensorflow:lr = 0.00023619374 (10.680 sec)
INFO:tensorflow:global_step/sec: 9.55588
INFO:tensorflow:loss = 5.4226823, step = 7700 (10.461 sec)
INFO:tensorflow:lr = 0.00023798575 (10.461 sec)
INFO:tensorflow:global_step/sec: 9.60524
INFO:tensorflow:loss = 5.445272, step = 7800 (10.415 sec)
INFO:tensorflow:lr = 0.00023977782 (10.416 sec)
INFO:tensorflow:global_step/sec: 9.69397
INFO:tensorflow:loss = 5.3613033, step = 7900 (10.315 sec)
INFO:tensorflow:lr = 0.00024156983 (10.314 sec)
INFO:tensorflow:global_step/sec: 9.31659
INFO:tensorflow:loss = 5.3422837, step = 8000 (10.732 sec)
INFO:tensorflow:lr = 0.00024336184 (10.733 sec)
INFO:tensorflow:global_step/sec: 9.05782
INFO:tensorflow:loss = 5.2976117, step = 8100 (11.041 sec)
INFO:tensorflow:lr = 0.00024515385 (11.041 sec)
INFO:tensorflow:global_step/sec: 9.30279
INFO:tensorflow:loss = 5.369702, step = 8200 (10.750 sec)
INFO:tensorflow:lr = 0.00024694586 (10.750 sec)
INFO:tensorflow:global_step/sec: 9.33253
INFO:tensorflow:loss = 5.3575172, step = 8300 (10.719 sec)
INFO:tensorflow:lr = 0.00024873787 (10.719 sec)
INFO:tensorflow:global_step/sec: 9.33383
INFO:tensorflow:loss = 5.2213154, step = 8400 (10.708 sec)
INFO:tensorflow:lr = 0.00025052993 (10.708 sec)
INFO:tensorflow:global_step/sec: 9.49957
INFO:tensorflow:loss = 5.300407, step = 8500 (10.529 sec)
INFO:tensorflow:lr = 0.00025232194 (10.529 sec)
INFO:tensorflow:global_step/sec: 9.51828
INFO:tensorflow:loss = 5.315953, step = 8600 (10.506 sec)
INFO:tensorflow:lr = 0.00025411395 (10.506 sec)
INFO:tensorflow:global_step/sec: 9.24184
INFO:tensorflow:loss = 5.412649, step = 8700 (10.818 sec)
INFO:tensorflow:lr = 0.00025590602 (10.818 sec)
INFO:tensorflow:global_step/sec: 9.24331
INFO:tensorflow:loss = 5.3078456, step = 8800 (10.819 sec)
INFO:tensorflow:lr = 0.00025769803 (10.820 sec)
INFO:tensorflow:global_step/sec: 9.1363
INFO:tensorflow:loss = 5.4023676, step = 8900 (10.947 sec)
INFO:tensorflow:lr = 0.00025949004 (10.946 sec)
INFO:tensorflow:global_step/sec: 8.98517
INFO:tensorflow:loss = 5.483456, step = 9000 (11.130 sec)
INFO:tensorflow:lr = 0.00026128205 (11.130 sec)
INFO:tensorflow:global_step/sec: 9.49301
INFO:tensorflow:loss = 5.274673, step = 9100 (10.532 sec)
INFO:tensorflow:lr = 0.00026307406 (10.534 sec)
INFO:tensorflow:global_step/sec: 9.47168
INFO:tensorflow:loss = 5.335271, step = 9200 (10.560 sec)
INFO:tensorflow:lr = 0.00026486607 (10.558 sec)
INFO:tensorflow:global_step/sec: 9.68788
INFO:tensorflow:loss = 5.4493904, step = 9300 (10.323 sec)
INFO:tensorflow:lr = 0.00026665814 (10.322 sec)
INFO:tensorflow:global_step/sec: 9.57234
INFO:tensorflow:loss = 5.276461, step = 9400 (10.445 sec)
INFO:tensorflow:lr = 0.00026845015 (10.446 sec)
INFO:tensorflow:global_step/sec: 9.63206
INFO:tensorflow:loss = 5.2798014, step = 9500 (10.383 sec)
INFO:tensorflow:lr = 0.00027024216 (10.383 sec)
INFO:tensorflow:global_step/sec: 9.16588
INFO:tensorflow:loss = 5.3169374, step = 9600 (10.911 sec)
INFO:tensorflow:lr = 0.00027203423 (10.911 sec)
INFO:tensorflow:global_step/sec: 9.43001
INFO:tensorflow:loss = 5.4106855, step = 9700 (10.603 sec)
INFO:tensorflow:lr = 0.00027382624 (10.603 sec)
INFO:tensorflow:global_step/sec: 9.23734
INFO:tensorflow:loss = 5.334615, step = 9800 (10.824 sec)
INFO:tensorflow:lr = 0.00027561825 (10.825 sec)
INFO:tensorflow:global_step/sec: 9.48916
INFO:tensorflow:loss = 5.2714796, step = 9900 (10.538 sec)
INFO:tensorflow:lr = 0.00027741026 (10.539 sec)
INFO:tensorflow:global_step/sec: 9.45148
INFO:tensorflow:loss = 5.395485, step = 10000 (10.583 sec)
INFO:tensorflow:lr = 0.00027920227 (10.581 sec)
INFO:tensorflow:global_step/sec: 9.40397
INFO:tensorflow:loss = 5.2520475, step = 10100 (10.634 sec)
INFO:tensorflow:lr = 0.00028099428 (10.635 sec)
INFO:tensorflow:global_step/sec: 9.40283
INFO:tensorflow:loss = 5.1954203, step = 10200 (10.634 sec)
INFO:tensorflow:lr = 0.0002827863 (10.633 sec)
INFO:tensorflow:global_step/sec: 9.40064
INFO:tensorflow:loss = 5.2193794, step = 10300 (10.641 sec)
INFO:tensorflow:lr = 0.00028457836 (10.641 sec)
INFO:tensorflow:global_step/sec: 9.34103
INFO:tensorflow:loss = 5.309771, step = 10400 (10.703 sec)
INFO:tensorflow:lr = 0.00028637037 (10.702 sec)
INFO:tensorflow:global_step/sec: 9.19379
INFO:tensorflow:loss = 5.3481083, step = 10500 (10.878 sec)
INFO:tensorflow:lr = 0.00028816244 (10.878 sec)
INFO:tensorflow:global_step/sec: 9.43225
INFO:tensorflow:loss = 5.2674665, step = 10600 (10.601 sec)
INFO:tensorflow:lr = 0.00028995445 (10.602 sec)
INFO:tensorflow:global_step/sec: 9.05061
INFO:tensorflow:loss = 5.3742795, step = 10700 (11.048 sec)
INFO:tensorflow:lr = 0.00029174646 (11.048 sec)
INFO:tensorflow:global_step/sec: 9.13986
INFO:tensorflow:loss = 5.3197327, step = 10800 (10.943 sec)
INFO:tensorflow:lr = 0.00029353847 (10.943 sec)
INFO:tensorflow:global_step/sec: 8.81399
INFO:tensorflow:loss = 5.2734423, step = 10900 (11.343 sec)
INFO:tensorflow:lr = 0.00029533048 (11.343 sec)
INFO:tensorflow:global_step/sec: 9.6329
INFO:tensorflow:loss = 5.4017906, step = 11000 (10.381 sec)
INFO:tensorflow:lr = 0.0002971225 (10.381 sec)
INFO:tensorflow:global_step/sec: 9.39276
INFO:tensorflow:loss = 5.2871704, step = 11100 (10.646 sec)
INFO:tensorflow:lr = 0.0002989145 (10.646 sec)
INFO:tensorflow:global_step/sec: 8.89523
INFO:tensorflow:loss = 5.2185893, step = 11200 (11.246 sec)
INFO:tensorflow:lr = 0.00030070657 (11.244 sec)
INFO:tensorflow:global_step/sec: 9.39774
INFO:tensorflow:loss = 5.3076186, step = 11300 (10.639 sec)
INFO:tensorflow:lr = 0.00030249858 (10.640 sec)
INFO:tensorflow:global_step/sec: 9.85811
INFO:tensorflow:loss = 5.3705683, step = 11400 (10.146 sec)
INFO:tensorflow:lr = 0.00030429062 (10.144 sec)
INFO:tensorflow:global_step/sec: 9.47035
INFO:tensorflow:loss = 5.3904276, step = 11500 (10.557 sec)
INFO:tensorflow:lr = 0.00030608266 (10.559 sec)
INFO:tensorflow:global_step/sec: 9.28909
INFO:tensorflow:loss = 5.2577066, step = 11600 (10.765 sec)
INFO:tensorflow:lr = 0.00030787467 (10.764 sec)
INFO:tensorflow:global_step/sec: 9.84435
INFO:tensorflow:loss = 5.3427906, step = 11700 (10.157 sec)
INFO:tensorflow:lr = 0.00030966668 (10.157 sec)
INFO:tensorflow:global_step/sec: 9.8187
INFO:tensorflow:loss = 5.196344, step = 11800 (10.184 sec)
INFO:tensorflow:lr = 0.0003114587 (10.187 sec)
INFO:tensorflow:global_step/sec: 9.58391
INFO:tensorflow:loss = 5.1877666, step = 11900 (10.437 sec)
INFO:tensorflow:lr = 0.0003132507 (10.435 sec)
INFO:tensorflow:global_step/sec: 9.5932
INFO:tensorflow:loss = 5.273709, step = 12000 (10.423 sec)
INFO:tensorflow:lr = 0.0003150427 (10.423 sec)
INFO:tensorflow:global_step/sec: 9.53313
INFO:tensorflow:loss = 5.275315, step = 12100 (10.490 sec)
INFO:tensorflow:lr = 0.00031683478 (10.490 sec)
INFO:tensorflow:global_step/sec: 9.24848
INFO:tensorflow:loss = 5.347632, step = 12200 (10.811 sec)
INFO:tensorflow:lr = 0.0003186268 (10.810 sec)
INFO:tensorflow:global_step/sec: 9.76986
INFO:tensorflow:loss = 5.1148157, step = 12300 (10.239 sec)
INFO:tensorflow:lr = 0.00032041883 (10.240 sec)
INFO:tensorflow:global_step/sec: 9.41562
INFO:tensorflow:loss = 5.227973, step = 12400 (10.619 sec)
INFO:tensorflow:lr = 0.00032221084 (10.619 sec)
INFO:tensorflow:global_step/sec: 9.49199
INFO:tensorflow:loss = 5.2387366, step = 12500 (10.534 sec)
INFO:tensorflow:lr = 0.00032400287 (10.533 sec)
INFO:tensorflow:global_step/sec: 9.57954
INFO:tensorflow:loss = 5.0977798, step = 12600 (10.439 sec)
INFO:tensorflow:lr = 0.00032579488 (10.440 sec)
INFO:tensorflow:global_step/sec: 9.33589
INFO:tensorflow:loss = 5.2063737, step = 12700 (10.710 sec)
INFO:tensorflow:lr = 0.0003275869 (10.710 sec)
INFO:tensorflow:global_step/sec: 9.10658
INFO:tensorflow:loss = 5.295964, step = 12800 (10.991 sec)
INFO:tensorflow:lr = 0.0003293789 (10.992 sec)
INFO:tensorflow:global_step/sec: 9.52902
INFO:tensorflow:loss = 5.3541036, step = 12900 (10.486 sec)
INFO:tensorflow:lr = 0.00033117092 (10.485 sec)
INFO:tensorflow:global_step/sec: 9.67691
INFO:tensorflow:loss = 5.1890435, step = 13000 (10.332 sec)
INFO:tensorflow:lr = 0.00033296298 (10.333 sec)
INFO:tensorflow:global_step/sec: 9.59669
INFO:tensorflow:loss = 5.0991793, step = 13100 (10.423 sec)
INFO:tensorflow:lr = 0.000334755 (10.421 sec)
INFO:tensorflow:global_step/sec: 9.23686
INFO:tensorflow:loss = 5.2923245, step = 13200 (10.825 sec)
INFO:tensorflow:lr = 0.00033654703 (10.825 sec)
INFO:tensorflow:global_step/sec: 9.38213
INFO:tensorflow:loss = 5.431452, step = 13300 (10.660 sec)
INFO:tensorflow:lr = 0.00033833904 (10.660 sec)
INFO:tensorflow:global_step/sec: 9.31188
INFO:tensorflow:loss = 5.1751623, step = 13400 (10.741 sec)
INFO:tensorflow:lr = 0.00034013108 (10.742 sec)
INFO:tensorflow:global_step/sec: 9.24699
INFO:tensorflow:loss = 5.3359776, step = 13500 (10.809 sec)
INFO:tensorflow:lr = 0.0003419231 (10.809 sec)
INFO:tensorflow:global_step/sec: 9.41265
INFO:tensorflow:loss = 5.3060904, step = 13600 (10.627 sec)
INFO:tensorflow:lr = 0.0003437151 (10.627 sec)
INFO:tensorflow:global_step/sec: 9.25623
INFO:tensorflow:loss = 5.199159, step = 13700 (10.801 sec)
INFO:tensorflow:lr = 0.0003455071 (10.802 sec)
INFO:tensorflow:global_step/sec: 9.47971
INFO:tensorflow:loss = 5.248309, step = 13800 (10.551 sec)
INFO:tensorflow:lr = 0.00034729912 (10.550 sec)
INFO:tensorflow:global_step/sec: 9.4577
INFO:tensorflow:loss = 5.3227744, step = 13900 (10.575 sec)
INFO:tensorflow:lr = 0.0003490912 (10.574 sec)
INFO:tensorflow:global_step/sec: 9.80108
INFO:tensorflow:loss = 5.2083745, step = 14000 (10.201 sec)
INFO:tensorflow:lr = 0.0003508832 (10.202 sec)
INFO:tensorflow:global_step/sec: 9.12215
INFO:tensorflow:loss = 5.0114703, step = 14100 (10.964 sec)
INFO:tensorflow:lr = 0.00035267524 (10.963 sec)
INFO:tensorflow:global_step/sec: 9.48393
INFO:tensorflow:loss = 5.240639, step = 14200 (10.543 sec)
INFO:tensorflow:lr = 0.00035446728 (10.544 sec)
INFO:tensorflow:global_step/sec: 8.99056
INFO:tensorflow:loss = 5.213593, step = 14300 (11.122 sec)
INFO:tensorflow:lr = 0.0003562593 (11.121 sec)
INFO:tensorflow:global_step/sec: 9.37065
INFO:tensorflow:loss = 5.1901445, step = 14400 (10.670 sec)
INFO:tensorflow:lr = 0.0003580513 (10.674 sec)
INFO:tensorflow:global_step/sec: 9.02198
INFO:tensorflow:loss = 5.0916905, step = 14500 (11.086 sec)
INFO:tensorflow:lr = 0.0003598433 (11.083 sec)
INFO:tensorflow:global_step/sec: 9.33291
INFO:tensorflow:loss = 5.1945896, step = 14600 (10.715 sec)
INFO:tensorflow:lr = 0.00036163532 (10.715 sec)
INFO:tensorflow:global_step/sec: 9.47573
INFO:tensorflow:loss = 5.136004, step = 14700 (10.552 sec)
INFO:tensorflow:lr = 0.00036342733 (10.553 sec)
INFO:tensorflow:global_step/sec: 9.61313
INFO:tensorflow:loss = 5.096282, step = 14800 (10.404 sec)
INFO:tensorflow:lr = 0.0003652194 (10.403 sec)
INFO:tensorflow:global_step/sec: 9.48212
INFO:tensorflow:loss = 5.1238246, step = 14900 (10.546 sec)
INFO:tensorflow:lr = 0.0003670114 (10.546 sec)
INFO:tensorflow:global_step/sec: 9.17427
INFO:tensorflow:loss = 5.2331834, step = 15000 (10.900 sec)
INFO:tensorflow:lr = 0.00036880345 (10.900 sec)
INFO:tensorflow:global_step/sec: 9.52248
INFO:tensorflow:loss = 5.3915863, step = 15100 (10.501 sec)
INFO:tensorflow:lr = 0.0003705955 (10.500 sec)
INFO:tensorflow:global_step/sec: 9.50163
INFO:tensorflow:loss = 5.1143703, step = 15200 (10.523 sec)
INFO:tensorflow:lr = 0.0003723875 (10.523 sec)
INFO:tensorflow:global_step/sec: 9.51777
INFO:tensorflow:loss = 5.182392, step = 15300 (10.506 sec)
INFO:tensorflow:lr = 0.0003741795 (10.507 sec)
INFO:tensorflow:global_step/sec: 9.58017
INFO:tensorflow:loss = 5.202959, step = 15400 (10.440 sec)
INFO:tensorflow:lr = 0.00037597152 (10.439 sec)
INFO:tensorflow:global_step/sec: 9.61115
INFO:tensorflow:loss = 5.3520365, step = 15500 (10.404 sec)
INFO:tensorflow:lr = 0.00037776353 (10.405 sec)
INFO:tensorflow:global_step/sec: 9.4878
INFO:tensorflow:loss = 5.1896334, step = 15600 (10.540 sec)
INFO:tensorflow:lr = 0.0003795556 (10.540 sec)
INFO:tensorflow:global_step/sec: 9.52809
INFO:tensorflow:loss = 5.2342257, step = 15700 (10.496 sec)
INFO:tensorflow:lr = 0.0003813476 (10.495 sec)
INFO:tensorflow:global_step/sec: 9.4086
INFO:tensorflow:loss = 5.2121305, step = 15800 (10.629 sec)
INFO:tensorflow:lr = 0.00038313962 (10.629 sec)
INFO:tensorflow:global_step/sec: 9.38529
INFO:tensorflow:loss = 5.181575, step = 15900 (10.655 sec)
INFO:tensorflow:lr = 0.00038493166 (10.654 sec)
INFO:tensorflow:global_step/sec: 9.37624
INFO:tensorflow:loss = 5.264327, step = 16000 (10.666 sec)
INFO:tensorflow:lr = 0.00038672367 (10.666 sec)
INFO:tensorflow:global_step/sec: 8.98048
INFO:tensorflow:loss = 5.263517, step = 16100 (11.134 sec)
INFO:tensorflow:lr = 0.0003885157 (11.136 sec)
INFO:tensorflow:global_step/sec: 9.37316
INFO:tensorflow:loss = 5.229684, step = 16200 (10.670 sec)
INFO:tensorflow:lr = 0.00039030772 (10.669 sec)
INFO:tensorflow:global_step/sec: 9.4933
INFO:tensorflow:loss = 5.246074, step = 16300 (10.534 sec)
INFO:tensorflow:lr = 0.00039209973 (10.534 sec)
INFO:tensorflow:global_step/sec: 9.10833
INFO:tensorflow:loss = 5.3365836, step = 16400 (10.976 sec)
INFO:tensorflow:lr = 0.00039389174 (10.976 sec)
INFO:tensorflow:global_step/sec: 9.6444
INFO:tensorflow:loss = 5.253062, step = 16500 (10.370 sec)
INFO:tensorflow:lr = 0.00039568378 (10.370 sec)
INFO:tensorflow:global_step/sec: 9.30588
INFO:tensorflow:loss = 5.277682, step = 16600 (10.746 sec)
INFO:tensorflow:lr = 0.00039747581 (10.748 sec)
INFO:tensorflow:global_step/sec: 9.51747
INFO:tensorflow:loss = 5.296861, step = 16700 (10.506 sec)
INFO:tensorflow:lr = 0.00039926782 (10.505 sec)
INFO:tensorflow:global_step/sec: 9.57023
INFO:tensorflow:loss = 5.0729628, step = 16800 (10.450 sec)
INFO:tensorflow:lr = 0.00040105986 (10.451 sec)
INFO:tensorflow:global_step/sec: 9.68155
INFO:tensorflow:loss = 5.2019506, step = 16900 (10.331 sec)
INFO:tensorflow:lr = 0.00040285187 (10.332 sec)
INFO:tensorflow:global_step/sec: 9.52325
INFO:tensorflow:loss = 5.2418485, step = 17000 (10.496 sec)
INFO:tensorflow:lr = 0.0004046439 (10.497 sec)
INFO:tensorflow:global_step/sec: 9.47466
INFO:tensorflow:loss = 5.156396, step = 17100 (10.557 sec)
INFO:tensorflow:lr = 0.00040643592 (10.556 sec)
INFO:tensorflow:global_step/sec: 9.43814
INFO:tensorflow:loss = 5.28926, step = 17200 (10.593 sec)
INFO:tensorflow:lr = 0.00040822793 (10.595 sec)
INFO:tensorflow:global_step/sec: 9.30126
INFO:tensorflow:loss = 5.2130136, step = 17300 (10.751 sec)
INFO:tensorflow:lr = 0.00041001994 (10.748 sec)
INFO:tensorflow:global_step/sec: 9.67383
INFO:tensorflow:loss = 5.0762405, step = 17400 (10.338 sec)
INFO:tensorflow:lr = 0.00041181198 (10.338 sec)
INFO:tensorflow:global_step/sec: 9.60888
INFO:tensorflow:loss = 5.1943226, step = 17500 (10.407 sec)
INFO:tensorflow:lr = 0.00041360402 (10.408 sec)
INFO:tensorflow:global_step/sec: 9.50219
INFO:tensorflow:loss = 5.294518, step = 17600 (10.525 sec)
INFO:tensorflow:lr = 0.00041539603 (10.527 sec)
INFO:tensorflow:global_step/sec: 9.43169
INFO:tensorflow:loss = 5.0493245, step = 17700 (10.603 sec)
INFO:tensorflow:lr = 0.00041718807 (10.600 sec)
INFO:tensorflow:global_step/sec: 9.67039
INFO:tensorflow:loss = 5.2163715, step = 17800 (10.342 sec)
INFO:tensorflow:lr = 0.00041898008 (10.342 sec)
INFO:tensorflow:global_step/sec: 9.2621
INFO:tensorflow:loss = 4.999476, step = 17900 (10.794 sec)
INFO:tensorflow:lr = 0.0004207721 (10.795 sec)
INFO:tensorflow:global_step/sec: 9.72487
INFO:tensorflow:loss = 5.1670976, step = 18000 (10.284 sec)
INFO:tensorflow:lr = 0.00042256413 (10.285 sec)
INFO:tensorflow:global_step/sec: 9.45329
INFO:tensorflow:loss = 5.088095, step = 18100 (10.580 sec)
INFO:tensorflow:lr = 0.00042435614 (10.580 sec)
INFO:tensorflow:global_step/sec: 9.84978
INFO:tensorflow:loss = 5.1727524, step = 18200 (10.151 sec)
INFO:tensorflow:lr = 0.00042614815 (10.150 sec)
INFO:tensorflow:global_step/sec: 9.39427
INFO:tensorflow:loss = 4.9821234, step = 18300 (10.644 sec)
INFO:tensorflow:lr = 0.0004279402 (10.644 sec)
INFO:tensorflow:global_step/sec: 9.62825
INFO:tensorflow:loss = 5.046456, step = 18400 (10.386 sec)
INFO:tensorflow:lr = 0.0004297322 (10.387 sec)
INFO:tensorflow:global_step/sec: 9.32231
INFO:tensorflow:loss = 5.0560246, step = 18500 (10.729 sec)
INFO:tensorflow:lr = 0.00043152424 (10.728 sec)
INFO:tensorflow:global_step/sec: 9.62334
INFO:tensorflow:loss = 5.2404575, step = 18600 (10.391 sec)
INFO:tensorflow:lr = 0.00043331628 (10.391 sec)
INFO:tensorflow:global_step/sec: 9.65077
INFO:tensorflow:loss = 5.1200423, step = 18700 (10.362 sec)
INFO:tensorflow:lr = 0.0004351083 (10.363 sec)
INFO:tensorflow:global_step/sec: 9.44334
INFO:tensorflow:loss = 4.978124, step = 18800 (10.588 sec)
INFO:tensorflow:lr = 0.0004369003 (10.587 sec)
INFO:tensorflow:global_step/sec: 9.5336
INFO:tensorflow:loss = 5.1311655, step = 18900 (10.489 sec)
INFO:tensorflow:lr = 0.0004386923 (10.491 sec)
INFO:tensorflow:global_step/sec: 9.7246
INFO:tensorflow:loss = 5.125824, step = 19000 (10.283 sec)
INFO:tensorflow:lr = 0.00044048435 (10.283 sec)
INFO:tensorflow:global_step/sec: 8.98551
INFO:tensorflow:loss = 5.180024, step = 19100 (11.127 sec)
INFO:tensorflow:lr = 0.00044227636 (11.127 sec)
INFO:tensorflow:global_step/sec: 9.40787
INFO:tensorflow:loss = 5.2193055, step = 19200 (10.632 sec)
INFO:tensorflow:lr = 0.0004440684 (10.632 sec)
INFO:tensorflow:global_step/sec: 9.52289
INFO:tensorflow:loss = 5.1205177, step = 19300 (10.500 sec)
INFO:tensorflow:lr = 0.0004458604 (10.501 sec)
INFO:tensorflow:global_step/sec: 9.24599
INFO:tensorflow:loss = 5.092633, step = 19400 (10.816 sec)
INFO:tensorflow:lr = 0.00044765242 (10.815 sec)
INFO:tensorflow:global_step/sec: 9.2783
INFO:tensorflow:loss = 5.195169, step = 19500 (10.775 sec)
INFO:tensorflow:lr = 0.0004494445 (10.776 sec)
INFO:tensorflow:global_step/sec: 9.66929
INFO:tensorflow:loss = 5.181858, step = 19600 (10.345 sec)
INFO:tensorflow:lr = 0.0004512365 (10.345 sec)
INFO:tensorflow:global_step/sec: 9.71316
INFO:tensorflow:loss = 5.076673, step = 19700 (10.295 sec)
INFO:tensorflow:lr = 0.0004530285 (10.296 sec)
INFO:tensorflow:global_step/sec: 9.45051
INFO:tensorflow:loss = 5.0781593, step = 19800 (10.579 sec)
INFO:tensorflow:lr = 0.00045482052 (10.577 sec)
INFO:tensorflow:global_step/sec: 9.55697
INFO:tensorflow:loss = 4.937835, step = 19900 (10.466 sec)
INFO:tensorflow:lr = 0.00045661256 (10.467 sec)
INFO:tensorflow:global_step/sec: 9.1631
INFO:tensorflow:loss = 5.1505866, step = 20000 (10.913 sec)
INFO:tensorflow:lr = 0.0004584046 (10.912 sec)
INFO:tensorflow:global_step/sec: 9.4915
INFO:tensorflow:loss = 5.133694, step = 20100 (10.536 sec)
INFO:tensorflow:lr = 0.0004601966 (10.536 sec)
INFO:tensorflow:global_step/sec: 9.57019
INFO:tensorflow:loss = 5.06313, step = 20200 (10.448 sec)
INFO:tensorflow:lr = 0.00046198862 (10.449 sec)
INFO:tensorflow:global_step/sec: 9.79037
INFO:tensorflow:loss = 5.0452394, step = 20300 (10.212 sec)
INFO:tensorflow:lr = 0.00046378063 (10.213 sec)
INFO:tensorflow:global_step/sec: 9.27914
INFO:tensorflow:loss = 5.2335463, step = 20400 (10.779 sec)
INFO:tensorflow:lr = 0.00046557267 (10.779 sec)
INFO:tensorflow:global_step/sec: 9.13557
INFO:tensorflow:loss = 5.10505, step = 20500 (10.947 sec)
INFO:tensorflow:lr = 0.0004673647 (10.946 sec)
INFO:tensorflow:global_step/sec: 9.16514
INFO:tensorflow:loss = 5.094977, step = 20600 (10.908 sec)
INFO:tensorflow:lr = 0.00046915672 (10.909 sec)
INFO:tensorflow:global_step/sec: 9.38076
INFO:tensorflow:loss = 5.1793923, step = 20700 (10.662 sec)
INFO:tensorflow:lr = 0.00047094873 (10.661 sec)
INFO:tensorflow:global_step/sec: 9.5493
INFO:tensorflow:loss = 5.0512815, step = 20800 (10.473 sec)
INFO:tensorflow:lr = 0.00047274074 (10.473 sec)
INFO:tensorflow:global_step/sec: 9.49999
INFO:tensorflow:loss = 5.065953, step = 20900 (10.524 sec)
INFO:tensorflow:lr = 0.0004745328 (10.525 sec)
INFO:tensorflow:global_step/sec: 9.53287
INFO:tensorflow:loss = 5.0509553, step = 21000 (10.492 sec)
INFO:tensorflow:lr = 0.0004763248 (10.493 sec)
INFO:tensorflow:global_step/sec: 9.482
INFO:tensorflow:loss = 5.1141, step = 21100 (10.546 sec)
INFO:tensorflow:lr = 0.00047811682 (10.544 sec)
INFO:tensorflow:global_step/sec: 9.48085
INFO:tensorflow:loss = 5.178147, step = 21200 (10.549 sec)
INFO:tensorflow:lr = 0.00047990883 (10.549 sec)
INFO:tensorflow:global_step/sec: 9.05672
INFO:tensorflow:loss = 5.056297, step = 21300 (11.041 sec)
INFO:tensorflow:lr = 0.00048170084 (11.041 sec)
INFO:tensorflow:global_step/sec: 9.70846
INFO:tensorflow:loss = 5.2183027, step = 21400 (10.299 sec)
INFO:tensorflow:lr = 0.0004834929 (10.300 sec)
INFO:tensorflow:global_step/sec: 9.50661
INFO:tensorflow:loss = 5.1443934, step = 21500 (10.520 sec)
INFO:tensorflow:lr = 0.00048528492 (10.519 sec)
INFO:tensorflow:global_step/sec: 9.26474
INFO:tensorflow:loss = 5.140825, step = 21600 (10.792 sec)
INFO:tensorflow:lr = 0.00048707693 (10.792 sec)
INFO:tensorflow:global_step/sec: 9.38964
INFO:tensorflow:loss = 5.0313034, step = 21700 (10.651 sec)
INFO:tensorflow:lr = 0.0004888689 (10.651 sec)
INFO:tensorflow:global_step/sec: 9.62222
INFO:tensorflow:loss = 5.055342, step = 21800 (10.391 sec)
INFO:tensorflow:lr = 0.000490661 (10.397 sec)
INFO:tensorflow:global_step/sec: 9.53237
INFO:tensorflow:loss = 5.0808406, step = 21900 (10.492 sec)
INFO:tensorflow:lr = 0.000492453 (10.487 sec)
INFO:tensorflow:global_step/sec: 9.74152
INFO:tensorflow:loss = 5.1622124, step = 22000 (10.266 sec)
INFO:tensorflow:lr = 0.00049424503 (10.265 sec)
INFO:tensorflow:global_step/sec: 9.516
INFO:tensorflow:loss = 5.157396, step = 22100 (10.509 sec)
INFO:tensorflow:lr = 0.000496037 (10.510 sec)
INFO:tensorflow:global_step/sec: 9.07583
INFO:tensorflow:loss = 5.137361, step = 22200 (11.017 sec)
INFO:tensorflow:lr = 0.00049782905 (11.016 sec)
INFO:tensorflow:global_step/sec: 9.45514
INFO:tensorflow:loss = 5.0328975, step = 22300 (10.577 sec)
INFO:tensorflow:lr = 0.0004996211 (10.578 sec)
INFO:tensorflow:global_step/sec: 9.51083
INFO:tensorflow:loss = 5.121226, step = 22400 (10.512 sec)
INFO:tensorflow:lr = 0.00050141313 (10.512 sec)
INFO:tensorflow:global_step/sec: 9.33951
INFO:tensorflow:loss = 4.971635, step = 22500 (10.710 sec)
INFO:tensorflow:lr = 0.0005032051 (10.711 sec)
INFO:tensorflow:global_step/sec: 9.64867
INFO:tensorflow:loss = 5.090142, step = 22600 (10.364 sec)
INFO:tensorflow:lr = 0.00050499715 (10.363 sec)
INFO:tensorflow:global_step/sec: 9.67859
INFO:tensorflow:loss = 5.094214, step = 22700 (10.332 sec)
INFO:tensorflow:lr = 0.0005067892 (10.334 sec)
INFO:tensorflow:global_step/sec: 9.09592
INFO:tensorflow:loss = 5.1820188, step = 22800 (10.993 sec)
INFO:tensorflow:lr = 0.00050858123 (10.994 sec)
INFO:tensorflow:global_step/sec: 9.12329
INFO:tensorflow:loss = 5.186921, step = 22900 (10.964 sec)
INFO:tensorflow:lr = 0.0005103732 (10.959 sec)
INFO:tensorflow:global_step/sec: 9.28302
INFO:tensorflow:loss = 5.2035203, step = 23000 (10.771 sec)
INFO:tensorflow:lr = 0.00051216525 (10.772 sec)
INFO:tensorflow:global_step/sec: 9.33365
INFO:tensorflow:loss = 4.9803834, step = 23100 (10.715 sec)
INFO:tensorflow:lr = 0.00051395723 (10.714 sec)
INFO:tensorflow:global_step/sec: 9.53787
INFO:tensorflow:loss = 5.1692257, step = 23200 (10.485 sec)
INFO:tensorflow:lr = 0.0005157493 (10.485 sec)
INFO:tensorflow:global_step/sec: 9.36692
INFO:tensorflow:loss = 5.1639504, step = 23300 (10.675 sec)
INFO:tensorflow:lr = 0.0005175413 (10.676 sec)
INFO:tensorflow:global_step/sec: 9.60574
INFO:tensorflow:loss = 5.258109, step = 23400 (10.411 sec)
INFO:tensorflow:lr = 0.00051933335 (10.410 sec)
INFO:tensorflow:global_step/sec: 9.51388
INFO:tensorflow:loss = 5.2187853, step = 23500 (10.510 sec)
INFO:tensorflow:lr = 0.00052112533 (10.510 sec)
INFO:tensorflow:global_step/sec: 9.13428
INFO:tensorflow:loss = 5.10759, step = 23600 (10.949 sec)
INFO:tensorflow:lr = 0.0005229174 (10.949 sec)
INFO:tensorflow:global_step/sec: 9.40798
INFO:tensorflow:loss = 5.1671615, step = 23700 (10.628 sec)
INFO:tensorflow:lr = 0.0005247094 (10.628 sec)
INFO:tensorflow:global_step/sec: 9.46928
INFO:tensorflow:loss = 5.1048703, step = 23800 (10.561 sec)
INFO:tensorflow:lr = 0.00052650145 (10.561 sec)
INFO:tensorflow:global_step/sec: 8.96179
INFO:tensorflow:loss = 5.1232705, step = 23900 (11.159 sec)
INFO:tensorflow:lr = 0.0005282934 (11.159 sec)
INFO:tensorflow:global_step/sec: 8.94983
INFO:tensorflow:loss = 5.196185, step = 24000 (11.172 sec)
INFO:tensorflow:lr = 0.00053008547 (11.172 sec)
INFO:tensorflow:global_step/sec: 8.9806
INFO:tensorflow:loss = 5.142331, step = 24100 (11.133 sec)
INFO:tensorflow:lr = 0.0005318775 (11.134 sec)
INFO:tensorflow:global_step/sec: 9.2065
INFO:tensorflow:loss = 5.077751, step = 24200 (10.862 sec)
INFO:tensorflow:lr = 0.00053366955 (10.863 sec)
INFO:tensorflow:global_step/sec: 9.15252
INFO:tensorflow:loss = 5.1835365, step = 24300 (10.927 sec)
INFO:tensorflow:lr = 0.0005354615 (10.928 sec)
INFO:tensorflow:global_step/sec: 9.12864
INFO:tensorflow:loss = 5.1684585, step = 24400 (10.955 sec)
INFO:tensorflow:lr = 0.00053725357 (10.953 sec)
INFO:tensorflow:global_step/sec: 9.39152
INFO:tensorflow:loss = 5.2003446, step = 24500 (10.647 sec)
INFO:tensorflow:lr = 0.0005390456 (10.647 sec)
INFO:tensorflow:global_step/sec: 9.26444
INFO:tensorflow:loss = 5.1787515, step = 24600 (10.795 sec)
INFO:tensorflow:lr = 0.00054083765 (10.795 sec)
INFO:tensorflow:global_step/sec: 9.06221
INFO:tensorflow:loss = 5.062696, step = 24700 (11.034 sec)
INFO:tensorflow:lr = 0.0005426296 (11.034 sec)
INFO:tensorflow:global_step/sec: 9.33903
INFO:tensorflow:loss = 5.015447, step = 24800 (10.710 sec)
INFO:tensorflow:lr = 0.00054442167 (10.711 sec)
INFO:tensorflow:global_step/sec: 8.95611
INFO:tensorflow:loss = 5.108074, step = 24900 (11.162 sec)
INFO:tensorflow:lr = 0.00054621365 (11.165 sec)
INFO:tensorflow:global_step/sec: 8.97971
INFO:tensorflow:loss = 5.1784277, step = 25000 (11.137 sec)
INFO:tensorflow:lr = 0.00054800574 (11.134 sec)
INFO:tensorflow:global_step/sec: 8.8737
INFO:tensorflow:loss = 5.0889997, step = 25100 (11.272 sec)
INFO:tensorflow:lr = 0.0005497977 (11.271 sec)
INFO:tensorflow:global_step/sec: 8.99304
INFO:tensorflow:loss = 5.1338367, step = 25200 (11.118 sec)
INFO:tensorflow:lr = 0.00055158976 (11.118 sec)
INFO:tensorflow:global_step/sec: 9.20564
INFO:tensorflow:loss = 5.049507, step = 25300 (10.863 sec)
INFO:tensorflow:lr = 0.00055338175 (10.865 sec)
INFO:tensorflow:global_step/sec: 8.86175
INFO:tensorflow:loss = 5.004919, step = 25400 (11.285 sec)
INFO:tensorflow:lr = 0.00055517384 (11.284 sec)
INFO:tensorflow:global_step/sec: 8.75508
INFO:tensorflow:loss = 5.2403455, step = 25500 (11.422 sec)
INFO:tensorflow:lr = 0.0005569658 (11.422 sec)
INFO:tensorflow:global_step/sec: 9.18257
INFO:tensorflow:loss = 5.125975, step = 25600 (10.890 sec)
INFO:tensorflow:lr = 0.00055875786 (10.890 sec)
INFO:tensorflow:global_step/sec: 8.96184
INFO:tensorflow:loss = 5.0116687, step = 25700 (11.157 sec)
INFO:tensorflow:lr = 0.00056054984 (11.158 sec)
INFO:tensorflow:global_step/sec: 8.52834
INFO:tensorflow:loss = 5.0436788, step = 25800 (11.728 sec)
INFO:tensorflow:lr = 0.0005623419 (11.726 sec)
INFO:tensorflow:global_step/sec: 9.19879
INFO:tensorflow:loss = 5.07081, step = 25900 (10.872 sec)
INFO:tensorflow:lr = 0.0005641339 (10.874 sec)
INFO:tensorflow:global_step/sec: 9.07916
INFO:tensorflow:loss = 5.0617657, step = 26000 (11.013 sec)
INFO:tensorflow:lr = 0.00056592596 (11.011 sec)
INFO:tensorflow:global_step/sec: 9.15357
INFO:tensorflow:loss = 5.0214477, step = 26100 (10.926 sec)
INFO:tensorflow:lr = 0.00056771794 (10.926 sec)
INFO:tensorflow:global_step/sec: 8.74457
INFO:tensorflow:loss = 5.129234, step = 26200 (11.431 sec)
INFO:tensorflow:lr = 0.00056951 (11.432 sec)
INFO:tensorflow:global_step/sec: 9.20484
INFO:tensorflow:loss = 5.0483747, step = 26300 (10.866 sec)
INFO:tensorflow:lr = 0.000571302 (10.865 sec)
INFO:tensorflow:global_step/sec: 9.0836
INFO:tensorflow:loss = 5.135254, step = 26400 (11.011 sec)
INFO:tensorflow:lr = 0.00057309406 (11.011 sec)
INFO:tensorflow:global_step/sec: 9.65397
INFO:tensorflow:loss = 5.09909, step = 26500 (10.356 sec)
INFO:tensorflow:lr = 0.00057488604 (10.357 sec)
INFO:tensorflow:global_step/sec: 9.27336
INFO:tensorflow:loss = 5.0942545, step = 26600 (10.782 sec)
INFO:tensorflow:lr = 0.0005766781 (10.781 sec)
INFO:tensorflow:global_step/sec: 9.31149
INFO:tensorflow:loss = 5.074828, step = 26700 (10.741 sec)
INFO:tensorflow:lr = 0.00057847006 (10.742 sec)
INFO:tensorflow:global_step/sec: 9.25461
INFO:tensorflow:loss = 5.084673, step = 26800 (10.805 sec)
INFO:tensorflow:lr = 0.00058026216 (10.805 sec)
INFO:tensorflow:global_step/sec: 8.69425
INFO:tensorflow:loss = 4.935843, step = 26900 (11.502 sec)
INFO:tensorflow:lr = 0.00058205414 (11.501 sec)
INFO:tensorflow:global_step/sec: 9.09111
INFO:tensorflow:loss = 5.1584783, step = 27000 (10.999 sec)
INFO:tensorflow:lr = 0.0005838462 (11.000 sec)
INFO:tensorflow:global_step/sec: 9.30537
INFO:tensorflow:loss = 5.091224, step = 27100 (10.745 sec)
INFO:tensorflow:lr = 0.00058563816 (10.744 sec)
INFO:tensorflow:global_step/sec: 9.42432
INFO:tensorflow:loss = 4.955161, step = 27200 (10.614 sec)
INFO:tensorflow:lr = 0.0005874302 (10.614 sec)
INFO:tensorflow:global_step/sec: 9.34376
INFO:tensorflow:loss = 4.995959, step = 27300 (10.702 sec)
INFO:tensorflow:lr = 0.00058922224 (10.702 sec)
INFO:tensorflow:global_step/sec: 8.91428
INFO:tensorflow:loss = 5.2494087, step = 27400 (11.218 sec)
INFO:tensorflow:lr = 0.0005910143 (11.219 sec)
INFO:tensorflow:global_step/sec: 8.99617
INFO:tensorflow:loss = 5.1566463, step = 27500 (11.116 sec)
INFO:tensorflow:lr = 0.00059280626 (11.115 sec)
INFO:tensorflow:global_step/sec: 8.97073
INFO:tensorflow:loss = 5.157406, step = 27600 (11.148 sec)
INFO:tensorflow:lr = 0.0005945983 (11.148 sec)
INFO:tensorflow:global_step/sec: 9.24686
INFO:tensorflow:loss = 5.03269, step = 27700 (10.813 sec)
INFO:tensorflow:lr = 0.00059639034 (10.813 sec)
INFO:tensorflow:global_step/sec: 8.96445
INFO:tensorflow:loss = 4.99574, step = 27800 (11.155 sec)
INFO:tensorflow:lr = 0.0005981824 (11.157 sec)
INFO:tensorflow:global_step/sec: 9.02618
INFO:tensorflow:loss = 5.0618687, step = 27900 (11.080 sec)
INFO:tensorflow:lr = 0.00059997436 (11.079 sec)
INFO:tensorflow:global_step/sec: 9.36297
INFO:tensorflow:loss = 4.967255, step = 28000 (10.680 sec)
INFO:tensorflow:lr = 0.0006017664 (10.680 sec)
INFO:tensorflow:global_step/sec: 9.27127
INFO:tensorflow:loss = 4.918425, step = 28100 (10.786 sec)
INFO:tensorflow:lr = 0.00060355844 (10.786 sec)
INFO:tensorflow:global_step/sec: 9.49182
INFO:tensorflow:loss = 5.1641984, step = 28200 (10.535 sec)
INFO:tensorflow:lr = 0.0006053505 (10.535 sec)
INFO:tensorflow:global_step/sec: 9.3774
INFO:tensorflow:loss = 5.039311, step = 28300 (10.664 sec)
INFO:tensorflow:lr = 0.00060714246 (10.664 sec)
INFO:tensorflow:global_step/sec: 9.19179
INFO:tensorflow:loss = 5.066581, step = 28400 (10.877 sec)
INFO:tensorflow:lr = 0.0006089345 (10.878 sec)
INFO:tensorflow:global_step/sec: 9.48473
INFO:tensorflow:loss = 5.068813, step = 28500 (10.545 sec)
INFO:tensorflow:lr = 0.0006107265 (10.544 sec)
INFO:tensorflow:global_step/sec: 8.85006
INFO:tensorflow:loss = 4.9568076, step = 28600 (11.299 sec)
INFO:tensorflow:lr = 0.0006125186 (11.299 sec)
INFO:tensorflow:global_step/sec: 9.14199
INFO:tensorflow:loss = 5.0653033, step = 28700 (10.940 sec)
INFO:tensorflow:lr = 0.00061431056 (10.939 sec)
INFO:tensorflow:global_step/sec: 9.46158
INFO:tensorflow:loss = 5.0284724, step = 28800 (10.570 sec)
INFO:tensorflow:lr = 0.0006161026 (10.569 sec)
INFO:tensorflow:global_step/sec: 9.54863
INFO:tensorflow:loss = 5.1366715, step = 28900 (10.472 sec)
INFO:tensorflow:lr = 0.0006178946 (10.473 sec)
INFO:tensorflow:global_step/sec: 9.43582
INFO:tensorflow:loss = 5.264217, step = 29000 (10.597 sec)
INFO:tensorflow:lr = 0.0006196867 (10.597 sec)
INFO:tensorflow:global_step/sec: 9.48442
INFO:tensorflow:loss = 5.1028385, step = 29100 (10.544 sec)
INFO:tensorflow:lr = 0.00062147866 (10.545 sec)
INFO:tensorflow:global_step/sec: 9.38207
INFO:tensorflow:loss = 5.248551, step = 29200 (10.658 sec)
INFO:tensorflow:lr = 0.0006232707 (10.658 sec)
INFO:tensorflow:global_step/sec: 9.62767
INFO:tensorflow:loss = 5.110166, step = 29300 (10.387 sec)
INFO:tensorflow:lr = 0.0006250627 (10.386 sec)
INFO:tensorflow:global_step/sec: 9.6329
INFO:tensorflow:loss = 5.151551, step = 29400 (10.382 sec)
INFO:tensorflow:lr = 0.0006268547 (10.382 sec)
INFO:tensorflow:global_step/sec: 9.51864
INFO:tensorflow:loss = 4.9802246, step = 29500 (10.505 sec)
INFO:tensorflow:lr = 0.00062864675 (10.505 sec)
INFO:tensorflow:global_step/sec: 9.01068
INFO:tensorflow:loss = 5.120793, step = 29600 (11.098 sec)
INFO:tensorflow:lr = 0.0006304388 (11.098 sec)
INFO:tensorflow:global_step/sec: 9.1528
INFO:tensorflow:loss = 5.0916967, step = 29700 (10.925 sec)
INFO:tensorflow:lr = 0.0006322308 (10.925 sec)
INFO:tensorflow:global_step/sec: 9.1005
INFO:tensorflow:loss = 4.9674788, step = 29800 (10.989 sec)
INFO:tensorflow:lr = 0.0006340228 (10.989 sec)
INFO:tensorflow:global_step/sec: 9.53958
INFO:tensorflow:loss = 5.112809, step = 29900 (10.482 sec)
INFO:tensorflow:lr = 0.00063581485 (10.483 sec)
INFO:tensorflow:global_step/sec: 9.47696
INFO:tensorflow:loss = 5.2655168, step = 30000 (10.552 sec)
INFO:tensorflow:lr = 0.0006376069 (10.550 sec)
INFO:tensorflow:global_step/sec: 9.58024
INFO:tensorflow:loss = 5.2323723, step = 30100 (10.440 sec)
INFO:tensorflow:lr = 0.0006393989 (10.441 sec)
INFO:tensorflow:global_step/sec: 9.1579
INFO:tensorflow:loss = 5.038887, step = 30200 (10.916 sec)
INFO:tensorflow:lr = 0.0006411909 (10.919 sec)
INFO:tensorflow:global_step/sec: 9.4468
INFO:tensorflow:loss = 4.993733, step = 30300 (10.589 sec)
INFO:tensorflow:lr = 0.00064298295 (10.586 sec)
INFO:tensorflow:global_step/sec: 9.4222
INFO:tensorflow:loss = 5.165243, step = 30400 (10.609 sec)
INFO:tensorflow:lr = 0.000644775 (10.611 sec)
INFO:tensorflow:global_step/sec: 9.10203
INFO:tensorflow:loss = 5.080005, step = 30500 (10.987 sec)
INFO:tensorflow:lr = 0.000646567 (10.987 sec)
INFO:tensorflow:global_step/sec: 9.37739
INFO:tensorflow:loss = 5.001014, step = 30600 (10.665 sec)
INFO:tensorflow:lr = 0.000648359 (10.666 sec)
INFO:tensorflow:global_step/sec: 9.53468
INFO:tensorflow:loss = 5.0422983, step = 30700 (10.488 sec)
INFO:tensorflow:lr = 0.000650151 (10.488 sec)
INFO:tensorflow:global_step/sec: 9.26078
INFO:tensorflow:loss = 5.10322, step = 30800 (10.796 sec)
INFO:tensorflow:lr = 0.0006519431 (10.797 sec)
INFO:tensorflow:global_step/sec: 9.43497
INFO:tensorflow:loss = 5.1736207, step = 30900 (10.602 sec)
INFO:tensorflow:lr = 0.0006537351 (10.601 sec)
INFO:tensorflow:global_step/sec: 8.9818
INFO:tensorflow:loss = 5.086377, step = 31000 (11.132 sec)
INFO:tensorflow:lr = 0.0006555271 (11.133 sec)
INFO:tensorflow:global_step/sec: 9.2658
INFO:tensorflow:loss = 5.021378, step = 31100 (10.793 sec)
INFO:tensorflow:lr = 0.0006573191 (10.792 sec)
INFO:tensorflow:global_step/sec: 9.16881
INFO:tensorflow:loss = 4.9313107, step = 31200 (10.907 sec)
INFO:tensorflow:lr = 0.0006591112 (10.907 sec)
INFO:tensorflow:global_step/sec: 9.09478
INFO:tensorflow:loss = 4.967325, step = 31300 (10.996 sec)
INFO:tensorflow:lr = 0.00066090317 (10.996 sec)
INFO:tensorflow:global_step/sec: 9.47929
INFO:tensorflow:loss = 5.143935, step = 31400 (10.549 sec)
INFO:tensorflow:lr = 0.0006626952 (10.550 sec)
INFO:tensorflow:global_step/sec: 9.00435
INFO:tensorflow:loss = 5.146677, step = 31500 (11.104 sec)
INFO:tensorflow:lr = 0.0006644872 (11.104 sec)
INFO:tensorflow:global_step/sec: 9.15019
INFO:tensorflow:loss = 5.2024193, step = 31600 (10.926 sec)
INFO:tensorflow:lr = 0.00066627923 (10.927 sec)
INFO:tensorflow:global_step/sec: 9.58678
INFO:tensorflow:loss = 4.9923096, step = 31700 (10.431 sec)
INFO:tensorflow:lr = 0.00066807127 (10.431 sec)
INFO:tensorflow:global_step/sec: 9.40868
INFO:tensorflow:loss = 5.0396233, step = 31800 (10.630 sec)
INFO:tensorflow:lr = 0.0006698633 (10.631 sec)
INFO:tensorflow:global_step/sec: 9.33834
INFO:tensorflow:loss = 5.1495657, step = 31900 (10.710 sec)
INFO:tensorflow:lr = 0.0006716553 (10.709 sec)
INFO:tensorflow:global_step/sec: 9.27637
INFO:tensorflow:loss = 5.214168, step = 32000 (10.779 sec)
INFO:tensorflow:lr = 0.0006734473 (10.780 sec)
INFO:tensorflow:global_step/sec: 9.10232
INFO:tensorflow:loss = 5.0548754, step = 32100 (10.988 sec)
INFO:tensorflow:lr = 0.00067523937 (10.987 sec)
INFO:tensorflow:global_step/sec: 8.94189
INFO:tensorflow:loss = 5.0729666, step = 32200 (11.183 sec)
INFO:tensorflow:lr = 0.0006770314 (11.182 sec)
INFO:tensorflow:global_step/sec: 9.31504
INFO:tensorflow:loss = 5.0461087, step = 32300 (10.736 sec)
INFO:tensorflow:lr = 0.0006788234 (10.735 sec)
INFO:tensorflow:global_step/sec: 9.21138
INFO:tensorflow:loss = 5.0706806, step = 32400 (10.855 sec)
INFO:tensorflow:lr = 0.0006806154 (10.856 sec)
INFO:tensorflow:global_step/sec: 9.39348
INFO:tensorflow:loss = 5.036418, step = 32500 (10.647 sec)
INFO:tensorflow:lr = 0.0006824074 (10.646 sec)
INFO:tensorflow:global_step/sec: 9.36836
INFO:tensorflow:loss = 5.0860696, step = 32600 (10.671 sec)
INFO:tensorflow:lr = 0.0006841995 (10.671 sec)
INFO:tensorflow:global_step/sec: 9.22372
INFO:tensorflow:loss = 5.1222463, step = 32700 (10.841 sec)
INFO:tensorflow:lr = 0.0006859915 (10.843 sec)
INFO:tensorflow:global_step/sec: 9.54108
INFO:tensorflow:loss = 5.114647, step = 32800 (10.481 sec)
INFO:tensorflow:lr = 0.0006877835 (10.481 sec)
INFO:tensorflow:global_step/sec: 9.67795
INFO:tensorflow:loss = 5.099014, step = 32900 (10.333 sec)
INFO:tensorflow:lr = 0.0006895755 (10.332 sec)
INFO:tensorflow:global_step/sec: 9.31083
INFO:tensorflow:loss = 5.1770754, step = 33000 (10.741 sec)
INFO:tensorflow:lr = 0.00069136755 (10.742 sec)
INFO:tensorflow:global_step/sec: 9.5378
INFO:tensorflow:loss = 5.069852, step = 33100 (10.485 sec)
INFO:tensorflow:lr = 0.0006931596 (10.484 sec)
INFO:tensorflow:global_step/sec: 9.67884
INFO:tensorflow:loss = 5.210137, step = 33200 (10.333 sec)
INFO:tensorflow:lr = 0.0006949516 (10.332 sec)
INFO:tensorflow:global_step/sec: 9.53279
INFO:tensorflow:loss = 4.9941797, step = 33300 (10.489 sec)
INFO:tensorflow:lr = 0.0006967436 (10.490 sec)
INFO:tensorflow:global_step/sec: 9.60949
INFO:tensorflow:loss = 5.0948496, step = 33400 (10.406 sec)
INFO:tensorflow:lr = 0.00069853564 (10.406 sec)
INFO:tensorflow:global_step/sec: 9.79415
INFO:tensorflow:loss = 5.22286, step = 33500 (10.210 sec)
INFO:tensorflow:lr = 0.0007003277 (10.210 sec)
INFO:tensorflow:global_step/sec: 9.53751
INFO:tensorflow:loss = 4.872001, step = 33600 (10.484 sec)
INFO:tensorflow:lr = 0.0007021197 (10.484 sec)
INFO:tensorflow:global_step/sec: 9.55892
INFO:tensorflow:loss = 5.152949, step = 33700 (10.465 sec)
INFO:tensorflow:lr = 0.0007039117 (10.464 sec)
INFO:tensorflow:global_step/sec: 9.48322
INFO:tensorflow:loss = 5.1330466, step = 33800 (10.543 sec)
INFO:tensorflow:lr = 0.00070570374 (10.544 sec)
INFO:tensorflow:global_step/sec: 9.46137
INFO:tensorflow:loss = 5.1620946, step = 33900 (10.567 sec)
INFO:tensorflow:lr = 0.0007074958 (10.568 sec)
INFO:tensorflow:global_step/sec: 9.46544
INFO:tensorflow:loss = 5.143143, step = 34000 (10.567 sec)
INFO:tensorflow:lr = 0.0007092878 (10.567 sec)
INFO:tensorflow:global_step/sec: 9.0955
INFO:tensorflow:loss = 5.0488315, step = 34100 (10.994 sec)
INFO:tensorflow:lr = 0.0007110798 (10.994 sec)
INFO:tensorflow:global_step/sec: 9.53646
INFO:tensorflow:loss = 4.9134398, step = 34200 (10.486 sec)
INFO:tensorflow:lr = 0.00071287184 (10.486 sec)
INFO:tensorflow:global_step/sec: 9.59935
INFO:tensorflow:loss = 5.149152, step = 34300 (10.418 sec)
INFO:tensorflow:lr = 0.0007146638 (10.418 sec)
INFO:tensorflow:global_step/sec: 9.02973
INFO:tensorflow:loss = 5.0462427, step = 34400 (11.072 sec)
INFO:tensorflow:lr = 0.00071645586 (11.076 sec)
INFO:tensorflow:global_step/sec: 9.32217
INFO:tensorflow:loss = 5.03478, step = 34500 (10.730 sec)
INFO:tensorflow:lr = 0.0007182479 (10.726 sec)
INFO:tensorflow:global_step/sec: 9.35269
INFO:tensorflow:loss = 4.9798565, step = 34600 (10.692 sec)
INFO:tensorflow:lr = 0.00072003994 (10.691 sec)
INFO:tensorflow:global_step/sec: 9.51789
INFO:tensorflow:loss = 5.096134, step = 34700 (10.505 sec)
INFO:tensorflow:lr = 0.0007218319 (10.506 sec)
INFO:tensorflow:global_step/sec: 9.52418
INFO:tensorflow:loss = 4.980329, step = 34800 (10.501 sec)
INFO:tensorflow:lr = 0.00072362396 (10.500 sec)
INFO:tensorflow:global_step/sec: 9.24779
INFO:tensorflow:loss = 5.10829, step = 34900 (10.811 sec)
INFO:tensorflow:lr = 0.000725416 (10.811 sec)
INFO:tensorflow:global_step/sec: 9.66491
INFO:tensorflow:loss = 5.2325635, step = 35000 (10.348 sec)
INFO:tensorflow:lr = 0.00072720804 (10.348 sec)
INFO:tensorflow:global_step/sec: 9.48265
INFO:tensorflow:loss = 5.120744, step = 35100 (10.546 sec)
INFO:tensorflow:lr = 0.000729 (10.546 sec)
INFO:tensorflow:global_step/sec: 9.3831
INFO:tensorflow:loss = 5.130047, step = 35200 (10.655 sec)
INFO:tensorflow:lr = 0.00073079206 (10.655 sec)
INFO:tensorflow:global_step/sec: 9.0511
INFO:tensorflow:loss = 5.078965, step = 35300 (11.051 sec)
INFO:tensorflow:lr = 0.0007325841 (11.051 sec)
INFO:tensorflow:global_step/sec: 9.425
INFO:tensorflow:loss = 5.008522, step = 35400 (10.608 sec)
INFO:tensorflow:lr = 0.0007343761 (10.609 sec)
INFO:tensorflow:global_step/sec: 9.24663
INFO:tensorflow:loss = 5.1093845, step = 35500 (10.820 sec)
INFO:tensorflow:lr = 0.0007361681 (10.820 sec)
INFO:tensorflow:global_step/sec: 9.31256
INFO:tensorflow:loss = 4.984288, step = 35600 (10.733 sec)
INFO:tensorflow:lr = 0.00073796016 (10.732 sec)
INFO:tensorflow:global_step/sec: 9.46158
INFO:tensorflow:loss = 5.137832, step = 35700 (10.572 sec)
INFO:tensorflow:lr = 0.0007397522 (10.572 sec)
INFO:tensorflow:global_step/sec: 9.40468
INFO:tensorflow:loss = 5.016031, step = 35800 (10.632 sec)
INFO:tensorflow:lr = 0.0007415442 (10.633 sec)
INFO:tensorflow:global_step/sec: 9.00724
INFO:tensorflow:loss = 5.049596, step = 35900 (11.102 sec)
INFO:tensorflow:lr = 0.0007433362 (11.102 sec)
INFO:tensorflow:global_step/sec: 9.46032
INFO:tensorflow:loss = 5.1828265, step = 36000 (10.570 sec)
INFO:tensorflow:lr = 0.00074512826 (10.569 sec)
INFO:tensorflow:global_step/sec: 9.3646
INFO:tensorflow:loss = 5.058272, step = 36100 (10.681 sec)
INFO:tensorflow:lr = 0.00074692024 (10.682 sec)
INFO:tensorflow:global_step/sec: 9.49169
INFO:tensorflow:loss = 5.245429, step = 36200 (10.533 sec)
INFO:tensorflow:lr = 0.0007487123 (10.533 sec)
INFO:tensorflow:global_step/sec: 9.51424
INFO:tensorflow:loss = 5.0098643, step = 36300 (10.513 sec)
INFO:tensorflow:lr = 0.0007505043 (10.511 sec)
INFO:tensorflow:global_step/sec: 9.2906
INFO:tensorflow:loss = 5.0751667, step = 36400 (10.763 sec)
INFO:tensorflow:lr = 0.00075229636 (10.765 sec)
INFO:tensorflow:global_step/sec: 9.38383
INFO:tensorflow:loss = 5.029735, step = 36500 (10.657 sec)
INFO:tensorflow:lr = 0.00075408834 (10.656 sec)
INFO:tensorflow:global_step/sec: 9.37703
INFO:tensorflow:loss = 4.932591, step = 36600 (10.663 sec)
INFO:tensorflow:lr = 0.0007558804 (10.663 sec)
INFO:tensorflow:global_step/sec: 9.4795
INFO:tensorflow:loss = 5.153864, step = 36700 (10.550 sec)
INFO:tensorflow:lr = 0.0007576724 (10.550 sec)
INFO:tensorflow:global_step/sec: 9.38389
INFO:tensorflow:loss = 5.1201468, step = 36800 (10.658 sec)
INFO:tensorflow:lr = 0.0007594644 (10.658 sec)
INFO:tensorflow:global_step/sec: 9.46879
INFO:tensorflow:loss = 5.0017695, step = 36900 (10.559 sec)
INFO:tensorflow:lr = 0.00076125644 (10.560 sec)
INFO:tensorflow:global_step/sec: 9.3874
INFO:tensorflow:loss = 4.984927, step = 37000 (10.653 sec)
INFO:tensorflow:lr = 0.0007630485 (10.652 sec)
INFO:tensorflow:global_step/sec: 9.26789
INFO:tensorflow:loss = 4.8914003, step = 37100 (10.790 sec)
INFO:tensorflow:lr = 0.0007648405 (10.790 sec)
INFO:tensorflow:global_step/sec: 9.33466
INFO:tensorflow:loss = 4.935514, step = 37200 (10.713 sec)
INFO:tensorflow:lr = 0.0007666325 (10.713 sec)
INFO:tensorflow:global_step/sec: 8.86929
INFO:tensorflow:loss = 5.0787344, step = 37300 (11.275 sec)
INFO:tensorflow:lr = 0.00076842454 (11.279 sec)
INFO:tensorflow:global_step/sec: 9.17307
INFO:tensorflow:loss = 4.892041, step = 37400 (10.900 sec)
INFO:tensorflow:lr = 0.0007702166 (10.897 sec)
INFO:tensorflow:global_step/sec: 9.57276
INFO:tensorflow:loss = 5.1717734, step = 37500 (10.447 sec)
INFO:tensorflow:lr = 0.0007720086 (10.448 sec)
INFO:tensorflow:global_step/sec: 9.39006
INFO:tensorflow:loss = 5.0274525, step = 37600 (10.648 sec)
INFO:tensorflow:lr = 0.0007738006 (10.647 sec)
INFO:tensorflow:global_step/sec: 9.01223
INFO:tensorflow:loss = 5.040253, step = 37700 (11.099 sec)
INFO:tensorflow:lr = 0.00077559263 (11.099 sec)
INFO:tensorflow:global_step/sec: 9.32665
INFO:tensorflow:loss = 5.2148137, step = 37800 (10.723 sec)
INFO:tensorflow:lr = 0.0007773846 (10.722 sec)
INFO:tensorflow:global_step/sec: 9.31095
INFO:tensorflow:loss = 4.983381, step = 37900 (10.738 sec)
INFO:tensorflow:lr = 0.00077917665 (10.740 sec)
INFO:tensorflow:global_step/sec: 9.27172
INFO:tensorflow:loss = 5.139436, step = 38000 (10.788 sec)
INFO:tensorflow:lr = 0.0007809687 (10.787 sec)
INFO:tensorflow:global_step/sec: 9.22479
INFO:tensorflow:loss = 4.9605513, step = 38100 (10.838 sec)
INFO:tensorflow:lr = 0.00078276073 (10.837 sec)
INFO:tensorflow:global_step/sec: 9.55009
INFO:tensorflow:loss = 5.0494156, step = 38200 (10.473 sec)
INFO:tensorflow:lr = 0.0007845527 (10.472 sec)
INFO:tensorflow:global_step/sec: 9.16568
INFO:tensorflow:loss = 5.0706954, step = 38300 (10.909 sec)
INFO:tensorflow:lr = 0.00078634475 (10.909 sec)
INFO:tensorflow:global_step/sec: 9.26769
INFO:tensorflow:loss = 4.9262986, step = 38400 (10.788 sec)
INFO:tensorflow:lr = 0.0007881368 (10.790 sec)
INFO:tensorflow:global_step/sec: 9.38006
INFO:tensorflow:loss = 4.9493513, step = 38500 (10.665 sec)
INFO:tensorflow:lr = 0.00078992883 (10.663 sec)
INFO:tensorflow:global_step/sec: 9.63205
INFO:tensorflow:loss = 4.980843, step = 38600 (10.381 sec)
INFO:tensorflow:lr = 0.0007917208 (10.381 sec)
INFO:tensorflow:global_step/sec: 9.39848
INFO:tensorflow:loss = 4.944447, step = 38700 (10.641 sec)
INFO:tensorflow:lr = 0.00079351285 (10.647 sec)
INFO:tensorflow:global_step/sec: 9.16364
INFO:tensorflow:loss = 5.029889, step = 38800 (10.913 sec)
INFO:tensorflow:lr = 0.00079530483 (10.907 sec)
INFO:tensorflow:global_step/sec: 9.26539
INFO:tensorflow:loss = 4.991285, step = 38900 (10.790 sec)
INFO:tensorflow:lr = 0.00079709693 (10.791 sec)
INFO:tensorflow:global_step/sec: 9.25618
INFO:tensorflow:loss = 4.978191, step = 39000 (10.806 sec)
INFO:tensorflow:lr = 0.0007988889 (10.804 sec)
INFO:tensorflow:global_step/sec: 9.49369
INFO:tensorflow:loss = 4.971673, step = 39100 (10.533 sec)
INFO:tensorflow:lr = 0.00079931895 (10.534 sec)
INFO:tensorflow:global_step/sec: 9.44335
INFO:tensorflow:loss = 4.9710107, step = 39200 (10.589 sec)
INFO:tensorflow:lr = 0.00079752697 (10.590 sec)
INFO:tensorflow:global_step/sec: 9.38353
INFO:tensorflow:loss = 5.0888605, step = 39300 (10.655 sec)
INFO:tensorflow:lr = 0.00079573493 (10.657 sec)
INFO:tensorflow:global_step/sec: 9.34901
INFO:tensorflow:loss = 5.1406965, step = 39400 (10.699 sec)
INFO:tensorflow:lr = 0.00079394295 (10.697 sec)
INFO:tensorflow:global_step/sec: 9.40024
INFO:tensorflow:loss = 5.0318465, step = 39500 (10.637 sec)
INFO:tensorflow:lr = 0.0007921509 (10.638 sec)
INFO:tensorflow:global_step/sec: 9.57672
INFO:tensorflow:loss = 4.9905806, step = 39600 (10.440 sec)
INFO:tensorflow:lr = 0.0007903589 (10.442 sec)
INFO:tensorflow:global_step/sec: 9.23393
INFO:tensorflow:loss = 5.020684, step = 39700 (10.832 sec)
INFO:tensorflow:lr = 0.0007885669 (10.829 sec)
INFO:tensorflow:global_step/sec: 9.67045
INFO:tensorflow:loss = 4.956047, step = 39800 (10.340 sec)
INFO:tensorflow:lr = 0.00078677485 (10.341 sec)
INFO:tensorflow:global_step/sec: 9.50074
INFO:tensorflow:loss = 5.0920815, step = 39900 (10.526 sec)
INFO:tensorflow:lr = 0.00078498287 (10.526 sec)
INFO:tensorflow:global_step/sec: 9.41762
INFO:tensorflow:loss = 4.9771967, step = 40000 (10.617 sec)
INFO:tensorflow:lr = 0.0007831908 (10.617 sec)
INFO:tensorflow:global_step/sec: 9.4584
INFO:tensorflow:loss = 5.0041213, step = 40100 (10.572 sec)
INFO:tensorflow:lr = 0.00078139873 (10.574 sec)
INFO:tensorflow:global_step/sec: 9.30685
INFO:tensorflow:loss = 5.129474, step = 40200 (10.747 sec)
INFO:tensorflow:lr = 0.00077960675 (10.745 sec)
INFO:tensorflow:global_step/sec: 9.10269
INFO:tensorflow:loss = 5.1569552, step = 40300 (10.985 sec)
INFO:tensorflow:lr = 0.0007778147 (10.986 sec)
INFO:tensorflow:global_step/sec: 9.20877
INFO:tensorflow:loss = 5.0918455, step = 40400 (10.861 sec)
INFO:tensorflow:lr = 0.00077602273 (10.860 sec)
INFO:tensorflow:global_step/sec: 9.00773
INFO:tensorflow:loss = 4.9962397, step = 40500 (11.098 sec)
INFO:tensorflow:lr = 0.0007742307 (11.100 sec)
INFO:tensorflow:global_step/sec: 9.2361
INFO:tensorflow:loss = 5.024959, step = 40600 (10.829 sec)
INFO:tensorflow:lr = 0.0007724387 (10.828 sec)
INFO:tensorflow:global_step/sec: 9.43495
INFO:tensorflow:loss = 5.0192513, step = 40700 (10.599 sec)
INFO:tensorflow:lr = 0.0007706467 (10.598 sec)
INFO:tensorflow:global_step/sec: 9.45791
INFO:tensorflow:loss = 5.0989456, step = 40800 (10.574 sec)
INFO:tensorflow:lr = 0.00076885463 (10.574 sec)
INFO:tensorflow:global_step/sec: 9.43583
INFO:tensorflow:loss = 5.1665564, step = 40900 (10.598 sec)
INFO:tensorflow:lr = 0.00076706253 (10.598 sec)
INFO:tensorflow:global_step/sec: 9.29103
INFO:tensorflow:loss = 5.0380497, step = 41000 (10.760 sec)
INFO:tensorflow:lr = 0.00076527055 (10.761 sec)
INFO:tensorflow:global_step/sec: 9.33781
INFO:tensorflow:loss = 4.9364643, step = 41100 (10.711 sec)
INFO:tensorflow:lr = 0.0007634785 (10.710 sec)
INFO:tensorflow:global_step/sec: 9.44074
INFO:tensorflow:loss = 5.0375924, step = 41200 (10.590 sec)
INFO:tensorflow:lr = 0.00076168653 (10.591 sec)
INFO:tensorflow:global_step/sec: 9.11934
INFO:tensorflow:loss = 5.2311244, step = 41300 (10.966 sec)
INFO:tensorflow:lr = 0.0007598945 (10.968 sec)
INFO:tensorflow:global_step/sec: 9.49467
INFO:tensorflow:loss = 5.061126, step = 41400 (10.534 sec)
INFO:tensorflow:lr = 0.0007581025 (10.531 sec)
INFO:tensorflow:global_step/sec: 9.18987
INFO:tensorflow:loss = 5.0643272, step = 41500 (10.882 sec)
INFO:tensorflow:lr = 0.0007563105 (10.882 sec)
INFO:tensorflow:global_step/sec: 9.25841
INFO:tensorflow:loss = 5.047558, step = 41600 (10.801 sec)
INFO:tensorflow:lr = 0.0007545185 (10.802 sec)
INFO:tensorflow:global_step/sec: 9.57013
INFO:tensorflow:loss = 4.957797, step = 41700 (10.448 sec)
INFO:tensorflow:lr = 0.00075272645 (10.447 sec)
INFO:tensorflow:global_step/sec: 9.60469
INFO:tensorflow:loss = 4.9590044, step = 41800 (10.411 sec)
INFO:tensorflow:lr = 0.00075093436 (10.411 sec)
INFO:tensorflow:global_step/sec: 9.32542
INFO:tensorflow:loss = 5.100118, step = 41900 (10.727 sec)
INFO:tensorflow:lr = 0.0007491423 (10.726 sec)
INFO:tensorflow:global_step/sec: 9.33525
INFO:tensorflow:loss = 5.0650043, step = 42000 (10.713 sec)
INFO:tensorflow:lr = 0.00074735034 (10.713 sec)
INFO:tensorflow:global_step/sec: 9.32104
INFO:tensorflow:loss = 4.912504, step = 42100 (10.726 sec)
INFO:tensorflow:lr = 0.0007455583 (10.726 sec)
INFO:tensorflow:global_step/sec: 9.55293
INFO:tensorflow:loss = 5.0278497, step = 42200 (10.470 sec)
INFO:tensorflow:lr = 0.0007437663 (10.469 sec)
INFO:tensorflow:global_step/sec: 9.28278
INFO:tensorflow:loss = 5.1773267, step = 42300 (10.772 sec)
INFO:tensorflow:lr = 0.0007419743 (10.773 sec)
INFO:tensorflow:global_step/sec: 9.45226
INFO:tensorflow:loss = 5.0096893, step = 42400 (10.579 sec)
INFO:tensorflow:lr = 0.0007401823 (10.579 sec)
INFO:tensorflow:global_step/sec: 9.50569
INFO:tensorflow:loss = 4.9839783, step = 42500 (10.518 sec)
INFO:tensorflow:lr = 0.00073839026 (10.517 sec)
INFO:tensorflow:global_step/sec: 9.37434
INFO:tensorflow:loss = 4.950326, step = 42600 (10.667 sec)
INFO:tensorflow:lr = 0.0007365983 (10.669 sec)
INFO:tensorflow:global_step/sec: 9.42829
INFO:tensorflow:loss = 5.3135276, step = 42700 (10.609 sec)
INFO:tensorflow:lr = 0.0007348062 (10.611 sec)
INFO:tensorflow:global_step/sec: 9.51614
INFO:tensorflow:loss = 5.0509195, step = 42800 (10.510 sec)
INFO:tensorflow:lr = 0.00073301414 (10.506 sec)
INFO:tensorflow:global_step/sec: 9.70035
INFO:tensorflow:loss = 4.912774, step = 42900 (10.307 sec)
INFO:tensorflow:lr = 0.0007312221 (10.308 sec)
INFO:tensorflow:global_step/sec: 9.53879
INFO:tensorflow:loss = 4.9984446, step = 43000 (10.485 sec)
INFO:tensorflow:lr = 0.0007294301 (10.484 sec)
INFO:tensorflow:global_step/sec: 9.41552
INFO:tensorflow:loss = 4.9727006, step = 43100 (10.621 sec)
INFO:tensorflow:lr = 0.0007276381 (10.620 sec)
INFO:tensorflow:global_step/sec: 9.34481
INFO:tensorflow:loss = 5.032983, step = 43200 (10.701 sec)
INFO:tensorflow:lr = 0.0007258461 (10.701 sec)
INFO:tensorflow:global_step/sec: 9.57305
INFO:tensorflow:loss = 5.0043697, step = 43300 (10.446 sec)
INFO:tensorflow:lr = 0.00072405406 (10.447 sec)
INFO:tensorflow:global_step/sec: 9.43392
INFO:tensorflow:loss = 4.9424596, step = 43400 (10.599 sec)
INFO:tensorflow:lr = 0.0007222621 (10.599 sec)
INFO:tensorflow:global_step/sec: 9.50868
INFO:tensorflow:loss = 5.039576, step = 43500 (10.514 sec)
INFO:tensorflow:lr = 0.00072047004 (10.517 sec)
INFO:tensorflow:global_step/sec: 9.34554
INFO:tensorflow:loss = 5.0512376, step = 43600 (10.702 sec)
INFO:tensorflow:lr = 0.00071867794 (10.700 sec)
INFO:tensorflow:global_step/sec: 9.3316
INFO:tensorflow:loss = 4.8776455, step = 43700 (10.718 sec)
INFO:tensorflow:lr = 0.00071688596 (10.718 sec)
INFO:tensorflow:global_step/sec: 9.42098
INFO:tensorflow:loss = 4.9753275, step = 43800 (10.613 sec)
INFO:tensorflow:lr = 0.0007150939 (10.613 sec)
INFO:tensorflow:global_step/sec: 9.34695
INFO:tensorflow:loss = 5.1619225, step = 43900 (10.696 sec)
INFO:tensorflow:lr = 0.0007133019 (10.696 sec)
INFO:tensorflow:global_step/sec: 9.66996
INFO:tensorflow:loss = 4.897856, step = 44000 (10.344 sec)
INFO:tensorflow:lr = 0.0007115099 (10.344 sec)
INFO:tensorflow:global_step/sec: 9.31438
INFO:tensorflow:loss = 4.9865513, step = 44100 (10.735 sec)
INFO:tensorflow:lr = 0.00070971786 (10.734 sec)
INFO:tensorflow:global_step/sec: 9.32455
INFO:tensorflow:loss = 5.0448728, step = 44200 (10.726 sec)
INFO:tensorflow:lr = 0.0007079259 (10.729 sec)
INFO:tensorflow:global_step/sec: 9.48733
INFO:tensorflow:loss = 5.148017, step = 44300 (10.540 sec)
INFO:tensorflow:lr = 0.00070613384 (10.538 sec)
INFO:tensorflow:global_step/sec: 9.52093
INFO:tensorflow:loss = 4.945143, step = 44400 (10.501 sec)
INFO:tensorflow:lr = 0.00070434186 (10.501 sec)
INFO:tensorflow:global_step/sec: 9.47543
INFO:tensorflow:loss = 5.114677, step = 44500 (10.554 sec)
INFO:tensorflow:lr = 0.00070254976 (10.555 sec)
INFO:tensorflow:global_step/sec: 9.50398
INFO:tensorflow:loss = 4.9485087, step = 44600 (10.523 sec)
INFO:tensorflow:lr = 0.0007007577 (10.523 sec)
INFO:tensorflow:global_step/sec: 9.55528
INFO:tensorflow:loss = 5.1555524, step = 44700 (10.467 sec)
INFO:tensorflow:lr = 0.0006989657 (10.466 sec)
INFO:tensorflow:global_step/sec: 9.40312
INFO:tensorflow:loss = 5.0596046, step = 44800 (10.633 sec)
INFO:tensorflow:lr = 0.0006971737 (10.634 sec)
INFO:tensorflow:global_step/sec: 9.53255
INFO:tensorflow:loss = 4.982494, step = 44900 (10.491 sec)
INFO:tensorflow:lr = 0.00069538166 (10.491 sec)
INFO:tensorflow:global_step/sec: 9.27382
INFO:tensorflow:loss = 5.0182176, step = 45000 (10.781 sec)
INFO:tensorflow:lr = 0.0006935897 (10.781 sec)
INFO:tensorflow:global_step/sec: 9.39855
INFO:tensorflow:loss = 5.0612426, step = 45100 (10.644 sec)
INFO:tensorflow:lr = 0.00069179764 (10.644 sec)
INFO:tensorflow:global_step/sec: 9.62145
INFO:tensorflow:loss = 4.989537, step = 45200 (10.391 sec)
INFO:tensorflow:lr = 0.00069000566 (10.392 sec)
INFO:tensorflow:global_step/sec: 9.51908
INFO:tensorflow:loss = 5.0123234, step = 45300 (10.505 sec)
INFO:tensorflow:lr = 0.0006882136 (10.504 sec)
INFO:tensorflow:global_step/sec: 9.51554
INFO:tensorflow:loss = 4.9916244, step = 45400 (10.513 sec)
INFO:tensorflow:lr = 0.0006864215 (10.513 sec)
INFO:tensorflow:global_step/sec: 9.50763
INFO:tensorflow:loss = 5.095473, step = 45500 (10.514 sec)
INFO:tensorflow:lr = 0.00068462954 (10.515 sec)
INFO:tensorflow:global_step/sec: 9.37476
INFO:tensorflow:loss = 5.039061, step = 45600 (10.665 sec)
INFO:tensorflow:lr = 0.0006828375 (10.665 sec)
INFO:tensorflow:global_step/sec: 9.37142
INFO:tensorflow:loss = 4.960917, step = 45700 (10.671 sec)
INFO:tensorflow:lr = 0.00068104547 (10.670 sec)
INFO:tensorflow:global_step/sec: 9.5341
INFO:tensorflow:loss = 5.003217, step = 45800 (10.490 sec)
INFO:tensorflow:lr = 0.0006792535 (10.491 sec)
INFO:tensorflow:global_step/sec: 9.6098
INFO:tensorflow:loss = 5.049335, step = 45900 (10.406 sec)
INFO:tensorflow:lr = 0.00067746144 (10.406 sec)
INFO:tensorflow:global_step/sec: 9.08646
INFO:tensorflow:loss = 5.0135064, step = 46000 (11.003 sec)
INFO:tensorflow:lr = 0.00067566946 (11.005 sec)
INFO:tensorflow:global_step/sec: 8.75664
INFO:tensorflow:loss = 5.2109594, step = 46100 (11.422 sec)
INFO:tensorflow:lr = 0.0006738774 (11.421 sec)
INFO:tensorflow:global_step/sec: 9.44446
INFO:tensorflow:loss = 4.8870687, step = 46200 (10.587 sec)
INFO:tensorflow:lr = 0.00067208544 (10.588 sec)
INFO:tensorflow:global_step/sec: 9.28042
INFO:tensorflow:loss = 4.961076, step = 46300 (10.776 sec)
INFO:tensorflow:lr = 0.00067029335 (10.775 sec)
INFO:tensorflow:global_step/sec: 9.67404
INFO:tensorflow:loss = 4.972317, step = 46400 (10.337 sec)
INFO:tensorflow:lr = 0.0006685013 (10.336 sec)
INFO:tensorflow:global_step/sec: 9.38469
INFO:tensorflow:loss = 5.1011596, step = 46500 (10.656 sec)
INFO:tensorflow:lr = 0.0006667093 (10.657 sec)
INFO:tensorflow:global_step/sec: 9.30863
INFO:tensorflow:loss = 5.058347, step = 46600 (10.742 sec)
INFO:tensorflow:lr = 0.0006649173 (10.742 sec)
INFO:tensorflow:global_step/sec: 9.1271
INFO:tensorflow:loss = 4.9873576, step = 46700 (10.957 sec)
INFO:tensorflow:lr = 0.00066312525 (10.957 sec)
INFO:tensorflow:global_step/sec: 9.22479
INFO:tensorflow:loss = 4.923091, step = 46800 (10.839 sec)
INFO:tensorflow:lr = 0.00066133327 (10.840 sec)
INFO:tensorflow:global_step/sec: 9.53795
INFO:tensorflow:loss = 5.0580025, step = 46900 (10.486 sec)
INFO:tensorflow:lr = 0.0006595412 (10.486 sec)
INFO:tensorflow:global_step/sec: 9.2316
INFO:tensorflow:loss = 5.000795, step = 47000 (10.833 sec)
INFO:tensorflow:lr = 0.00065774925 (10.834 sec)
INFO:tensorflow:global_step/sec: 9.18839
INFO:tensorflow:loss = 4.871485, step = 47100 (10.882 sec)
INFO:tensorflow:lr = 0.0006559572 (10.882 sec)
INFO:tensorflow:global_step/sec: 9.37196
INFO:tensorflow:loss = 4.8876815, step = 47200 (10.670 sec)
INFO:tensorflow:lr = 0.0006541651 (10.669 sec)
INFO:tensorflow:global_step/sec: 9.21784
INFO:tensorflow:loss = 5.043597, step = 47300 (10.848 sec)
INFO:tensorflow:lr = 0.0006523731 (10.849 sec)
INFO:tensorflow:global_step/sec: 9.28215
INFO:tensorflow:loss = 5.1398096, step = 47400 (10.771 sec)
INFO:tensorflow:lr = 0.0006505811 (10.771 sec)
INFO:tensorflow:global_step/sec: 9.21442
INFO:tensorflow:loss = 4.897675, step = 47500 (10.855 sec)
INFO:tensorflow:lr = 0.0006487891 (10.856 sec)
INFO:tensorflow:global_step/sec: 9.35004
INFO:tensorflow:loss = 5.0621614, step = 47600 (10.694 sec)
INFO:tensorflow:lr = 0.00064699707 (10.694 sec)
INFO:tensorflow:global_step/sec: 9.58694
INFO:tensorflow:loss = 5.076569, step = 47700 (10.432 sec)
INFO:tensorflow:lr = 0.00064520503 (10.432 sec)
INFO:tensorflow:global_step/sec: 9.32931
INFO:tensorflow:loss = 5.0489826, step = 47800 (10.718 sec)
INFO:tensorflow:lr = 0.00064341305 (10.718 sec)
INFO:tensorflow:global_step/sec: 9.38456
INFO:tensorflow:loss = 4.988774, step = 47900 (10.654 sec)
INFO:tensorflow:lr = 0.000641621 (10.654 sec)
INFO:tensorflow:global_step/sec: 9.39305
INFO:tensorflow:loss = 5.1435757, step = 48000 (10.646 sec)
INFO:tensorflow:lr = 0.000639829 (10.646 sec)
INFO:tensorflow:global_step/sec: 9.53771
INFO:tensorflow:loss = 4.9170194, step = 48100 (10.486 sec)
INFO:tensorflow:lr = 0.00063803693 (10.488 sec)
INFO:tensorflow:global_step/sec: 9.35327
INFO:tensorflow:loss = 4.9841676, step = 48200 (10.692 sec)
INFO:tensorflow:lr = 0.0006362449 (10.690 sec)
INFO:tensorflow:global_step/sec: 9.45542
INFO:tensorflow:loss = 5.089015, step = 48300 (10.575 sec)
INFO:tensorflow:lr = 0.0006344529 (10.575 sec)
INFO:tensorflow:global_step/sec: 9.48675
INFO:tensorflow:loss = 5.1919518, step = 48400 (10.542 sec)
INFO:tensorflow:lr = 0.0006326609 (10.542 sec)
INFO:tensorflow:global_step/sec: 9.16907
INFO:tensorflow:loss = 4.9903545, step = 48500 (10.907 sec)
INFO:tensorflow:lr = 0.0006308689 (10.906 sec)
INFO:tensorflow:global_step/sec: 9.37422
INFO:tensorflow:loss = 5.088899, step = 48600 (10.667 sec)
INFO:tensorflow:lr = 0.00062907685 (10.667 sec)
INFO:tensorflow:global_step/sec: 9.48194
INFO:tensorflow:loss = 4.8907027, step = 48700 (10.547 sec)
INFO:tensorflow:lr = 0.0006272848 (10.548 sec)
INFO:tensorflow:global_step/sec: 9.2032
INFO:tensorflow:loss = 4.991426, step = 48800 (10.866 sec)
INFO:tensorflow:lr = 0.00062549283 (10.865 sec)
INFO:tensorflow:global_step/sec: 9.37849
INFO:tensorflow:loss = 4.985582, step = 48900 (10.660 sec)
INFO:tensorflow:lr = 0.0006237008 (10.660 sec)
INFO:tensorflow:global_step/sec: 8.8565
INFO:tensorflow:loss = 5.1335573, step = 49000 (11.293 sec)
INFO:tensorflow:lr = 0.0006219087 (11.293 sec)
INFO:tensorflow:global_step/sec: 9.43977
INFO:tensorflow:loss = 4.9458632, step = 49100 (10.592 sec)
INFO:tensorflow:lr = 0.0006201167 (10.592 sec)
INFO:tensorflow:global_step/sec: 9.51817
INFO:tensorflow:loss = 4.918289, step = 49200 (10.508 sec)
INFO:tensorflow:lr = 0.0006183247 (10.507 sec)
INFO:tensorflow:global_step/sec: 9.52134
INFO:tensorflow:loss = 5.1120663, step = 49300 (10.502 sec)
INFO:tensorflow:lr = 0.0006165327 (10.502 sec)
INFO:tensorflow:global_step/sec: 9.35327
INFO:tensorflow:loss = 5.1775713, step = 49400 (10.692 sec)
INFO:tensorflow:lr = 0.00061474065 (10.693 sec)
INFO:tensorflow:global_step/sec: 9.17388
INFO:tensorflow:loss = 4.9629602, step = 49500 (10.899 sec)
INFO:tensorflow:lr = 0.0006129486 (10.899 sec)
INFO:tensorflow:global_step/sec: 9.58739
INFO:tensorflow:loss = 5.0018997, step = 49600 (10.432 sec)
INFO:tensorflow:lr = 0.00061115663 (10.432 sec)
INFO:tensorflow:global_step/sec: 9.43341
INFO:tensorflow:loss = 5.023855, step = 49700 (10.602 sec)
INFO:tensorflow:lr = 0.0006093646 (10.603 sec)
INFO:tensorflow:global_step/sec: 9.51348
INFO:tensorflow:loss = 5.024961, step = 49800 (10.511 sec)
INFO:tensorflow:lr = 0.0006075726 (10.509 sec)
INFO:tensorflow:global_step/sec: 9.368
INFO:tensorflow:loss = 4.97074, step = 49900 (10.672 sec)
INFO:tensorflow:lr = 0.0006057805 (10.673 sec)
INFO:tensorflow:global_step/sec: 9.59222
INFO:tensorflow:loss = 5.1788244, step = 50000 (10.428 sec)
INFO:tensorflow:lr = 0.0006039885 (10.426 sec)
INFO:tensorflow:global_step/sec: 9.53275
INFO:tensorflow:loss = 4.973164, step = 50100 (10.496 sec)
INFO:tensorflow:lr = 0.0006021965 (10.497 sec)
INFO:tensorflow:global_step/sec: 9.53317
INFO:tensorflow:loss = 4.9693937, step = 50200 (10.481 sec)
INFO:tensorflow:lr = 0.00060040446 (10.480 sec)
INFO:tensorflow:global_step/sec: 9.65217
INFO:tensorflow:loss = 4.950563, step = 50300 (10.363 sec)
INFO:tensorflow:lr = 0.0005986125 (10.362 sec)
INFO:tensorflow:global_step/sec: 9.58435
INFO:tensorflow:loss = 4.9971285, step = 50400 (10.433 sec)
INFO:tensorflow:lr = 0.00059682044 (10.435 sec)
INFO:tensorflow:global_step/sec: 9.66644
INFO:tensorflow:loss = 4.944786, step = 50500 (10.344 sec)
INFO:tensorflow:lr = 0.0005950284 (10.344 sec)
INFO:tensorflow:global_step/sec: 9.55486
INFO:tensorflow:loss = 5.0195484, step = 50600 (10.466 sec)
INFO:tensorflow:lr = 0.0005932364 (10.467 sec)
INFO:tensorflow:global_step/sec: 9.52583
INFO:tensorflow:loss = 4.9206963, step = 50700 (10.497 sec)
INFO:tensorflow:lr = 0.0005914444 (10.495 sec)
INFO:tensorflow:global_step/sec: 9.60777
INFO:tensorflow:loss = 4.938217, step = 50800 (10.409 sec)
INFO:tensorflow:lr = 0.0005896523 (10.411 sec)
INFO:tensorflow:global_step/sec: 9.36609
INFO:tensorflow:loss = 4.8422065, step = 50900 (10.677 sec)
INFO:tensorflow:lr = 0.0005878603 (10.675 sec)
INFO:tensorflow:global_step/sec: 9.52744
INFO:tensorflow:loss = 5.123394, step = 51000 (10.498 sec)
INFO:tensorflow:lr = 0.00058606826 (10.497 sec)
INFO:tensorflow:global_step/sec: 9.28914
INFO:tensorflow:loss = 5.1042643, step = 51100 (10.766 sec)
INFO:tensorflow:lr = 0.0005842762 (10.766 sec)
INFO:tensorflow:global_step/sec: 9.37764
INFO:tensorflow:loss = 5.131436, step = 51200 (10.664 sec)
INFO:tensorflow:lr = 0.00058248424 (10.664 sec)
INFO:tensorflow:global_step/sec: 9.7
INFO:tensorflow:loss = 4.88273, step = 51300 (10.308 sec)
INFO:tensorflow:lr = 0.0005806922 (10.309 sec)
INFO:tensorflow:global_step/sec: 9.60252
INFO:tensorflow:loss = 5.05976, step = 51400 (10.414 sec)
INFO:tensorflow:lr = 0.0005789002 (10.414 sec)
INFO:tensorflow:global_step/sec: 9.59476
INFO:tensorflow:loss = 5.0464835, step = 51500 (10.423 sec)
INFO:tensorflow:lr = 0.0005771082 (10.423 sec)
INFO:tensorflow:global_step/sec: 9.50306
INFO:tensorflow:loss = 4.9827104, step = 51600 (10.522 sec)
INFO:tensorflow:lr = 0.0005753162 (10.522 sec)
INFO:tensorflow:global_step/sec: 9.38927
INFO:tensorflow:loss = 5.0948005, step = 51700 (10.652 sec)
INFO:tensorflow:lr = 0.0005735241 (10.652 sec)
INFO:tensorflow:global_step/sec: 9.51707
INFO:tensorflow:loss = 5.045607, step = 51800 (10.506 sec)
INFO:tensorflow:lr = 0.00057173206 (10.506 sec)
INFO:tensorflow:global_step/sec: 9.3261
INFO:tensorflow:loss = 4.8977003, step = 51900 (10.722 sec)
INFO:tensorflow:lr = 0.0005699401 (10.723 sec)
INFO:tensorflow:global_step/sec: 9.1791
INFO:tensorflow:loss = 4.9912663, step = 52000 (10.894 sec)
INFO:tensorflow:lr = 0.00056814804 (10.894 sec)
INFO:tensorflow:global_step/sec: 9.39772
INFO:tensorflow:loss = 5.0285425, step = 52100 (10.640 sec)
INFO:tensorflow:lr = 0.000566356 (10.639 sec)
INFO:tensorflow:global_step/sec: 9.62507
INFO:tensorflow:loss = 4.894455, step = 52200 (10.392 sec)
INFO:tensorflow:lr = 0.000564564 (10.391 sec)
INFO:tensorflow:global_step/sec: 9.70896
INFO:tensorflow:loss = 5.096932, step = 52300 (10.301 sec)
INFO:tensorflow:lr = 0.000562772 (10.300 sec)
INFO:tensorflow:global_step/sec: 9.66523
INFO:tensorflow:loss = 4.941601, step = 52400 (10.345 sec)
INFO:tensorflow:lr = 0.00056098 (10.347 sec)
INFO:tensorflow:global_step/sec: 9.52534
INFO:tensorflow:loss = 4.811992, step = 52500 (10.498 sec)
INFO:tensorflow:lr = 0.00055918796 (10.498 sec)
INFO:tensorflow:global_step/sec: 9.35068
INFO:tensorflow:loss = 4.9959965, step = 52600 (10.695 sec)
INFO:tensorflow:lr = 0.00055739586 (10.695 sec)
INFO:tensorflow:global_step/sec: 9.41247
INFO:tensorflow:loss = 4.9717216, step = 52700 (10.621 sec)
INFO:tensorflow:lr = 0.0005556039 (10.622 sec)
INFO:tensorflow:global_step/sec: 9.58399
INFO:tensorflow:loss = 5.0410056, step = 52800 (10.436 sec)
INFO:tensorflow:lr = 0.00055381184 (10.438 sec)
INFO:tensorflow:global_step/sec: 9.53823
INFO:tensorflow:loss = 5.0049415, step = 52900 (10.483 sec)
INFO:tensorflow:lr = 0.00055201986 (10.479 sec)
INFO:tensorflow:global_step/sec: 9.46178
INFO:tensorflow:loss = 5.153189, step = 53000 (10.572 sec)
INFO:tensorflow:lr = 0.0005502278 (10.572 sec)
INFO:tensorflow:global_step/sec: 9.38044
INFO:tensorflow:loss = 5.045697, step = 53100 (10.659 sec)
INFO:tensorflow:lr = 0.0005484358 (10.659 sec)
INFO:tensorflow:global_step/sec: 9.31813
INFO:tensorflow:loss = 4.9369597, step = 53200 (10.734 sec)
INFO:tensorflow:lr = 0.0005466438 (10.734 sec)
INFO:tensorflow:global_step/sec: 9.6857
INFO:tensorflow:loss = 5.1337905, step = 53300 (10.320 sec)
INFO:tensorflow:lr = 0.00054485176 (10.320 sec)
INFO:tensorflow:global_step/sec: 9.61851
INFO:tensorflow:loss = 5.0458665, step = 53400 (10.399 sec)
INFO:tensorflow:lr = 0.0005430598 (10.399 sec)
INFO:tensorflow:global_step/sec: 9.5056
INFO:tensorflow:loss = 5.0202885, step = 53500 (10.520 sec)
INFO:tensorflow:lr = 0.0005412677 (10.520 sec)
INFO:tensorflow:global_step/sec: 9.49133
INFO:tensorflow:loss = 5.0235763, step = 53600 (10.534 sec)
INFO:tensorflow:lr = 0.00053947564 (10.537 sec)
INFO:tensorflow:global_step/sec: 9.3808
INFO:tensorflow:loss = 4.963329, step = 53700 (10.662 sec)
INFO:tensorflow:lr = 0.00053768366 (10.660 sec)
INFO:tensorflow:global_step/sec: 9.46908
INFO:tensorflow:loss = 4.926022, step = 53800 (10.559 sec)
INFO:tensorflow:lr = 0.0005358916 (10.561 sec)
INFO:tensorflow:global_step/sec: 9.32461
INFO:tensorflow:loss = 5.055513, step = 53900 (10.726 sec)
INFO:tensorflow:lr = 0.0005340996 (10.724 sec)
INFO:tensorflow:global_step/sec: 9.46242
INFO:tensorflow:loss = 4.9468417, step = 54000 (10.569 sec)
INFO:tensorflow:lr = 0.0005323076 (10.568 sec)
INFO:tensorflow:global_step/sec: 9.21342
INFO:tensorflow:loss = 5.1168594, step = 54100 (10.854 sec)
INFO:tensorflow:lr = 0.00053051556 (10.854 sec)
INFO:tensorflow:global_step/sec: 9.40069
INFO:tensorflow:loss = 4.906956, step = 54200 (10.637 sec)
INFO:tensorflow:lr = 0.0005287236 (10.637 sec)
INFO:tensorflow:global_step/sec: 9.24311
INFO:tensorflow:loss = 5.017212, step = 54300 (10.820 sec)
INFO:tensorflow:lr = 0.00052693154 (10.819 sec)
INFO:tensorflow:global_step/sec: 9.28552
INFO:tensorflow:loss = 5.069296, step = 54400 (10.770 sec)
INFO:tensorflow:lr = 0.00052513945 (10.769 sec)
INFO:tensorflow:global_step/sec: 9.49994
INFO:tensorflow:loss = 4.862296, step = 54500 (10.527 sec)
INFO:tensorflow:lr = 0.00052334747 (10.527 sec)
INFO:tensorflow:global_step/sec: 9.47938
INFO:tensorflow:loss = 5.018501, step = 54600 (10.548 sec)
INFO:tensorflow:lr = 0.0005215554 (10.549 sec)
INFO:tensorflow:global_step/sec: 9.65287
INFO:tensorflow:loss = 4.8488297, step = 54700 (10.360 sec)
INFO:tensorflow:lr = 0.00051976345 (10.359 sec)
INFO:tensorflow:global_step/sec: 9.33386
INFO:tensorflow:loss = 4.833839, step = 54800 (10.715 sec)
INFO:tensorflow:lr = 0.0005179714 (10.715 sec)
INFO:tensorflow:global_step/sec: 9.21715
INFO:tensorflow:loss = 4.930398, step = 54900 (10.847 sec)
INFO:tensorflow:lr = 0.00051617937 (10.847 sec)
INFO:tensorflow:global_step/sec: 9.2234
INFO:tensorflow:loss = 4.9686055, step = 55000 (10.843 sec)
INFO:tensorflow:lr = 0.0005143874 (10.843 sec)
INFO:tensorflow:global_step/sec: 9.50335
INFO:tensorflow:loss = 5.014664, step = 55100 (10.520 sec)
INFO:tensorflow:lr = 0.00051259535 (10.521 sec)
INFO:tensorflow:global_step/sec: 9.50403
INFO:tensorflow:loss = 4.879487, step = 55200 (10.524 sec)
INFO:tensorflow:lr = 0.00051080337 (10.524 sec)
INFO:tensorflow:global_step/sec: 9.44257
INFO:tensorflow:loss = 5.077122, step = 55300 (10.592 sec)
INFO:tensorflow:lr = 0.00050901127 (10.591 sec)
INFO:tensorflow:global_step/sec: 9.43012
INFO:tensorflow:loss = 4.927007, step = 55400 (10.604 sec)
INFO:tensorflow:lr = 0.0005072192 (10.606 sec)
INFO:tensorflow:global_step/sec: 9.391
INFO:tensorflow:loss = 5.031347, step = 55500 (10.648 sec)
INFO:tensorflow:lr = 0.00050542725 (10.647 sec)
INFO:tensorflow:global_step/sec: 9.33362
INFO:tensorflow:loss = 4.995307, step = 55600 (10.714 sec)
INFO:tensorflow:lr = 0.0005036352 (10.714 sec)
INFO:tensorflow:global_step/sec: 9.42561
INFO:tensorflow:loss = 5.0791025, step = 55700 (10.606 sec)
INFO:tensorflow:lr = 0.0005018432 (10.608 sec)
INFO:tensorflow:global_step/sec: 9.58455
INFO:tensorflow:loss = 4.891101, step = 55800 (10.437 sec)
INFO:tensorflow:lr = 0.0005000512 (10.434 sec)
INFO:tensorflow:global_step/sec: 9.49456
INFO:tensorflow:loss = 4.8779163, step = 55900 (10.531 sec)
INFO:tensorflow:lr = 0.00049825915 (10.531 sec)
INFO:tensorflow:global_step/sec: 9.55119
INFO:tensorflow:loss = 4.991086, step = 56000 (10.470 sec)
INFO:tensorflow:lr = 0.00049646717 (10.471 sec)
INFO:tensorflow:global_step/sec: 9.55963
INFO:tensorflow:loss = 4.9344687, step = 56100 (10.461 sec)
INFO:tensorflow:lr = 0.0004946751 (10.460 sec)
INFO:tensorflow:global_step/sec: 9.46301
INFO:tensorflow:loss = 4.91139, step = 56200 (10.567 sec)
INFO:tensorflow:lr = 0.00049288303 (10.567 sec)
INFO:tensorflow:global_step/sec: 9.54473
INFO:tensorflow:loss = 4.9695115, step = 56300 (10.478 sec)
INFO:tensorflow:lr = 0.00049109105 (10.478 sec)
INFO:tensorflow:global_step/sec: 9.63134
INFO:tensorflow:loss = 4.8727584, step = 56400 (10.382 sec)
INFO:tensorflow:lr = 0.000489299 (10.382 sec)
INFO:tensorflow:global_step/sec: 9.52309
INFO:tensorflow:loss = 5.0401974, step = 56500 (10.502 sec)
INFO:tensorflow:lr = 0.00048750703 (10.502 sec)
INFO:tensorflow:global_step/sec: 9.65266
INFO:tensorflow:loss = 4.8206286, step = 56600 (10.356 sec)
INFO:tensorflow:lr = 0.00048571502 (10.357 sec)
INFO:tensorflow:global_step/sec: 9.64404
INFO:tensorflow:loss = 4.914998, step = 56700 (10.371 sec)
INFO:tensorflow:lr = 0.000483923 (10.372 sec)
INFO:tensorflow:global_step/sec: 9.50445
INFO:tensorflow:loss = 4.927148, step = 56800 (10.523 sec)
INFO:tensorflow:lr = 0.00048213097 (10.522 sec)
INFO:tensorflow:global_step/sec: 9.68784
INFO:tensorflow:loss = 4.9782243, step = 56900 (10.321 sec)
INFO:tensorflow:lr = 0.00048033896 (10.321 sec)
INFO:tensorflow:global_step/sec: 9.50577
INFO:tensorflow:loss = 4.9523335, step = 57000 (10.518 sec)
INFO:tensorflow:lr = 0.00047854695 (10.518 sec)
INFO:tensorflow:global_step/sec: 9.50627
INFO:tensorflow:loss = 5.079739, step = 57100 (10.522 sec)
INFO:tensorflow:lr = 0.00047675485 (10.521 sec)
INFO:tensorflow:global_step/sec: 9.33996
INFO:tensorflow:loss = 5.0048776, step = 57200 (10.707 sec)
INFO:tensorflow:lr = 0.00047496284 (10.707 sec)
INFO:tensorflow:global_step/sec: 9.66024
INFO:tensorflow:loss = 4.9426284, step = 57300 (10.348 sec)
INFO:tensorflow:lr = 0.00047317083 (10.349 sec)
INFO:tensorflow:global_step/sec: 9.76629
INFO:tensorflow:loss = 5.041515, step = 57400 (10.241 sec)
INFO:tensorflow:lr = 0.00047137882 (10.241 sec)
INFO:tensorflow:global_step/sec: 9.49702
INFO:tensorflow:loss = 4.868841, step = 57500 (10.531 sec)
INFO:tensorflow:lr = 0.0004695868 (10.530 sec)
INFO:tensorflow:global_step/sec: 9.54386
INFO:tensorflow:loss = 5.01951, step = 57600 (10.477 sec)
INFO:tensorflow:lr = 0.0004677948 (10.477 sec)
INFO:tensorflow:global_step/sec: 9.60844
INFO:tensorflow:loss = 4.9338384, step = 57700 (10.408 sec)
INFO:tensorflow:lr = 0.0004660028 (10.409 sec)
INFO:tensorflow:global_step/sec: 9.30532
INFO:tensorflow:loss = 4.9551077, step = 57800 (10.748 sec)
INFO:tensorflow:lr = 0.00046421075 (10.747 sec)
INFO:tensorflow:global_step/sec: 9.30418
INFO:tensorflow:loss = 5.0550914, step = 57900 (10.746 sec)
INFO:tensorflow:lr = 0.00046241874 (10.747 sec)
INFO:tensorflow:global_step/sec: 9.43973
INFO:tensorflow:loss = 4.955047, step = 58000 (10.596 sec)
INFO:tensorflow:lr = 0.00046062664 (10.594 sec)
INFO:tensorflow:global_step/sec: 9.3222
INFO:tensorflow:loss = 5.0358024, step = 58100 (10.725 sec)
INFO:tensorflow:lr = 0.00045883463 (10.729 sec)
INFO:tensorflow:global_step/sec: 9.26154
INFO:tensorflow:loss = 4.8850555, step = 58200 (10.797 sec)
INFO:tensorflow:lr = 0.00045704262 (10.794 sec)
INFO:tensorflow:global_step/sec: 9.51078
INFO:tensorflow:loss = 4.9150257, step = 58300 (10.513 sec)
INFO:tensorflow:lr = 0.0004552506 (10.512 sec)
INFO:tensorflow:global_step/sec: 9.46567
INFO:tensorflow:loss = 4.9813275, step = 58400 (10.567 sec)
INFO:tensorflow:lr = 0.0004534586 (10.567 sec)
INFO:tensorflow:global_step/sec: 9.51677
INFO:tensorflow:loss = 4.9747343, step = 58500 (10.509 sec)
INFO:tensorflow:lr = 0.0004516666 (10.509 sec)
INFO:tensorflow:global_step/sec: 9.37885
INFO:tensorflow:loss = 5.0838323, step = 58600 (10.661 sec)
INFO:tensorflow:lr = 0.00044987458 (10.661 sec)
INFO:tensorflow:global_step/sec: 9.44478
INFO:tensorflow:loss = 4.9560633, step = 58700 (10.586 sec)
INFO:tensorflow:lr = 0.00044808254 (10.587 sec)
INFO:tensorflow:global_step/sec: 9.36644
INFO:tensorflow:loss = 5.092565, step = 58800 (10.676 sec)
INFO:tensorflow:lr = 0.00044629053 (10.675 sec)
INFO:tensorflow:global_step/sec: 9.57847
INFO:tensorflow:loss = 5.1015515, step = 58900 (10.443 sec)
INFO:tensorflow:lr = 0.00044449844 (10.443 sec)
INFO:tensorflow:global_step/sec: 9.66363
INFO:tensorflow:loss = 4.9495497, step = 59000 (10.348 sec)
INFO:tensorflow:lr = 0.00044270643 (10.348 sec)
INFO:tensorflow:global_step/sec: 9.25838
INFO:tensorflow:loss = 4.9359717, step = 59100 (10.799 sec)
INFO:tensorflow:lr = 0.00044091442 (10.800 sec)
INFO:tensorflow:global_step/sec: 9.69576
INFO:tensorflow:loss = 5.07738, step = 59200 (10.315 sec)
INFO:tensorflow:lr = 0.0004391224 (10.314 sec)
INFO:tensorflow:global_step/sec: 9.55058
INFO:tensorflow:loss = 4.9461327, step = 59300 (10.470 sec)
INFO:tensorflow:lr = 0.0004373304 (10.470 sec)
INFO:tensorflow:global_step/sec: 9.63653
INFO:tensorflow:loss = 4.9056435, step = 59400 (10.377 sec)
INFO:tensorflow:lr = 0.0004355384 (10.376 sec)
INFO:tensorflow:global_step/sec: 9.62116
INFO:tensorflow:loss = 5.0284085, step = 59500 (10.396 sec)
INFO:tensorflow:lr = 0.00043374638 (10.397 sec)
INFO:tensorflow:global_step/sec: 9.38683
INFO:tensorflow:loss = 5.1237197, step = 59600 (10.651 sec)
INFO:tensorflow:lr = 0.00043195437 (10.651 sec)
INFO:tensorflow:global_step/sec: 9.33088
INFO:tensorflow:loss = 5.0115285, step = 59700 (10.717 sec)
INFO:tensorflow:lr = 0.00043016233 (10.717 sec)
INFO:tensorflow:global_step/sec: 9.35386
INFO:tensorflow:loss = 4.90229, step = 59800 (10.691 sec)
INFO:tensorflow:lr = 0.00042837023 (10.691 sec)
INFO:tensorflow:global_step/sec: 9.55297
INFO:tensorflow:loss = 4.9314675, step = 59900 (10.470 sec)
INFO:tensorflow:lr = 0.00042657822 (10.469 sec)
INFO:tensorflow:global_step/sec: 9.66096
INFO:tensorflow:loss = 4.955702, step = 60000 (10.351 sec)
INFO:tensorflow:lr = 0.0004247862 (10.351 sec)
INFO:tensorflow:global_step/sec: 9.70059
INFO:tensorflow:loss = 4.94967, step = 60100 (10.307 sec)
INFO:tensorflow:lr = 0.0004229942 (10.308 sec)
INFO:tensorflow:global_step/sec: 9.20308
INFO:tensorflow:loss = 4.909738, step = 60200 (10.868 sec)
INFO:tensorflow:lr = 0.0004212022 (10.866 sec)
INFO:tensorflow:global_step/sec: 9.69645
INFO:tensorflow:loss = 4.9466395, step = 60300 (10.312 sec)
INFO:tensorflow:lr = 0.00041941018 (10.311 sec)
INFO:tensorflow:global_step/sec: 9.47731
INFO:tensorflow:loss = 4.934656, step = 60400 (10.552 sec)
INFO:tensorflow:lr = 0.00041761817 (10.554 sec)
INFO:tensorflow:global_step/sec: 9.50505
INFO:tensorflow:loss = 4.9714603, step = 60500 (10.520 sec)
INFO:tensorflow:lr = 0.00041582616 (10.519 sec)
INFO:tensorflow:global_step/sec: 9.40126
INFO:tensorflow:loss = 5.0126266, step = 60600 (10.637 sec)
INFO:tensorflow:lr = 0.00041403406 (10.637 sec)
INFO:tensorflow:global_step/sec: 9.37801
INFO:tensorflow:loss = 4.950481, step = 60700 (10.661 sec)
INFO:tensorflow:lr = 0.00041224205 (10.662 sec)
INFO:tensorflow:global_step/sec: 9.29794
INFO:tensorflow:loss = 4.879242, step = 60800 (10.756 sec)
INFO:tensorflow:lr = 0.00041045 (10.755 sec)
INFO:tensorflow:global_step/sec: 9.71345
INFO:tensorflow:loss = 4.974057, step = 60900 (10.295 sec)
INFO:tensorflow:lr = 0.000408658 (10.297 sec)
INFO:tensorflow:global_step/sec: 9.37829
INFO:tensorflow:loss = 5.0180244, step = 61000 (10.664 sec)
INFO:tensorflow:lr = 0.000406866 (10.664 sec)
INFO:tensorflow:global_step/sec: 9.43858
INFO:tensorflow:loss = 4.973854, step = 61100 (10.592 sec)
INFO:tensorflow:lr = 0.00040507398 (10.593 sec)
INFO:tensorflow:global_step/sec: 9.48033
INFO:tensorflow:loss = 4.9414907, step = 61200 (10.548 sec)
INFO:tensorflow:lr = 0.00040328197 (10.548 sec)
INFO:tensorflow:global_step/sec: 9.44247
INFO:tensorflow:loss = 5.125211, step = 61300 (10.593 sec)
INFO:tensorflow:lr = 0.00040148996 (10.592 sec)
INFO:tensorflow:global_step/sec: 9.5666
INFO:tensorflow:loss = 5.022872, step = 61400 (10.451 sec)
INFO:tensorflow:lr = 0.00039969795 (10.451 sec)
INFO:tensorflow:global_step/sec: 9.49959
INFO:tensorflow:loss = 5.071593, step = 61500 (10.527 sec)
INFO:tensorflow:lr = 0.00039790585 (10.527 sec)
INFO:tensorflow:global_step/sec: 9.35442
INFO:tensorflow:loss = 4.858541, step = 61600 (10.690 sec)
INFO:tensorflow:lr = 0.00039611384 (10.690 sec)
INFO:tensorflow:global_step/sec: 9.48098
INFO:tensorflow:loss = 5.007348, step = 61700 (10.551 sec)
INFO:tensorflow:lr = 0.0003943218 (10.549 sec)
INFO:tensorflow:global_step/sec: 9.1506
INFO:tensorflow:loss = 5.004369, step = 61800 (10.925 sec)
INFO:tensorflow:lr = 0.0003925298 (10.926 sec)
INFO:tensorflow:global_step/sec: 9.38337
INFO:tensorflow:loss = 5.111853, step = 61900 (10.658 sec)
INFO:tensorflow:lr = 0.00039073778 (10.659 sec)
INFO:tensorflow:global_step/sec: 9.1282
INFO:tensorflow:loss = 4.9753876, step = 62000 (10.955 sec)
INFO:tensorflow:lr = 0.00038894577 (10.953 sec)
INFO:tensorflow:global_step/sec: 9.50418
INFO:tensorflow:loss = 5.13612, step = 62100 (10.523 sec)
INFO:tensorflow:lr = 0.00038715376 (10.524 sec)
INFO:tensorflow:global_step/sec: 9.69161
INFO:tensorflow:loss = 4.8964615, step = 62200 (10.318 sec)
INFO:tensorflow:lr = 0.00038536175 (10.318 sec)
INFO:tensorflow:global_step/sec: 9.43908
INFO:tensorflow:loss = 4.858383, step = 62300 (10.592 sec)
INFO:tensorflow:lr = 0.00038356974 (10.592 sec)
INFO:tensorflow:global_step/sec: 9.5179
INFO:tensorflow:loss = 5.02234, step = 62400 (10.509 sec)
INFO:tensorflow:lr = 0.00038177765 (10.509 sec)
INFO:tensorflow:global_step/sec: 9.29735
INFO:tensorflow:loss = 5.062104, step = 62500 (10.755 sec)
INFO:tensorflow:lr = 0.00037998563 (10.756 sec)
INFO:tensorflow:global_step/sec: 9.37313
INFO:tensorflow:loss = 4.941778, step = 62600 (10.670 sec)
INFO:tensorflow:lr = 0.00037819362 (10.669 sec)
INFO:tensorflow:global_step/sec: 9.20457
INFO:tensorflow:loss = 4.950422, step = 62700 (10.865 sec)
INFO:tensorflow:lr = 0.0003764016 (10.865 sec)
INFO:tensorflow:global_step/sec: 9.48811
INFO:tensorflow:loss = 4.9543242, step = 62800 (10.538 sec)
INFO:tensorflow:lr = 0.00037460958 (10.538 sec)
INFO:tensorflow:global_step/sec: 9.45625
INFO:tensorflow:loss = 4.930625, step = 62900 (10.575 sec)
INFO:tensorflow:lr = 0.00037281757 (10.575 sec)
INFO:tensorflow:global_step/sec: 9.50433
INFO:tensorflow:loss = 4.8656454, step = 63000 (10.521 sec)
INFO:tensorflow:lr = 0.00037102556 (10.523 sec)
INFO:tensorflow:global_step/sec: 9.39489
INFO:tensorflow:loss = 4.959864, step = 63100 (10.643 sec)
INFO:tensorflow:lr = 0.00036923354 (10.642 sec)
INFO:tensorflow:global_step/sec: 9.35715
INFO:tensorflow:loss = 4.9245005, step = 63200 (10.687 sec)
INFO:tensorflow:lr = 0.00036744153 (10.687 sec)
INFO:tensorflow:global_step/sec: 9.25415
INFO:tensorflow:loss = 4.8904448, step = 63300 (10.805 sec)
INFO:tensorflow:lr = 0.00036564944 (10.805 sec)
INFO:tensorflow:global_step/sec: 9.49882
INFO:tensorflow:loss = 5.053169, step = 63400 (10.531 sec)
INFO:tensorflow:lr = 0.00036385743 (10.533 sec)
INFO:tensorflow:global_step/sec: 9.36792
INFO:tensorflow:loss = 5.0318637, step = 63500 (10.673 sec)
INFO:tensorflow:lr = 0.00036206542 (10.672 sec)
INFO:tensorflow:global_step/sec: 9.46675
INFO:tensorflow:loss = 5.0811977, step = 63600 (10.562 sec)
INFO:tensorflow:lr = 0.0003602734 (10.561 sec)
INFO:tensorflow:global_step/sec: 9.09736
INFO:tensorflow:loss = 5.030285, step = 63700 (10.993 sec)
INFO:tensorflow:lr = 0.00035848137 (10.994 sec)
INFO:tensorflow:global_step/sec: 9.36226
INFO:tensorflow:loss = 5.0793304, step = 63800 (10.680 sec)
INFO:tensorflow:lr = 0.00035668936 (10.680 sec)
INFO:tensorflow:global_step/sec: 9.31215
INFO:tensorflow:loss = 4.9413056, step = 63900 (10.742 sec)
INFO:tensorflow:lr = 0.00035489735 (10.743 sec)
INFO:tensorflow:global_step/sec: 9.16499
INFO:tensorflow:loss = 5.0602775, step = 64000 (10.909 sec)
INFO:tensorflow:lr = 0.00035310534 (10.909 sec)
INFO:tensorflow:global_step/sec: 9.24164
INFO:tensorflow:loss = 5.0007434, step = 64100 (10.822 sec)
INFO:tensorflow:lr = 0.00035131333 (10.821 sec)
INFO:tensorflow:global_step/sec: 9.4054
INFO:tensorflow:loss = 4.8613544, step = 64200 (10.631 sec)
INFO:tensorflow:lr = 0.00034952123 (10.631 sec)
INFO:tensorflow:global_step/sec: 9.44425
INFO:tensorflow:loss = 5.021824, step = 64300 (10.589 sec)
INFO:tensorflow:lr = 0.00034772922 (10.590 sec)
INFO:tensorflow:global_step/sec: 9.54381
INFO:tensorflow:loss = 4.8927984, step = 64400 (10.477 sec)
INFO:tensorflow:lr = 0.0003459372 (10.478 sec)
INFO:tensorflow:global_step/sec: 9.53895
INFO:tensorflow:loss = 4.904232, step = 64500 (10.486 sec)
INFO:tensorflow:lr = 0.0003441452 (10.484 sec)
INFO:tensorflow:global_step/sec: 9.34092
INFO:tensorflow:loss = 4.911748, step = 64600 (10.705 sec)
INFO:tensorflow:lr = 0.00034235316 (10.705 sec)
INFO:tensorflow:global_step/sec: 9.52615
INFO:tensorflow:loss = 5.046228, step = 64700 (10.496 sec)
INFO:tensorflow:lr = 0.00034056115 (10.497 sec)
INFO:tensorflow:global_step/sec: 9.33571
INFO:tensorflow:loss = 4.869957, step = 64800 (10.715 sec)
INFO:tensorflow:lr = 0.00033876914 (10.715 sec)
INFO:tensorflow:global_step/sec: 9.18679
INFO:tensorflow:loss = 4.893039, step = 64900 (10.880 sec)
INFO:tensorflow:lr = 0.00033697713 (10.879 sec)
INFO:tensorflow:global_step/sec: 9.39862
INFO:tensorflow:loss = 4.911767, step = 65000 (10.642 sec)
INFO:tensorflow:lr = 0.00033518512 (10.642 sec)
INFO:tensorflow:global_step/sec: 9.37078
INFO:tensorflow:loss = 4.999353, step = 65100 (10.671 sec)
INFO:tensorflow:lr = 0.00033339302 (10.670 sec)
INFO:tensorflow:global_step/sec: 9.57748
INFO:tensorflow:loss = 4.7816143, step = 65200 (10.443 sec)
INFO:tensorflow:lr = 0.000331601 (10.443 sec)
INFO:tensorflow:global_step/sec: 9.20538
INFO:tensorflow:loss = 4.998312, step = 65300 (10.863 sec)
INFO:tensorflow:lr = 0.000329809 (10.862 sec)
INFO:tensorflow:global_step/sec: 9.2367
INFO:tensorflow:loss = 4.85855, step = 65400 (10.826 sec)
INFO:tensorflow:lr = 0.00032801696 (10.826 sec)
INFO:tensorflow:global_step/sec: 9.14618
INFO:tensorflow:loss = 4.9665437, step = 65500 (10.934 sec)
INFO:tensorflow:lr = 0.00032622495 (10.934 sec)
INFO:tensorflow:global_step/sec: 9.45766
INFO:tensorflow:loss = 4.899067, step = 65600 (10.572 sec)
INFO:tensorflow:lr = 0.00032443294 (10.574 sec)
INFO:tensorflow:global_step/sec: 9.35764
INFO:tensorflow:loss = 4.946297, step = 65700 (10.687 sec)
INFO:tensorflow:lr = 0.00032264093 (10.686 sec)
INFO:tensorflow:global_step/sec: 9.2286
INFO:tensorflow:loss = 5.0569944, step = 65800 (10.833 sec)
INFO:tensorflow:lr = 0.00032084892 (10.835 sec)
INFO:tensorflow:global_step/sec: 9.36854
INFO:tensorflow:loss = 4.9371705, step = 65900 (10.677 sec)
INFO:tensorflow:lr = 0.0003190569 (10.675 sec)
INFO:tensorflow:global_step/sec: 9.62625
INFO:tensorflow:loss = 5.1222367, step = 66000 (10.386 sec)
INFO:tensorflow:lr = 0.0003172648 (10.387 sec)
INFO:tensorflow:global_step/sec: 9.64598
INFO:tensorflow:loss = 4.981133, step = 66100 (10.370 sec)
INFO:tensorflow:lr = 0.0003154728 (10.369 sec)
INFO:tensorflow:global_step/sec: 9.52698
INFO:tensorflow:loss = 4.897505, step = 66200 (10.494 sec)
INFO:tensorflow:lr = 0.0003136808 (10.494 sec)
INFO:tensorflow:global_step/sec: 9.47894
INFO:tensorflow:loss = 4.934471, step = 66300 (10.550 sec)
INFO:tensorflow:lr = 0.00031188875 (10.551 sec)
INFO:tensorflow:global_step/sec: 9.51108
INFO:tensorflow:loss = 4.993697, step = 66400 (10.515 sec)
INFO:tensorflow:lr = 0.00031009674 (10.515 sec)
INFO:tensorflow:global_step/sec: 9.49395
INFO:tensorflow:loss = 4.906064, step = 66500 (10.534 sec)
INFO:tensorflow:lr = 0.00030830473 (10.534 sec)
INFO:tensorflow:global_step/sec: 9.35772
INFO:tensorflow:loss = 5.0655017, step = 66600 (10.687 sec)
INFO:tensorflow:lr = 0.00030651272 (10.685 sec)
INFO:tensorflow:global_step/sec: 9.62836
INFO:tensorflow:loss = 5.0876303, step = 66700 (10.386 sec)
INFO:tensorflow:lr = 0.0003047207 (10.386 sec)
INFO:tensorflow:global_step/sec: 9.60331
INFO:tensorflow:loss = 5.1032166, step = 66800 (10.411 sec)
INFO:tensorflow:lr = 0.0003029287 (10.413 sec)
INFO:tensorflow:global_step/sec: 9.5167
INFO:tensorflow:loss = 4.8347807, step = 66900 (10.507 sec)
INFO:tensorflow:lr = 0.0003011366 (10.507 sec)
INFO:tensorflow:global_step/sec: 9.6098
INFO:tensorflow:loss = 4.870035, step = 67000 (10.407 sec)
INFO:tensorflow:lr = 0.0002993446 (10.406 sec)
INFO:tensorflow:global_step/sec: 9.5727
INFO:tensorflow:loss = 5.024679, step = 67100 (10.447 sec)
INFO:tensorflow:lr = 0.0002975526 (10.448 sec)
INFO:tensorflow:global_step/sec: 9.423
INFO:tensorflow:loss = 5.0179296, step = 67200 (10.610 sec)
INFO:tensorflow:lr = 0.00029576058 (10.611 sec)
INFO:tensorflow:global_step/sec: 9.37353
INFO:tensorflow:loss = 4.920677, step = 67300 (10.670 sec)
INFO:tensorflow:lr = 0.00029396854 (10.669 sec)
INFO:tensorflow:global_step/sec: 9.55465
INFO:tensorflow:loss = 5.009587, step = 67400 (10.466 sec)
INFO:tensorflow:lr = 0.00029217653 (10.468 sec)
INFO:tensorflow:global_step/sec: 9.47831
INFO:tensorflow:loss = 4.9350553, step = 67500 (10.550 sec)
INFO:tensorflow:lr = 0.00029038452 (10.548 sec)
INFO:tensorflow:global_step/sec: 9.74646
INFO:tensorflow:loss = 4.977632, step = 67600 (10.259 sec)
INFO:tensorflow:lr = 0.0002885925 (10.259 sec)
INFO:tensorflow:global_step/sec: 9.43465
INFO:tensorflow:loss = 5.0136166, step = 67700 (10.600 sec)
INFO:tensorflow:lr = 0.0002868005 (10.598 sec)
INFO:tensorflow:global_step/sec: 9.5977
INFO:tensorflow:loss = 5.0190306, step = 67800 (10.419 sec)
INFO:tensorflow:lr = 0.0002850084 (10.420 sec)
INFO:tensorflow:global_step/sec: 9.34549
INFO:tensorflow:loss = 4.88896, step = 67900 (10.700 sec)
INFO:tensorflow:lr = 0.0002832164 (10.701 sec)
INFO:tensorflow:global_step/sec: 9.46322
INFO:tensorflow:loss = 4.9357834, step = 68000 (10.565 sec)
INFO:tensorflow:lr = 0.00028142438 (10.566 sec)
INFO:tensorflow:global_step/sec: 9.31911
INFO:tensorflow:loss = 4.977519, step = 68100 (10.731 sec)
INFO:tensorflow:lr = 0.00027963237 (10.729 sec)
INFO:tensorflow:global_step/sec: 9.41641
INFO:tensorflow:loss = 5.1117687, step = 68200 (10.621 sec)
INFO:tensorflow:lr = 0.00027784036 (10.622 sec)
INFO:tensorflow:global_step/sec: 9.5828
INFO:tensorflow:loss = 5.0275726, step = 68300 (10.436 sec)
INFO:tensorflow:lr = 0.00027604832 (10.436 sec)
INFO:tensorflow:global_step/sec: 9.64405
INFO:tensorflow:loss = 4.9940405, step = 68400 (10.370 sec)
INFO:tensorflow:lr = 0.0002742563 (10.369 sec)
INFO:tensorflow:global_step/sec: 9.19497
INFO:tensorflow:loss = 4.910246, step = 68500 (10.875 sec)
INFO:tensorflow:lr = 0.0002724643 (10.875 sec)
INFO:tensorflow:global_step/sec: 9.37872
INFO:tensorflow:loss = 4.858515, step = 68600 (10.662 sec)
INFO:tensorflow:lr = 0.0002706723 (10.662 sec)
INFO:tensorflow:global_step/sec: 9.80252
INFO:tensorflow:loss = 4.936463, step = 68700 (10.199 sec)
INFO:tensorflow:lr = 0.0002688802 (10.200 sec)
INFO:tensorflow:global_step/sec: 9.47177
INFO:tensorflow:loss = 4.845891, step = 68800 (10.561 sec)
INFO:tensorflow:lr = 0.00026708818 (10.560 sec)
INFO:tensorflow:global_step/sec: 9.57415
INFO:tensorflow:loss = 4.938409, step = 68900 (10.442 sec)
INFO:tensorflow:lr = 0.00026529617 (10.441 sec)
INFO:tensorflow:global_step/sec: 9.68101
INFO:tensorflow:loss = 4.938816, step = 69000 (10.333 sec)
INFO:tensorflow:lr = 0.00026350416 (10.333 sec)
INFO:tensorflow:global_step/sec: 9.64886
INFO:tensorflow:loss = 5.0645933, step = 69100 (10.362 sec)
INFO:tensorflow:lr = 0.00026171215 (10.362 sec)
INFO:tensorflow:global_step/sec: 9.42143
INFO:tensorflow:loss = 5.025646, step = 69200 (10.615 sec)
INFO:tensorflow:lr = 0.0002599201 (10.615 sec)
INFO:tensorflow:global_step/sec: 9.37213
INFO:tensorflow:loss = 4.964944, step = 69300 (10.672 sec)
INFO:tensorflow:lr = 0.0002581281 (10.671 sec)
INFO:tensorflow:global_step/sec: 9.43661
INFO:tensorflow:loss = 5.027821, step = 69400 (10.593 sec)
INFO:tensorflow:lr = 0.0002563361 (10.593 sec)
INFO:tensorflow:global_step/sec: 9.58868
INFO:tensorflow:loss = 4.927606, step = 69500 (10.431 sec)
INFO:tensorflow:lr = 0.00025454408 (10.434 sec)
INFO:tensorflow:global_step/sec: 9.45007
INFO:tensorflow:loss = 5.012537, step = 69600 (10.582 sec)
INFO:tensorflow:lr = 0.00025275198 (10.580 sec)
INFO:tensorflow:global_step/sec: 9.74233
INFO:tensorflow:loss = 5.031917, step = 69700 (10.264 sec)
INFO:tensorflow:lr = 0.00025095997 (10.265 sec)
INFO:tensorflow:global_step/sec: 9.67259
INFO:tensorflow:loss = 5.0179787, step = 69800 (10.338 sec)
INFO:tensorflow:lr = 0.00024916796 (10.338 sec)
INFO:tensorflow:global_step/sec: 9.60014
INFO:tensorflow:loss = 4.998183, step = 69900 (10.417 sec)
INFO:tensorflow:lr = 0.00024737595 (10.416 sec)
INFO:tensorflow:global_step/sec: 9.80401
INFO:tensorflow:loss = 5.025863, step = 70000 (10.199 sec)
INFO:tensorflow:lr = 0.00024558394 (10.201 sec)
INFO:tensorflow:global_step/sec: 9.33348
INFO:tensorflow:loss = 4.792766, step = 70100 (10.715 sec)
INFO:tensorflow:lr = 0.00024379193 (10.713 sec)
INFO:tensorflow:global_step/sec: 9.44661
INFO:tensorflow:loss = 4.921283, step = 70200 (10.586 sec)
INFO:tensorflow:lr = 0.0002419999 (10.586 sec)
INFO:tensorflow:global_step/sec: 9.24838
INFO:tensorflow:loss = 4.991687, step = 70300 (10.813 sec)
INFO:tensorflow:lr = 0.00024020788 (10.813 sec)
INFO:tensorflow:global_step/sec: 9.79256
INFO:tensorflow:loss = 4.8852735, step = 70400 (10.212 sec)
INFO:tensorflow:lr = 0.00023841587 (10.214 sec)
INFO:tensorflow:global_step/sec: 10.103
INFO:tensorflow:loss = 4.789762, step = 70500 (9.897 sec)
INFO:tensorflow:lr = 0.00023662377 (9.896 sec)
INFO:tensorflow:global_step/sec: 10.0772
INFO:tensorflow:loss = 4.9286113, step = 70600 (9.921 sec)
INFO:tensorflow:lr = 0.00023483176 (9.920 sec)
INFO:tensorflow:global_step/sec: 10.1454
INFO:tensorflow:loss = 5.0289354, step = 70700 (9.858 sec)
INFO:tensorflow:lr = 0.00023303975 (9.859 sec)
INFO:tensorflow:global_step/sec: 10.0297
INFO:tensorflow:loss = 4.8507133, step = 70800 (9.969 sec)
INFO:tensorflow:lr = 0.00023124774 (9.970 sec)
INFO:tensorflow:global_step/sec: 9.95442
INFO:tensorflow:loss = 4.893291, step = 70900 (10.047 sec)
INFO:tensorflow:lr = 0.00022945573 (10.046 sec)
INFO:tensorflow:global_step/sec: 10.0049
INFO:tensorflow:loss = 4.885566, step = 71000 (9.994 sec)
INFO:tensorflow:lr = 0.00022766372 (9.993 sec)
INFO:tensorflow:global_step/sec: 9.98131
INFO:tensorflow:loss = 4.929348, step = 71100 (10.022 sec)
INFO:tensorflow:lr = 0.00022587168 (10.023 sec)
INFO:tensorflow:global_step/sec: 9.6712
INFO:tensorflow:loss = 4.8488727, step = 71200 (10.340 sec)
INFO:tensorflow:lr = 0.00022407967 (10.339 sec)
INFO:tensorflow:global_step/sec: 10.0553
INFO:tensorflow:loss = 4.949923, step = 71300 (9.943 sec)
INFO:tensorflow:lr = 0.00022228766 (9.943 sec)
INFO:tensorflow:global_step/sec: 9.79252
INFO:tensorflow:loss = 4.901216, step = 71400 (10.213 sec)
INFO:tensorflow:lr = 0.00022049557 (10.213 sec)
INFO:tensorflow:global_step/sec: 9.91906
INFO:tensorflow:loss = 4.825674, step = 71500 (10.082 sec)
INFO:tensorflow:lr = 0.00021870356 (10.082 sec)
INFO:tensorflow:global_step/sec: 9.67885
INFO:tensorflow:loss = 4.869548, step = 71600 (10.330 sec)
INFO:tensorflow:lr = 0.00021691155 (10.329 sec)
INFO:tensorflow:global_step/sec: 9.89107
INFO:tensorflow:loss = 4.974258, step = 71700 (10.110 sec)
INFO:tensorflow:lr = 0.00021511954 (10.111 sec)
INFO:tensorflow:global_step/sec: 9.72268
INFO:tensorflow:loss = 4.9382033, step = 71800 (10.287 sec)
INFO:tensorflow:lr = 0.00021332753 (10.286 sec)
INFO:tensorflow:global_step/sec: 9.70116
INFO:tensorflow:loss = 4.985545, step = 71900 (10.306 sec)
INFO:tensorflow:lr = 0.00021153552 (10.307 sec)
INFO:tensorflow:global_step/sec: 9.96855
INFO:tensorflow:loss = 4.9777007, step = 72000 (10.033 sec)
INFO:tensorflow:lr = 0.00020974349 (10.034 sec)
INFO:tensorflow:global_step/sec: 10.0754
INFO:tensorflow:loss = 5.077654, step = 72100 (9.928 sec)
INFO:tensorflow:lr = 0.00020795148 (9.927 sec)
INFO:tensorflow:global_step/sec: 9.83021
INFO:tensorflow:loss = 4.855007, step = 72200 (10.171 sec)
INFO:tensorflow:lr = 0.00020615946 (10.172 sec)
INFO:tensorflow:global_step/sec: 10.0145
INFO:tensorflow:loss = 4.9308124, step = 72300 (9.985 sec)
INFO:tensorflow:lr = 0.00020436736 (9.984 sec)
INFO:tensorflow:global_step/sec: 9.84289
INFO:tensorflow:loss = 5.0317636, step = 72400 (10.161 sec)
INFO:tensorflow:lr = 0.00020257535 (10.162 sec)
INFO:tensorflow:global_step/sec: 9.59846
INFO:tensorflow:loss = 4.929287, step = 72500 (10.418 sec)
INFO:tensorflow:lr = 0.00020078334 (10.418 sec)
INFO:tensorflow:global_step/sec: 9.33104
INFO:tensorflow:loss = 4.75825, step = 72600 (10.717 sec)
INFO:tensorflow:lr = 0.00019899133 (10.717 sec)
INFO:tensorflow:global_step/sec: 10.0884
INFO:tensorflow:loss = 4.926864, step = 72700 (9.909 sec)
INFO:tensorflow:lr = 0.00019719932 (9.909 sec)
INFO:tensorflow:global_step/sec: 10.1122
INFO:tensorflow:loss = 4.856933, step = 72800 (9.892 sec)
INFO:tensorflow:lr = 0.00019540731 (9.892 sec)
INFO:tensorflow:global_step/sec: 10.2341
INFO:tensorflow:loss = 4.9881806, step = 72900 (9.771 sec)
INFO:tensorflow:lr = 0.0001936153 (9.771 sec)
INFO:tensorflow:global_step/sec: 10.0102
INFO:tensorflow:loss = 4.9389176, step = 73000 (9.989 sec)
INFO:tensorflow:lr = 0.00019182327 (9.990 sec)
INFO:tensorflow:global_step/sec: 10.1647
INFO:tensorflow:loss = 4.991569, step = 73100 (9.839 sec)
INFO:tensorflow:lr = 0.00019003126 (9.838 sec)
INFO:tensorflow:global_step/sec: 10.1237
INFO:tensorflow:loss = 4.8909016, step = 73200 (9.877 sec)
INFO:tensorflow:lr = 0.00018823917 (9.878 sec)
INFO:tensorflow:global_step/sec: 9.84481
INFO:tensorflow:loss = 5.0041265, step = 73300 (10.156 sec)
INFO:tensorflow:lr = 0.00018644714 (10.156 sec)
INFO:tensorflow:global_step/sec: 10.0639
INFO:tensorflow:loss = 5.0792747, step = 73400 (9.938 sec)
INFO:tensorflow:lr = 0.00018465513 (9.939 sec)
INFO:tensorflow:global_step/sec: 10.2123
INFO:tensorflow:loss = 4.9866824, step = 73500 (9.791 sec)
INFO:tensorflow:lr = 0.00018286312 (9.790 sec)
INFO:tensorflow:global_step/sec: 10.0307
INFO:tensorflow:loss = 4.9499493, step = 73600 (9.970 sec)
INFO:tensorflow:lr = 0.00018107111 (9.970 sec)
INFO:tensorflow:global_step/sec: 10.2258
INFO:tensorflow:loss = 4.94972, step = 73700 (9.780 sec)
INFO:tensorflow:lr = 0.0001792791 (9.779 sec)
INFO:tensorflow:global_step/sec: 10.1027
INFO:tensorflow:loss = 4.98489, step = 73800 (9.896 sec)
INFO:tensorflow:lr = 0.00017748709 (9.897 sec)
INFO:tensorflow:global_step/sec: 10.1063
INFO:tensorflow:loss = 4.840899, step = 73900 (9.895 sec)
INFO:tensorflow:lr = 0.00017569507 (9.897 sec)
INFO:tensorflow:global_step/sec: 9.72925
INFO:tensorflow:loss = 4.93603, step = 74000 (10.281 sec)
INFO:tensorflow:lr = 0.00017390306 (10.278 sec)
INFO:tensorflow:global_step/sec: 9.97716
INFO:tensorflow:loss = 4.947037, step = 74100 (10.021 sec)
INFO:tensorflow:lr = 0.00017211096 (10.021 sec)
INFO:tensorflow:global_step/sec: 10.0941
INFO:tensorflow:loss = 4.9884596, step = 74200 (9.908 sec)
INFO:tensorflow:lr = 0.00017031893 (9.907 sec)
INFO:tensorflow:global_step/sec: 10.0898
INFO:tensorflow:loss = 4.981375, step = 74300 (9.910 sec)
INFO:tensorflow:lr = 0.00016852692 (9.910 sec)
INFO:tensorflow:global_step/sec: 9.93729
INFO:tensorflow:loss = 5.059608, step = 74400 (10.066 sec)
INFO:tensorflow:lr = 0.00016673491 (10.066 sec)
INFO:tensorflow:global_step/sec: 10.0718
INFO:tensorflow:loss = 4.946367, step = 74500 (9.928 sec)
INFO:tensorflow:lr = 0.0001649429 (9.928 sec)
INFO:tensorflow:global_step/sec: 10.0538
INFO:tensorflow:loss = 4.967024, step = 74600 (9.945 sec)
INFO:tensorflow:lr = 0.0001631509 (9.946 sec)
INFO:tensorflow:global_step/sec: 9.94507
INFO:tensorflow:loss = 4.9448195, step = 74700 (10.056 sec)
INFO:tensorflow:lr = 0.00016135888 (10.055 sec)
INFO:tensorflow:global_step/sec: 10.0953
INFO:tensorflow:loss = 5.036145, step = 74800 (9.907 sec)
INFO:tensorflow:lr = 0.00015956686 (9.910 sec)
INFO:tensorflow:global_step/sec: 10.1335
INFO:tensorflow:loss = 5.024201, step = 74900 (9.865 sec)
INFO:tensorflow:lr = 0.00015777485 (9.865 sec)
INFO:tensorflow:global_step/sec: 10.1867
INFO:tensorflow:loss = 4.910201, step = 75000 (9.819 sec)
INFO:tensorflow:lr = 0.00015598275 (9.816 sec)
INFO:tensorflow:global_step/sec: 10.2466
INFO:tensorflow:loss = 4.8420153, step = 75100 (9.761 sec)
INFO:tensorflow:lr = 0.00015419074 (9.761 sec)
INFO:tensorflow:global_step/sec: 10.0962
INFO:tensorflow:loss = 4.831217, step = 75200 (9.903 sec)
INFO:tensorflow:lr = 0.00015239873 (9.904 sec)
INFO:tensorflow:global_step/sec: 10.1727
INFO:tensorflow:loss = 4.9388647, step = 75300 (9.831 sec)
INFO:tensorflow:lr = 0.0001506067 (9.830 sec)
INFO:tensorflow:global_step/sec: 10.0675
INFO:tensorflow:loss = 4.9850793, step = 75400 (9.933 sec)
INFO:tensorflow:lr = 0.0001488147 (9.933 sec)
INFO:tensorflow:global_step/sec: 10.2753
INFO:tensorflow:loss = 4.8257756, step = 75500 (9.729 sec)
INFO:tensorflow:lr = 0.00014702269 (9.730 sec)
INFO:tensorflow:global_step/sec: 10.0116
INFO:tensorflow:loss = 4.9931474, step = 75600 (9.990 sec)
INFO:tensorflow:lr = 0.00014523068 (9.997 sec)
INFO:tensorflow:global_step/sec: 9.89481
INFO:tensorflow:loss = 4.9086366, step = 75700 (10.107 sec)
INFO:tensorflow:lr = 0.00014343866 (10.099 sec)
INFO:tensorflow:global_step/sec: 9.89529
INFO:tensorflow:loss = 5.043158, step = 75800 (10.111 sec)
INFO:tensorflow:lr = 0.00014164664 (10.112 sec)
INFO:tensorflow:global_step/sec: 10.0947
INFO:tensorflow:loss = 5.024129, step = 75900 (9.901 sec)
INFO:tensorflow:lr = 0.00013985454 (9.900 sec)
INFO:tensorflow:global_step/sec: 10.2464
INFO:tensorflow:loss = 4.9341116, step = 76000 (9.757 sec)
INFO:tensorflow:lr = 0.00013806253 (9.759 sec)
INFO:tensorflow:global_step/sec: 9.97606
INFO:tensorflow:loss = 4.783729, step = 76100 (10.026 sec)
INFO:tensorflow:lr = 0.00013627052 (10.025 sec)
INFO:tensorflow:global_step/sec: 9.91226
INFO:tensorflow:loss = 4.995345, step = 76200 (10.089 sec)
INFO:tensorflow:lr = 0.0001344785 (10.088 sec)
INFO:tensorflow:global_step/sec: 10.2615
INFO:tensorflow:loss = 4.944727, step = 76300 (9.743 sec)
INFO:tensorflow:lr = 0.00013268649 (9.743 sec)
INFO:tensorflow:global_step/sec: 9.99959
INFO:tensorflow:loss = 4.956012, step = 76400 (10.004 sec)
INFO:tensorflow:lr = 0.00013089448 (10.003 sec)
INFO:tensorflow:global_step/sec: 9.87114
INFO:tensorflow:loss = 4.8436537, step = 76500 (10.130 sec)
INFO:tensorflow:lr = 0.00012910247 (10.129 sec)
INFO:tensorflow:global_step/sec: 9.78018
INFO:tensorflow:loss = 5.0111704, step = 76600 (10.224 sec)
INFO:tensorflow:lr = 0.00012731046 (10.225 sec)
INFO:tensorflow:global_step/sec: 10.2486
INFO:tensorflow:loss = 5.05041, step = 76700 (9.757 sec)
INFO:tensorflow:lr = 0.00012551843 (9.757 sec)
INFO:tensorflow:global_step/sec: 10.0346
INFO:tensorflow:loss = 5.1064954, step = 76800 (9.966 sec)
INFO:tensorflow:lr = 0.00012372634 (9.966 sec)
INFO:tensorflow:global_step/sec: 10.0762
INFO:tensorflow:loss = 5.0424175, step = 76900 (9.926 sec)
INFO:tensorflow:lr = 0.000121934325 (9.926 sec)
INFO:tensorflow:global_step/sec: 10.1965
INFO:tensorflow:loss = 4.9432592, step = 77000 (9.806 sec)
INFO:tensorflow:lr = 0.000120142315 (9.806 sec)
INFO:tensorflow:global_step/sec: 10.2105
INFO:tensorflow:loss = 4.9125285, step = 77100 (9.795 sec)
INFO:tensorflow:lr = 0.0001183503 (9.794 sec)
INFO:tensorflow:global_step/sec: 9.99767
INFO:tensorflow:loss = 4.88361, step = 77200 (10.000 sec)
INFO:tensorflow:lr = 0.00011655829 (10.004 sec)
INFO:tensorflow:global_step/sec: 10.0162
INFO:tensorflow:loss = 4.9388685, step = 77300 (9.987 sec)
INFO:tensorflow:lr = 0.00011476627 (9.983 sec)
INFO:tensorflow:global_step/sec: 9.99087
INFO:tensorflow:loss = 4.856239, step = 77400 (10.007 sec)
INFO:tensorflow:lr = 0.00011297426 (10.008 sec)
INFO:tensorflow:global_step/sec: 10.0051
INFO:tensorflow:loss = 4.9585834, step = 77500 (9.995 sec)
INFO:tensorflow:lr = 0.00011118225 (9.995 sec)
INFO:tensorflow:global_step/sec: 9.89371
INFO:tensorflow:loss = 5.098746, step = 77600 (10.105 sec)
INFO:tensorflow:lr = 0.00010939023 (10.108 sec)
INFO:tensorflow:global_step/sec: 10.0591
INFO:tensorflow:loss = 5.009875, step = 77700 (9.944 sec)
INFO:tensorflow:lr = 0.000107598134 (9.942 sec)
INFO:tensorflow:global_step/sec: 9.96559
INFO:tensorflow:loss = 4.862642, step = 77800 (10.036 sec)
INFO:tensorflow:lr = 0.000105806124 (10.036 sec)
INFO:tensorflow:global_step/sec: 10.0664
INFO:tensorflow:loss = 5.0611267, step = 77900 (9.929 sec)
INFO:tensorflow:lr = 0.00010401411 (9.929 sec)
INFO:tensorflow:global_step/sec: 9.93765
INFO:tensorflow:loss = 5.0704823, step = 78000 (10.066 sec)
INFO:tensorflow:lr = 0.0001022221 (10.065 sec)
INFO:tensorflow:global_step/sec: 10.1168
INFO:tensorflow:loss = 4.9549403, step = 78100 (9.884 sec)
INFO:tensorflow:lr = 0.00010043008 (9.884 sec)
INFO:tensorflow:Saving checkpoints for 78125 into ../model/transformer_rnn/model.ckpt.
INFO:tensorflow:Loss for final step: 4.9947433.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/inputs/queues/feeding_queue_runner.py:62: QueueRunner.__init__ (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.
Instructions for updating:
To construct input pipelines, use the `tf.data` module.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/inputs/queues/feeding_functions.py:500: add_queue_runner (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.
Instructions for updating:
To construct input pipelines, use the `tf.data` module.
INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Entity <bound method TiedDense.call of <__main__.TiedDense object at 0x7f6e1a554ba8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING: Entity <bound method TiedDense.call of <__main__.TiedDense object at 0x7f6e1a554ba8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/contrib/seq2seq/python/ops/beam_search_decoder.py:971: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.cast` instead.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/contrib/seq2seq/python/ops/beam_search_decoder.py:1252: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/transformer_rnn/model.ckpt-78125
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py:882: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.
Instructions for updating:
To construct input pipelines, use the `tf.data` module.
------------
unit test
Q: 你 好
A1: 你 好
A2: 你 好 !
A3: 谢 谢

Q: 早 上 好
A1: 早 上 好
A2: 早 安
A3: 早 上 好 !

Q: 晚 上 好
A1: 晚 上 好
A2: 晚 安
A3: 晚 上 好 !

Q: 再 见
A1: 再 见
A2: 好 的
A3: 哈 哈

Q: 好 久 不 见
A1: 好 久 不 见
A2: 好 久 不 见 了
A3: 好 久 没 见 你 了

Q: 想 死 你 了
A1: 想 死 你 了
A2: 我 也 想 你
A3: 我 也 想 死

Q: 谢 谢 你
A1: 不 客 气
A2: 谢 谢
A3: 不 用 谢

Q: 爱 你
A1: 爱 你
A2: 我 也 爱 你
A3: 我 爱 你

Q: 你 好 厉 害 啊
A1: 好 厉 害
A2: 哈 哈
A3: 我 也 是

Q: 你 叫 什 么
A1: 你 猜
A2: 不 知 道
A3: 我 不 知 道

Q: 你 几 岁 了
A1: 你 猜 我 猜 不 猜
A2: 你 猜
A3: 你 猜 我 猜 你 猜 不 猜

Q: 现 在 几 点
A1: 1 2 点
A2: 十 点 半
A3: 不 知 道

Q: 今 天 天 气 怎 么 样
A1: 我 也 是
A2: 今 天 下 雨 了
A3: 我 也 是 这 么 想 的

Q: 你 今 天 心 情 好 吗
A1: 心 情 不 好
A2: 我 也 是
A3: 心 情 好

Q: 我 们 现 在 在 哪 里
A1: 在 哪 里
A2: 你 在 哪 里
A3: 在 哪 里 ?

Q: 讲 个 笑 话
A1: 笑 死 我 了
A2: 哈 哈
A3: 笑 死 了

Q: 你 会 几 种 语 言 呀
A1: 不 会 吧
A2: 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈
A3: 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈 哈

Q: 你 觉 得 我 帅 吗
A1: 我 觉 得 你 帅
A2: 我 也 觉 得
A3: 不 帅

Q: 讨 厌 的 周 一
A1: 我 也 是
A2: 我 也 讨 厌
A3: 我 也 喜 欢

Q: 好 烦 啊
A1: 我 也 是
A2: 好 烦 啊
A3: 怎 么 了

Q: 天 气 真 好
A1: 是 的
A2: 是 啊
A3: 嗯 嗯

Q: 今 天 好 冷
A1: 我 也 是
A2: 好 冷
A3: 好 冷 啊

Q: 今 天 好 热
A1: 我 也 是
A2: 我 也 是 !
A3: 今 天 下 雨 了

Q: 下 雨 了
A1: 下 雨 了
A2: 我 也 是
A3: 是 的

Q: 风 好 大
A1: 是 的
A2: 哈 哈
A3: 我 也 是

Q: 终 于 周 五 了
A1: 我 也 是
A2: 哈 哈
A3: 我 也 是 !

Q: 我 想 去 唱 歌
A1: 去 吧
A2: 我 也 想 去
A3: 来 啊 来 啊

------------
INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Entity <bound method TiedDense.call of <__main__.TiedDense object at 0x7f6e3a1e2978>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING: Entity <bound method TiedDense.call of <__main__.TiedDense object at 0x7f6e3a1e2978>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-08-28T07:38:35Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/transformer_rnn/model.ckpt-78125
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/test.txt
INFO:tensorflow:Finished evaluation at 2020-08-28-07:38:52
INFO:tensorflow:Saving dict for global step 78125: global_step = 78125, loss = 3.7486782
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 78125: ../model/transformer_rnn/model.ckpt-78125
Perplexity: 42.465
Best Perplexity: 42.465
INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Entity <bound method TiedDense.call of <__main__.TiedDense object at 0x7f6e394b3b70>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING: Entity <bound method TiedDense.call of <__main__.TiedDense object at 0x7f6e394b3b70>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
[<tf.Variable 'Embedding/fasttext_vectors:0' shape=(3043, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/query/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/query/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/key/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/key/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/value/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/value/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/output/kernel:0' shape=(512, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/attention/multihead_attention/output/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/output/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/output/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/fnn/conv1/kernel:0' shape=(300, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/fnn/conv1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/fnn/conv2/kernel:0' shape=(1200, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_0/fnn/conv2/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/query/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/query/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/key/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/key/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/value/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/value/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/output/kernel:0' shape=(512, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/attention/multihead_attention/output/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/output/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/output/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/fnn/conv1/kernel:0' shape=(300, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/fnn/conv1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/fnn/conv2/kernel:0' shape=(1200, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_1/fnn/conv2/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/query/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/query/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/key/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/key/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/value/kernel:0' shape=(300, 512) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/value/bias:0' shape=(512,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/output/kernel:0' shape=(512, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/attention/multihead_attention/output/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/output/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/output/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/fnn/conv1/kernel:0' shape=(300, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/fnn/conv1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/fnn/conv2/kernel:0' shape=(1200, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/layer_2/fnn/conv2/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/LayerNorm/beta:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Encoder/transformer_encoder/LayerNorm/gamma:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer/kernel:0' shape=(300, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel:0' shape=(300, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/attention_v:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/bias:0' shape=(3043,) dtype=float32_ref>]
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/transformer_rnn/model.ckpt-78125
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/saver.py:1069: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file utilities to get mtimes.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 78125 into ../model/transformer_rnn/model.ckpt.
Reading ../data/train.txt
INFO:tensorflow:loss = 4.951897, step = 78125
INFO:tensorflow:lr = 0.000100008925
INFO:tensorflow:global_step/sec: 8.39226
INFO:tensorflow:loss = 4.996364, step = 78225 (11.919 sec)
INFO:tensorflow:lr = 0.000100904974 (11.919 sec)
INFO:tensorflow:global_step/sec: 9.52345
INFO:tensorflow:loss = 4.9936595, step = 78325 (10.501 sec)
INFO:tensorflow:lr = 0.00010180094 (10.501 sec)
INFO:tensorflow:global_step/sec: 9.43821
INFO:tensorflow:loss = 4.963275, step = 78425 (10.596 sec)
INFO:tensorflow:lr = 0.00010269699 (10.596 sec)
INFO:tensorflow:global_step/sec: 9.66368
INFO:tensorflow:loss = 5.0328145, step = 78525 (10.345 sec)
INFO:tensorflow:lr = 0.00010359304 (10.345 sec)
INFO:tensorflow:global_step/sec: 9.51804
INFO:tensorflow:loss = 5.0326567, step = 78625 (10.509 sec)
INFO:tensorflow:lr = 0.000104489 (10.509 sec)
INFO:tensorflow:global_step/sec: 9.50996
INFO:tensorflow:loss = 4.865527, step = 78725 (10.512 sec)
INFO:tensorflow:lr = 0.00010538505 (10.516 sec)
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-14-24582a8cfd13> in <module>()
     24 
     25 while True:
---> 26   estimator.train(input_fn=lambda: dataset(is_training=True, params=params))
     27 
     28   unit_test(estimator)

/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py in train(self, input_fn, hooks, steps, max_steps, saving_listeners)
    368 
    369       saving_listeners = _check_listeners_type(saving_listeners)
--> 370       loss = self._train_model(input_fn, hooks, saving_listeners)
    371       logging.info('Loss for final step: %s.', loss)
    372       return self

/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py in _train_model(self, input_fn, hooks, saving_listeners)
   1159       return self._train_model_distributed(input_fn, hooks, saving_listeners)
   1160     else:
-> 1161       return self._train_model_default(input_fn, hooks, saving_listeners)
   1162 
   1163   def _train_model_default(self, input_fn, hooks, saving_listeners):

/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py in _train_model_default(self, input_fn, hooks, saving_listeners)
   1193       return self._train_with_estimator_spec(estimator_spec, worker_hooks,
   1194                                              hooks, global_step_tensor,
-> 1195                                              saving_listeners)
   1196 
   1197   def _train_model_distributed(self, input_fn, hooks, saving_listeners):

/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py in _train_with_estimator_spec(self, estimator_spec, worker_hooks, hooks, global_step_tensor, saving_listeners)
   1492       any_step_done = False
   1493       while not mon_sess.should_stop():
-> 1494         _, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
   1495         any_step_done = True
   1496     if not any_step_done:

/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py in run(self, fetches, feed_dict, options, run_metadata)
    752         feed_dict=feed_dict,
    753         options=options,
--> 754         run_metadata=run_metadata)
    755 
    756   def run_step_fn(self, step_fn):

/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py in run(self, fetches, feed_dict, options, run_metadata)
   1257             feed_dict=feed_dict,
   1258             options=options,
-> 1259             run_metadata=run_metadata)
   1260       except _PREEMPTION_ERRORS as e:
   1261         logging.info(

/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py in run(self, *args, **kwargs)
   1343   def run(self, *args, **kwargs):
   1344     try:
-> 1345       return self._sess.run(*args, **kwargs)
   1346     except _PREEMPTION_ERRORS:
   1347       raise

/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py in run(self, fetches, feed_dict, options, run_metadata)
   1416         feed_dict=feed_dict,
   1417         options=options,
-> 1418         run_metadata=run_metadata)
   1419 
   1420     for hook in self._hooks:

/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py in run(self, *args, **kwargs)
   1174 
   1175   def run(self, *args, **kwargs):
-> 1176     return self._sess.run(*args, **kwargs)
   1177 
   1178   def run_step_fn(self, step_fn, raw_session, run_with_hooks):

/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
    954     try:
    955       result = self._run(None, fetches, feed_dict, options_ptr,
--> 956                          run_metadata_ptr)
    957       if run_metadata:
    958         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
   1178     if final_fetches or final_targets or (handle and feed_dict_tensor):
   1179       results = self._do_run(handle, final_targets, final_fetches,
-> 1180                              feed_dict_tensor, options, run_metadata)
   1181     else:
   1182       results = []

/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1357     if handle is None:
   1358       return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1359                            run_metadata)
   1360     else:
   1361       return self._do_call(_prun_fn, handle, feeds, fetches)

/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py in _do_call(self, fn, *args)
   1363   def _do_call(self, fn, *args):
   1364     try:
-> 1365       return fn(*args)
   1366     except errors.OpError as e:
   1367       message = compat.as_text(e.message)

/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
   1348       self._extend_graph()
   1349       return self._call_tf_sessionrun(options, feed_dict, fetch_list,
-> 1350                                       target_list, run_metadata)
   1351 
   1352     def _prun_fn(handle, feed_dict, fetch_list):

/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
   1441     return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
   1442                                             fetch_list, target_list,
-> 1443                                             run_metadata)
   1444 
   1445   def _call_tf_sessionprun(self, handle, feed_dict, fetch_list):

KeyboardInterrupt: