In [0]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need these
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/question_answering/babi/main')
In [0]:
import tensorflow as tf
print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
import pprint
import logging
import numpy as np
    
from copy import deepcopy
from pathlib import Path
from attn_gru_cell import AttentionGRUCell
TensorFlow Version 1.13.0-rc1
GPU Enabled: True
In [0]:
def babi_data_load(path):
    i = []
    q = []
    a = []

    i_len = []
    i_sent_len = []
    q_len = []

    for line in open(path):
        line = line.lower().rstrip()
        
        index = line.split()[0]
        if index == '1':
            fact = []
        
        if '?' in line:
            temp = line.split('\t')
            _q = temp[0].replace('?', '').split()[1:]
            _a = temp[1]
            fact_copied = deepcopy(fact)
            i.append(fact_copied)
            q.append(_q)
            a.append(_a)

            i_len.append(len(fact_copied))
            i_sent_len.append([len(s) for s in fact_copied])
            q_len.append(len(_q))
        else:
            tokens = line.replace('.', '').split()[1:]
            fact.append(tokens)
    
    return (i, q, a), (i_len, i_sent_len, q_len)
In [0]:
# stream data from text files
def gen_fn(f_path):
  print('Reading', f_path)
  (i, q, a), (i_len, _, q_len) = babi_data_load(f_path)
  for (_i, _q, _a, _i_len, _q_len) in zip(i, q, a, i_len, q_len):
      _i = [sent + ['<pad>'] * (params['max_sent_len'] - len(sent)) for sent in _i]
      yield (_i, _q, _i_len, _q_len), _a
      

def input_fn(mode, params):
  _shapes = (([None, params['max_sent_len']], [None,], (), ()), ())
  _types = ((tf.string, tf.string, tf.int32, tf.int32), tf.string)
  _pads = (('<pad>', '<pad>', -1, -1), '<pad>')
  
  if mode == tf.estimator.ModeKeys.TRAIN:
    ds = tf.data.Dataset.from_generator(
      lambda: gen_fn(params['train_path']),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.repeat()
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
  
  if mode == tf.estimator.ModeKeys.EVAL:
    ds = tf.data.Dataset.from_generator(
      lambda: gen_fn(params['test_path']),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
  
  return ds
In [0]:
def position_encoding(sent_size, embed_size):
    encoding = np.ones((embed_size, sent_size), dtype=np.float32)
    ls = sent_size + 1
    le = embed_size + 1
    for i in range(1, le):
        for j in range(1, ls):
            encoding[i-1, j-1] = (i - (le-1)/2) * (j - (ls-1)/2)
    encoding = 1 + 4 * encoding / embed_size / sent_size
    return tf.convert_to_tensor(np.transpose(encoding))
  

def gru(rnn_size):
    return tf.nn.rnn_cell.GRUCell(
        rnn_size, kernel_initializer=tf.initializers.orthogonal())
In [0]:
def embedding_module(params):
  return tf.get_variable('lookup', (params['vocab_size']+1, params['embed_dim']), tf.float32)


def input_module(features, vocab, embedding, params, is_training):
  inputs, _, inputs_len, _ = features
  inputs = vocab.lookup(inputs)
  
  inputs = tf.nn.embedding_lookup(embedding, inputs)
  position = position_encoding(params['max_sent_len'], params['embed_dim'])
  inputs = tf.reduce_sum(inputs * position, 2)                          
  o, _ = tf.nn.bidirectional_dynamic_rnn(gru(params['hidden_size']//2),
                                         gru(params['hidden_size']//2),
                                         inputs,
                                         inputs_len,
                                         dtype=tf.float32)
  fact_vecs = tf.concat(o, -1)                                   
  fact_vecs = tf.layers.dropout(fact_vecs, params['dropout_rate'], training=is_training)
  return fact_vecs


def question_module(features, vocab, embedding):
  _, questions, _, questions_len = features
  questions = vocab.lookup(questions)

  questions = tf.nn.embedding_lookup(embedding, questions)
  _, state = tf.nn.dynamic_rnn(gru(params['hidden_size']),
                               questions,
                               questions_len,
                               dtype=tf.float32)

  return state
  
  
def memory_module(features, fact_vecs, q_vec, params, is_training):
  proj_1 = tf.layers.Dense(params['embed_dim'], tf.tanh, name='attn_proj_1')
  proj_2 = tf.layers.Dense(1, name='attn_proj_2')
  attn_gru = AttentionGRUCell(params['hidden_size'], name='attn_gru')
  memory_proj = tf.layers.Dense(params['hidden_size'], tf.nn.relu, name='memory_proj')

  memory = q_vec
  for i in range(params['num_hops']):
      print('==> Memory Episode', i+1)
      episode = gen_episode(features,
                            memory,
                            q_vec,
                            fact_vecs,
                            proj_1,
                            proj_2,
                            attn_gru,
                            is_training)
      memory = memory_proj(tf.concat([memory, episode, q_vec], 1))

  return memory


def gen_episode(features, memory, q_vec, fact_vecs, proj_1, proj_2, attn_gru, is_training):
  def gen_attn(fact_vec):
      features = [fact_vec * q_vec,
                  fact_vec * memory,
                  tf.abs(fact_vec - q_vec),
                  tf.abs(fact_vec - memory)]
      feature_vec = tf.concat(features, 1)
      attention = proj_1(feature_vec)
      attention = proj_2(attention)
      return tf.squeeze(attention, 1)

  _, _, inputs_len, _ = features  
    
  # Gates (attentions) are activated, if sentence relevant to the question or memory
  attns = tf.map_fn(gen_attn, tf.transpose(fact_vecs, [1,0,2]))
  attns = tf.transpose(attns)                                      # (B, n_fact)
  attns = tf.nn.softmax(attns)                                     # (B, n_fact)
  attns = tf.expand_dims(attns, -1)                                # (B, n_fact, 1)

  # The relevant facts are summarized in another GRU
  _, episode = tf.nn.dynamic_rnn(attn_gru,
                                 tf.concat([fact_vecs, attns], 2), # (B, n_fact, D+1)
                                 inputs_len,
                                 dtype=tf.float32)
  return episode


def answer_module(memory, q_vec, params, is_training):
    memory = tf.layers.dropout(memory, params['dropout_rate'], training=is_training)
    output = tf.layers.dense(tf.concat((memory, q_vec), 1), params['vocab_size']+1)
    return output
In [0]:
def graph_fn(features, vocab, params, mode):
  is_training = (mode == tf.estimator.ModeKeys.TRAIN)

  with tf.variable_scope('word_embedding'):
    embedding = embedding_module(params)

  with tf.variable_scope('input_module'):
    fact_vecs = input_module(features, vocab, embedding, params, is_training)
  
  with tf.variable_scope('question_module'):
    q_vec = question_module(features, vocab, embedding)

  with tf.variable_scope('memory_module'):
    memory = memory_module(features, fact_vecs, q_vec, params, is_training)

  with tf.variable_scope('answer_module'):
    logits = answer_module(memory, q_vec, params, is_training)

  return logits
In [0]:
def model_fn(features, labels, mode, params):
    vocab = tf.contrib.lookup.index_table_from_file(
      params['vocab_path'], num_oov_buckets=1)
    labels = vocab.lookup(labels)
  
  
    logits = graph_fn(features, vocab, params, mode)
    
    
    if mode == tf.estimator.ModeKeys.PREDICT:
      return tf.estimator.EstimatorSpec(mode=mode,
                                        predictions=tf.argmax(logits, -1))
    else:
      loss_op = tf.nn.sparse_softmax_cross_entropy_with_logits(
        labels=labels, logits=logits)
      loss_op = tf.reduce_mean(loss_op)

      
    if mode == tf.estimator.ModeKeys.TRAIN:
      variables = tf.trainable_variables()
      tf.logging.info('\n'+pprint.pformat(variables))

      grads = tf.gradients(loss_op, variables)
      grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])

      global_step=tf.train.get_or_create_global_step()

      optim = tf.train.AdamOptimizer()

      train_op = optim.apply_gradients(
        zip(grads, variables), global_step=global_step)

      return tf.estimator.EstimatorSpec(mode=mode,
                                        loss=loss_op,
                                        train_op=train_op,)

    
    if mode == tf.estimator.ModeKeys.EVAL:
      acc_op = tf.metrics.accuracy(labels=labels,
                                   predictions=tf.argmax(logits, -1))

      return tf.estimator.EstimatorSpec(mode=mode,
                                        loss=loss_op,
                                        eval_metric_ops={'acc': acc_op})
In [0]:
params ={
    'model_dir': '../model/dmn',
    'log_path': '../log/dmn.txt',
    'vocab_path': '../vocab/word.txt',
    'train_path': '../data/qa5_three-arg-relations_train.txt',
    'test_path': '../data/qa5_three-arg-relations_test.txt',
    'vocab_size': 40,
    'batch_size': 100,
    'embed_dim': 80,
    'hidden_size': 80,
    'dropout_rate': 0.1,
    'num_hops': 2,
    'clip_norm': 5.0,
    'num_patience': 3,
}
In [0]:
"""
Show One Instance Of Data
"""
(i, q, a), (i_len, i_sent_len, q_len) = babi_data_load('../data/qa5_three-arg-relations_train.txt')
params['num_samples'] = len(i)
params['max_sent_len'] = max([sent_len for example in i_sent_len for sent_len in example])

print('Context:')
pprint.pprint(i[21])
print('\nQuestion:')
print(q[21])
print('\nAnswer:')
print(a[21])
print('\nContext Overall Length:')
print(i_len[21])
print('\nContext Each Sentence Length:')
print(i_sent_len[21])
print('\nQuestion Length:')
print(q_len[21])
Context:
[['mary', 'grabbed', 'the', 'milk', 'there'],
 ['jeff', 'travelled', 'to', 'the', 'hallway'],
 ['mary', 'handed', 'the', 'milk', 'to', 'jeff'],
 ['jeff', 'discarded', 'the', 'milk', 'there'],
 ['mary', 'journeyed', 'to', 'the', 'kitchen'],
 ['jeff', 'journeyed', 'to', 'the', 'garden']]

Question:
['who', 'gave', 'the', 'milk']

Answer:
mary

Context Overall Length:
6

Context Each Sentence Length:
[5, 5, 6, 5, 5, 5]

Question Length:
4
In [0]:
# Create directory if not exist
Path(os.path.dirname(params['log_path'])).mkdir(exist_ok=True)
Path(params['model_dir']).mkdir(exist_ok=True, parents=True)

# Logging
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(params['log_path'])
logger.addHandler(fh)

# Create an estimator
_eval_steps = 2*params['num_samples']//params['batch_size']
config = tf.estimator.RunConfig(
  save_checkpoints_steps=_eval_steps,
  keep_checkpoint_max=params['num_patience']+2,)

estimator = tf.estimator.Estimator(
  model_fn=model_fn,
  model_dir=params['model_dir'],
  config=config,
  params=params)

# This hook early-stops model if testing accuracy is not improved
hook = tf.contrib.estimator.stop_if_no_increase_hook(
  estimator=estimator,
  metric_name='acc',
  max_steps_without_increase=params['num_patience']*_eval_steps,
  run_every_secs=None,
  run_every_steps=_eval_steps)

# Train on training data and Evaluate on testing data
train_spec = tf.estimator.TrainSpec(
  input_fn=lambda: input_fn(mode=tf.estimator.ModeKeys.TRAIN,
                            params=params),
  hooks=[hook])

eval_spec = tf.estimator.EvalSpec(
  input_fn=lambda: input_fn(mode=tf.estimator.ModeKeys.EVAL,
                            params=params),
  steps=None,
  throttle_secs=10,)

tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
INFO:tensorflow:Using config: {'_model_dir': '../model/dmn', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 200, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true
graph_options {
  rewrite_options {
    meta_optimizer_iterations: ONE
  }
}
, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fd73cddd438>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}

WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
If you depend on functionality not listed there, please file an issue.

INFO:tensorflow:Not using Distribute Coordinator.
INFO:tensorflow:Running training and evaluation locally (non-distributed).
INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 200 or save_checkpoints_secs None.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:429: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, use
    tf.py_function, which takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/util.py:104: DatasetV1.make_initializable_iterator (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use `for ... in dataset:` to iterate over a dataset. If using `tf.estimator`, return the `Dataset` object directly from your input function. As a last resort, you can use `tf.compat.v1.data.make_initializable_iterator(dataset)`.
INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:From <ipython-input-5-d91de2831218>:14: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.
WARNING:tensorflow:From <ipython-input-6-e2247cd79295>:16: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From <ipython-input-6-e2247cd79295>:18: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dropout instead.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
==> Memory Episode 1
==> Memory Episode 2
WARNING:tensorflow:From <ipython-input-6-e2247cd79295>:86: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dense instead.
INFO:tensorflow:
[<tf.Variable 'word_embedding/lookup:0' shape=(41, 80) dtype=float32_ref>,
 <tf.Variable 'input_module/bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(120, 80) dtype=float32_ref>,
 <tf.Variable 'input_module/bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(80,) dtype=float32_ref>,
 <tf.Variable 'input_module/bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(120, 40) dtype=float32_ref>,
 <tf.Variable 'input_module/bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(40,) dtype=float32_ref>,
 <tf.Variable 'input_module/bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(120, 80) dtype=float32_ref>,
 <tf.Variable 'input_module/bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(80,) dtype=float32_ref>,
 <tf.Variable 'input_module/bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(120, 40) dtype=float32_ref>,
 <tf.Variable 'input_module/bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(40,) dtype=float32_ref>,
 <tf.Variable 'question_module/rnn/gru_cell/gates/kernel:0' shape=(160, 160) dtype=float32_ref>,
 <tf.Variable 'question_module/rnn/gru_cell/gates/bias:0' shape=(160,) dtype=float32_ref>,
 <tf.Variable 'question_module/rnn/gru_cell/candidate/kernel:0' shape=(160, 80) dtype=float32_ref>,
 <tf.Variable 'question_module/rnn/gru_cell/candidate/bias:0' shape=(80,) dtype=float32_ref>,
 <tf.Variable 'memory_module/attn_proj_1/kernel:0' shape=(320, 80) dtype=float32_ref>,
 <tf.Variable 'memory_module/attn_proj_1/bias:0' shape=(80,) dtype=float32_ref>,
 <tf.Variable 'memory_module/attn_proj_2/kernel:0' shape=(80, 1) dtype=float32_ref>,
 <tf.Variable 'memory_module/attn_proj_2/bias:0' shape=(1,) dtype=float32_ref>,
 <tf.Variable 'memory_module/rnn/attn_gru/gates/kernel:0' shape=(160, 80) dtype=float32_ref>,
 <tf.Variable 'memory_module/rnn/attn_gru/gates/bias:0' shape=(80,) dtype=float32_ref>,
 <tf.Variable 'memory_module/rnn/attn_gru/candidate/kernel:0' shape=(160, 80) dtype=float32_ref>,
 <tf.Variable 'memory_module/rnn/attn_gru/candidate/bias:0' shape=(80,) dtype=float32_ref>,
 <tf.Variable 'memory_module/memory_proj/kernel:0' shape=(240, 80) dtype=float32_ref>,
 <tf.Variable 'memory_module/memory_proj/bias:0' shape=(80,) dtype=float32_ref>,
 <tf.Variable 'answer_module/dense/kernel:0' shape=(160, 41) dtype=float32_ref>,
 <tf.Variable 'answer_module/dense/bias:0' shape=(41,) dtype=float32_ref>]
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 0 into ../model/dmn/model.ckpt.
Reading ../data/qa5_three-arg-relations_train.txt
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:loss = 3.7059402, step = 0
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.75133
INFO:tensorflow:loss = 1.3650131, step = 100 (57.105 sec)
INFO:tensorflow:Saving checkpoints for 200 into ../model/dmn/model.ckpt.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T11:54:24Z
INFO:tensorflow:Graph was finalized.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to check for files with this prefix.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-200
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-11:54:27
INFO:tensorflow:Saving dict for global step 200: acc = 0.782, global_step = 200, loss = 0.6338689
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 200: ../model/dmn/model.ckpt-200
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.61566
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/summary/summary_iterator.py:68: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.
Instructions for updating:
Use eager execution and: 
`tf.data.TFRecordDataset(path)`
INFO:tensorflow:loss = 0.7098075, step = 200 (61.914 sec)
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.75428
INFO:tensorflow:loss = 0.26251137, step = 300 (56.981 sec)
INFO:tensorflow:Saving checkpoints for 400 into ../model/dmn/model.ckpt.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T11:56:20Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-400
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-11:56:23
INFO:tensorflow:Saving dict for global step 400: acc = 0.843, global_step = 400, loss = 0.27265805
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 400: ../model/dmn/model.ckpt-400
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.69731
INFO:tensorflow:loss = 0.18384781, step = 400 (58.936 sec)
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.80313
INFO:tensorflow:loss = 0.22520888, step = 500 (55.439 sec)
INFO:tensorflow:Saving checkpoints for 600 into ../model/dmn/model.ckpt.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T11:58:17Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-600
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-11:58:20
INFO:tensorflow:Saving dict for global step 600: acc = 0.834, global_step = 600, loss = 0.25944102
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 600: ../model/dmn/model.ckpt-600
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.62637
INFO:tensorflow:loss = 0.41380847, step = 600 (61.503 sec)
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.78944
INFO:tensorflow:loss = 0.19449098, step = 700 (55.872 sec)
INFO:tensorflow:Saving checkpoints for 800 into ../model/dmn/model.ckpt.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T12:00:14Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-800
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-12:00:17
INFO:tensorflow:Saving dict for global step 800: acc = 0.832, global_step = 800, loss = 0.2575265
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 800: ../model/dmn/model.ckpt-800
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.62341
INFO:tensorflow:loss = 0.26415473, step = 800 (61.611 sec)
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.78614
INFO:tensorflow:loss = 0.22967395, step = 900 (55.973 sec)
INFO:tensorflow:Saving checkpoints for 1000 into ../model/dmn/model.ckpt.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T12:02:11Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-1000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-12:02:14
INFO:tensorflow:Saving dict for global step 1000: acc = 0.891, global_step = 1000, loss = 0.24205628
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1000: ../model/dmn/model.ckpt-1000
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.64112
INFO:tensorflow:loss = 0.17308657, step = 1000 (60.945 sec)
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.78743
INFO:tensorflow:loss = 0.12043685, step = 1100 (55.934 sec)
INFO:tensorflow:Saving checkpoints for 1200 into ../model/dmn/model.ckpt.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T12:04:09Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-1200
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-12:04:12
INFO:tensorflow:Saving dict for global step 1200: acc = 0.989, global_step = 1200, loss = 0.09721884
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1200: ../model/dmn/model.ckpt-1200
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.62864
INFO:tensorflow:loss = 0.04999805, step = 1200 (61.415 sec)
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.76553
INFO:tensorflow:loss = 0.030324563, step = 1300 (56.624 sec)
INFO:tensorflow:Saving checkpoints for 1400 into ../model/dmn/model.ckpt.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T12:06:06Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-1400
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-12:06:09
INFO:tensorflow:Saving dict for global step 1400: acc = 0.994, global_step = 1400, loss = 0.046021726
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1400: ../model/dmn/model.ckpt-1400
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.64764
INFO:tensorflow:loss = 0.04744132, step = 1400 (60.709 sec)
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.77611
INFO:tensorflow:loss = 0.0075320965, step = 1500 (56.292 sec)
INFO:tensorflow:Saving checkpoints for 1600 into ../model/dmn/model.ckpt.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T12:08:04Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-1600
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-12:08:07
INFO:tensorflow:Saving dict for global step 1600: acc = 0.994, global_step = 1600, loss = 0.047745097
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1600: ../model/dmn/model.ckpt-1600
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.61234
INFO:tensorflow:loss = 0.015278418, step = 1600 (62.033 sec)
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.7599
INFO:tensorflow:loss = 0.0033423228, step = 1700 (56.804 sec)
INFO:tensorflow:Saving checkpoints for 1800 into ../model/dmn/model.ckpt.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T12:10:03Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-1800
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-12:10:07
INFO:tensorflow:Saving dict for global step 1800: acc = 0.989, global_step = 1800, loss = 0.06317021
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1800: ../model/dmn/model.ckpt-1800
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.60111
INFO:tensorflow:loss = 0.00329536, step = 1800 (62.473 sec)
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.77323
INFO:tensorflow:loss = 0.0027753857, step = 1900 (56.382 sec)
INFO:tensorflow:Saving checkpoints for 2000 into ../model/dmn/model.ckpt.
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T12:12:02Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-2000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-12:12:06
INFO:tensorflow:Saving dict for global step 2000: acc = 0.993, global_step = 2000, loss = 0.057220567
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2000: ../model/dmn/model.ckpt-2000
Reading ../data/qa5_three-arg-relations_train.txt
INFO:tensorflow:global_step/sec: 1.59384
INFO:tensorflow:No increase in metric "acc" for 600 steps, which is greater than or equal to max steps (600) configured for early stopping.
INFO:tensorflow:Requesting early stopping at global step 2000
INFO:tensorflow:loss = 0.0014298395, step = 2000 (62.832 sec)
INFO:tensorflow:Saving checkpoints for 2001 into ../model/dmn/model.ckpt.
INFO:tensorflow:Skip the current checkpoint eval due to throttle secs (10 secs).
INFO:tensorflow:Calling model_fn.
==> Memory Episode 1
==> Memory Episode 2
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-13T12:12:10Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-2001
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/qa5_three-arg-relations_test.txt
INFO:tensorflow:Finished evaluation at 2019-02-13-12:12:13
INFO:tensorflow:Saving dict for global step 2001: acc = 0.993, global_step = 2001, loss = 0.05458973
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2001: ../model/dmn/model.ckpt-2001
INFO:tensorflow:Loss for final step: 0.0014298395.
Out[0]:
({'acc': 0.993, 'global_step': 2001, 'loss': 0.05458973}, [])