In [1]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need this cell
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/question_answering/babi/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
import tensorflow as tf
print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
import numpy as np
import os

from pathlib import Path
from attn_gru_cell import AttentionGRUCell
TensorFlow Version 1.13.0-rc1
GPU Enabled: True
In [0]:
def position_encoding(sent_size, embed_size):
    encoding = np.ones((embed_size, sent_size), dtype=np.float32)
    ls = sent_size + 1
    le = embed_size + 1
    for i in range(1, le):
        for j in range(1, ls):
            encoding[i-1, j-1] = (i - (le-1)/2) * (j - (ls-1)/2)
    encoding = 1 + 4 * encoding / embed_size / sent_size
    return tf.convert_to_tensor(np.transpose(encoding))

  
def gru(rnn_size):
    return tf.nn.rnn_cell.GRUCell(
        rnn_size, kernel_initializer=tf.initializers.orthogonal())

  
def embedding_module(params):
  return tf.get_variable('lookup', (params['vocab_size']+1, params['embed_dim']), tf.float32)


def input_module(features, vocab, embedding, params, is_training):
  if isinstance(features, dict):
    inputs = features['inputs']
    inputs_len = features['inputs_len']
  else:
    inputs, _, inputs_len, _ = features
  
  inputs = vocab.lookup(inputs)
  
  inputs = tf.nn.embedding_lookup(embedding, inputs)
  position = position_encoding(params['max_sent_len'], params['embed_dim'])
  inputs = tf.reduce_sum(inputs * position, 2)                          
  o, _ = tf.nn.bidirectional_dynamic_rnn(gru(params['hidden_size']//2),
                                         gru(params['hidden_size']//2),
                                         inputs,
                                         inputs_len,
                                         dtype=tf.float32)
  fact_vecs = tf.concat(o, -1)                                   
  fact_vecs = tf.layers.dropout(fact_vecs, params['dropout_rate'], training=is_training)
  return fact_vecs


def question_module(features, vocab, embedding):
  if isinstance(features, dict):
    questions = features['questions']
    questions_len = features['questions_len']
  else:
    _, questions, _, questions_len = features
  
  questions = vocab.lookup(questions)

  questions = tf.nn.embedding_lookup(embedding, questions)
  _, state = tf.nn.dynamic_rnn(gru(params['hidden_size']),
                               questions,
                               questions_len,
                               dtype=tf.float32)

  return state
  
  
def memory_module(features, fact_vecs, q_vec, params, is_training):
  proj_1 = tf.layers.Dense(params['embed_dim'], tf.tanh, name='attn_proj_1')
  proj_2 = tf.layers.Dense(1, name='attn_proj_2')
  attn_gru = AttentionGRUCell(params['hidden_size'], name='attn_gru')
  memory_proj = tf.layers.Dense(params['hidden_size'], tf.nn.relu, name='memory_proj')

  memory = q_vec
  for i in range(params['num_hops']):
      print('==> Memory Episode', i+1)
      episode = gen_episode(features,
                            memory,
                            q_vec,
                            fact_vecs,
                            proj_1,
                            proj_2,
                            attn_gru,
                            is_training)
      memory = memory_proj(tf.concat([memory, episode, q_vec], 1))

  return memory


def gen_episode(features, memory, q_vec, fact_vecs, proj_1, proj_2, attn_gru, is_training):
  def gen_attn(fact_vec):
      features = [fact_vec * q_vec,
                  fact_vec * memory,
                  tf.abs(fact_vec - q_vec),
                  tf.abs(fact_vec - memory)]
      feature_vec = tf.concat(features, 1)
      attention = proj_1(feature_vec)
      attention = proj_2(attention)
      return tf.squeeze(attention, 1)

  if isinstance(features, dict):
    inputs_len = features['inputs_len']
  else:
    _, _, inputs_len, _ = features 
    
  # Gates (attentions) are activated, if sentence relevant to the question or memory
  attns = tf.map_fn(gen_attn, tf.transpose(fact_vecs, [1,0,2]))
  attns = tf.transpose(attns)                                      # (B, n_fact)
  attns = tf.nn.softmax(attns)                                     # (B, n_fact)
  attns = tf.expand_dims(attns, -1)                                # (B, n_fact, 1)

  # The relevant facts are summarized in another GRU
  _, episode = tf.nn.dynamic_rnn(attn_gru,
                                 tf.concat([fact_vecs, attns], 2), # (B, n_fact, D+1)
                                 inputs_len,
                                 dtype=tf.float32)
  return episode


def answer_module(memory, q_vec, params, is_training):
    memory = tf.layers.dropout(memory, params['dropout_rate'], training=is_training)
    output = tf.layers.dense(tf.concat((memory, q_vec), 1), params['vocab_size']+1)
    return output
In [0]:
def graph_fn(features, vocab, params, mode):
  is_training = (mode == tf.estimator.ModeKeys.TRAIN)

  with tf.variable_scope('word_embedding'):
    embedding = embedding_module(params)

  with tf.variable_scope('input_module'):
    fact_vecs = input_module(features, vocab, embedding, params, is_training)
  
  with tf.variable_scope('question_module'):
    q_vec = question_module(features, vocab, embedding)

  with tf.variable_scope('memory_module'):
    memory = memory_module(features, fact_vecs, q_vec, params, is_training)

  with tf.variable_scope('answer_module'):
    logits = answer_module(memory, q_vec, params, is_training)

  return logits
In [0]:
def model_fn(features, labels, mode, params):
    vocab = tf.contrib.lookup.index_table_from_file(
      params['vocab_path'], num_oov_buckets=1)
  
    logits = graph_fn(features, vocab, params, mode)
    
    if mode == tf.estimator.ModeKeys.PREDICT:
      vocab_rev = tf.contrib.lookup.index_to_string_table_from_file(params['vocab_path'])
      predictions = tf.argmax(logits, -1)
      predictions = vocab_rev.lookup(predictions)
      return tf.estimator.EstimatorSpec(mode=mode,
                                        predictions=predictions,)
In [0]:
params ={
    'model_dir': '../model/dmn',
    'export_dir': '../model/dmn_export',
    'log_path': '../log/dmn.txt',
    'vocab_path': '../vocab/word.txt',
    'max_sent_len': 6,
    'vocab_size': 40,
    'batch_size': 100,
    'embed_dim': 80,
    'hidden_size': 80,
    'dropout_rate': 0.1,
    'num_hops': 2,
}
In [0]:
def serving_input_receiver_fn():
    inputs = tf.placeholder(tf.string, [None, None, None], 'inputs')
    questions = tf.placeholder(tf.string, [None, None], 'questions')
    inputs_len = tf.placeholder(tf.int32, [None], 'inputs_len')
    questions_len = tf.placeholder(tf.int32, [None], 'questions_len')
    
    features = {'inputs': inputs,
                'inputs_len': inputs_len,
                'questions': questions,
                'questions_len': questions_len,}
    receiver_tensors = features
    
    return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
In [8]:
estimator = tf.estimator.Estimator(model_fn, params['model_dir'], params=params)
estimator.export_saved_model(params['export_dir'], serving_input_receiver_fn)
INFO:tensorflow:Using default config.
INFO:tensorflow:Using config: {'_model_dir': '../model/dmn', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true
graph_options {
  rewrite_options {
    meta_optimizer_iterations: ONE
  }
}
, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fe86c45c908>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
INFO:tensorflow:Calling model_fn.

WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
If you depend on functionality not listed there, please file an issue.

WARNING:tensorflow:From <ipython-input-3-d0dff6cd0e01>:14: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.
WARNING:tensorflow:From <ipython-input-3-d0dff6cd0e01>:37: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From <ipython-input-3-d0dff6cd0e01>:39: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dropout instead.
==> Memory Episode 1
==> Memory Episode 2
WARNING:tensorflow:From <ipython-input-3-d0dff6cd0e01>:115: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dense instead.
INFO:tensorflow:Done calling model_fn.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/saved_model/signature_def_utils_impl.py:205: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.
INFO:tensorflow:Signatures INCLUDED in export for Classify: None
INFO:tensorflow:Signatures INCLUDED in export for Regress: None
INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default']
INFO:tensorflow:Signatures INCLUDED in export for Train: None
INFO:tensorflow:Signatures INCLUDED in export for Eval: None
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to check for files with this prefix.
INFO:tensorflow:Restoring parameters from ../model/dmn/model.ckpt-2001
INFO:tensorflow:Assets added to graph.
INFO:tensorflow:Assets written to: ../model/dmn_export/temp-b'1550120865'/assets
INFO:tensorflow:SavedModel written to: ../model/dmn_export/temp-b'1550120865'/saved_model.pb
Out[8]:
b'../model/dmn_export/1550120865'
In [9]:
inputs_example = [[
    ['fred', 'picked', 'up', 'the', 'football', 'there'],
    ['fred', 'gave', 'the', 'football', 'to', 'jev'],
]]
inputs_len_example = [2]
questions_example = [['what', 'did', 'fred', 'give', 'to', 'jeff']]
questions_len_example = [6]


subdirs = [x for x in Path(params['export_dir']).iterdir()
           if x.is_dir() and 'temp' not in str(x)]
latest = str(sorted(subdirs)[-1])
  
  
predict_fn = tf.contrib.predictor.from_saved_model(latest)
predictions = predict_fn(
  {'inputs': inputs_example,
   'inputs_len': inputs_len_example,
   'questions': questions_example,
   'questions_len': questions_len_example,})

print('Output:', predictions['output'][0])
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/predictor/saved_model_predictor.py:153: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.
INFO:tensorflow:Restoring parameters from ../model/dmn_export/1550120865/variables/variables
Output: b'football'