In [1]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/semantic_parsing/tree_slu/main')
Mounted at /content/gdrive
In [2]:
%tensorflow_version 2.x
!pip install -U tensorflow-addons
Collecting tensorflow-addons
  Downloading https://files.pythonhosted.org/packages/b3/f8/d6fca180c123f2851035c4493690662ebdad0849a9059d56035434bff5c9/tensorflow_addons-0.11.2-cp36-cp36m-manylinux2010_x86_64.whl (1.1MB)
     |████████████████████████████████| 1.1MB 4.7MB/s 
Requirement already satisfied, skipping upgrade: typeguard>=2.7 in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons) (2.7.1)
Installing collected packages: tensorflow-addons
  Found existing installation: tensorflow-addons 0.8.3
    Uninstalling tensorflow-addons-0.8.3:
      Successfully uninstalled tensorflow-addons-0.8.3
Successfully installed tensorflow-addons-0.11.2
In [3]:
from modified_beam_search_decoder import BeamSearchDecoder

import tensorflow as tf
import tensorflow_addons as tfa

import numpy as np
import pprint
import logging
import time
import nltk
import string

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.3.0
WARNING:tensorflow:From <ipython-input-3-8d74c55ef9c5>:14: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU Enabled: True
In [4]:
def get_idx(symbol2idx, symbol):
  return symbol2idx.get(symbol, len(symbol2idx))

# stream data from text files
def data_generator(f_path, params):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      text_raw, text_tokenized, label = line.split('\t')
      text_tokenized = text_tokenized.lower().split()
      label = label.replace('[', '[ ').lower().split()
      
      source = [params['tgt2idx'].get(w, len(params['tgt2idx'])) for w in text_tokenized]
      target = [params['tgt2idx'].get(w, len(params['tgt2idx'])) for w in label]
      target_in = [1] + target
      target_out = target + [2]

      max_char_len = max([len(w) for w in text_tokenized])
      chars = []
      for w in text_tokenized:
        temp = []
        for c in list(w):
          temp.append(get_idx(params['char2idx'], c))
        if len(temp) < max_char_len:
          temp += [0] * (max_char_len - len(temp))
        else:
          temp = temp[:max_char_len]
        chars.append(temp)

      yield (source, chars, target_in, target_out)
In [5]:
def dataset(is_training, params):
  _shapes = ([None], [None, None], [None], [None],)
  _types = (tf.int32, tf.int32, tf.int32, tf.int32)
  _pads = (0, 0, 0, 0)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['buffer_size'])
    ds = ds.padded_batch(params['train_batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['eval_batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [6]:
class Embed(tf.keras.Model):
  def __init__(self):
    super().__init__()
    self.embedding = tf.Variable(np.load('../vocab/word.npy'),
                                 dtype=tf.float32,
                                 name='pretrained_embedding')
  
  def call(self, inputs):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    return x
In [7]:
class Encoder(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])

    self.char_cnn = tf.keras.layers.Conv1D(
      filters=params['cnn_filters'], kernel_size=params['cnn_kernel_size'], padding='valid', activation=params['activation'])
    
    self.embed_fc = tf.keras.layers.Dense(params['rnn_units'], params['activation'], name='embed_fc')

    self.birnn = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(
      params['rnn_units'], return_state=True, return_sequences=True, zero_output_for_mask=True))

    self.state_fc = tf.keras.layers.Dense(params['rnn_units'], params['activation'], name='state_fc')

    self.out_fc = tf.keras.layers.Dense(params['rnn_units'], params['activation'], name='out_fc')
  
  def call(self, inputs, mask, training):
    rnn_mask = mask
    if rnn_mask.dtype != tf.bool:
      rnn_mask = tf.cast(rnn_mask, tf.bool)
    words, chars = inputs
    batch_sz = tf.shape(words)[0]
    word_len = tf.shape(words)[1]
    char_len = tf.shape(chars)[2]

    chars = tf.reshape(chars, (batch_sz*word_len, char_len, params['char_embed_size']))
    chars = self.char_cnn(chars)
    chars = tf.reduce_max(chars, 1)
    chars = tf.reshape(chars, (batch_sz, word_len, params['cnn_filters']))

    x = tf.concat((words, chars), -1)
    x = self.dropout(x, training=training)
    x = self.embed_fc(x)
    
    encoder_o, state_fw, state_bw = self.birnn(x, mask=rnn_mask)
    encoder_s = self.state_fc(tf.concat((state_fw, state_bw), -1))
    
    return self.out_fc(encoder_o), tuple([encoder_s])
In [8]:
class TiedDense(tf.keras.layers.Layer):
  def __init__(self, tied_embed, out_dim):
    super().__init__()
    self.tied_embed = tied_embed
    self.out_dim = out_dim
  
  def build(self, input_shape):
    self.bias = self.add_weight(name='bias',
                                shape=[self.out_dim],
                                trainable=True)
    super().build(input_shape)
  
  def call(self, inputs):
    x = tf.matmul(inputs, self.tied_embed, transpose_b=True)
    x = tf.nn.bias_add(x, self.bias)
    return tf.nn.softmax(x)
  
  def compute_output_shape(self, input_shape):
    return input_shape[:-1].concatenate(self.out_dim)
In [9]:
class Pointer(tf.keras.layers.Layer):
  def __init__(self, vocab_size):
    super().__init__()
    self.encoder_ids = None
    self.encoder_out = None
    self.vocab_size = vocab_size
    self.is_beam_search = None

  def call(self, inputs):
    _max_len = tf.shape(self.encoder_ids)[1]
    _batch_size_ori = tf.shape(inputs)[0]
    if self.is_beam_search:
      _batch_size= _batch_size_ori * params['beam_width']
    else:
      _batch_size = _batch_size_ori
    inputs = tf.reshape(inputs, (_batch_size, params['rnn_units']))

    attn_weights = tf.matmul(self.encoder_out, tf.expand_dims(inputs, -1))
    attn_weights = tf.squeeze(attn_weights, -1)
    updates = tf.nn.softmax(attn_weights)
    
    batch_nums = tf.range(0, _batch_size)
    batch_nums = tf.expand_dims(batch_nums, axis=1)
    batch_nums = tf.tile(batch_nums, [1, _max_len])

    indices = tf.stack([batch_nums, self.encoder_ids], axis=2)
    if self.is_beam_search:
      x = tf.scatter_nd(indices, updates, (_batch_size, self.vocab_size))
      return tf.reshape(x, (_batch_size_ori, params['beam_width'], self.vocab_size))
    else:
      x = tf.scatter_nd(indices, updates, (_batch_size, self.vocab_size))
      return x
  
  def compute_output_shape(self, input_shape):
    return input_shape[:-1].concatenate(self.vocab_size)
In [10]:
class OutputProj(tf.keras.layers.Layer):
  def __init__(self, tied_embed, vocab_size):
    super().__init__()
    self.generator = TiedDense(tied_embed, vocab_size)
    self.pointer = Pointer(vocab_size)
    self.vocab_size = vocab_size

  def build(self, input_shape):
    self.gate_fc = tf.keras.layers.Dense(1, tf.sigmoid, use_bias=False)
    super().build(input_shape)
  
  def call(self, inputs):
    gen_dist = self.generator(inputs)
    copy_dist = self.pointer(inputs)
    gate = self.gate_fc(inputs)
    return gate * gen_dist + (1 - gate) * copy_dist
  
  def compute_output_shape(self, input_shape):
    return input_shape[:-1].concatenate(self.vocab_size)
In [11]:
class Model(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.embed = Embed()

    self.char_embed = tf.keras.layers.Embedding(len(params['char2idx'])+1, params['char_embed_size'])

    self.encoder = Encoder(params)

    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])

    self.attn = tfa.seq2seq.BahdanauAttention(params['rnn_units'])

    self.decoder_cell = tfa.seq2seq.AttentionWrapper(
      tf.keras.layers.StackedRNNCells([tf.keras.layers.GRUCell(params['rnn_units'])]),
      self.attn,
      attention_layer_size=params['rnn_units'])
    
    self.proj_layer = OutputProj(self.embed.embedding, len(params['tgt2idx'])+1)

    self.teach_forcing = tfa.seq2seq.BasicDecoder(
      self.decoder_cell,
      tfa.seq2seq.sampler.TrainingSampler(),
      output_layer = self.proj_layer)

    self.beam_search = BeamSearchDecoder(
      self.decoder_cell,
      beam_width = params['beam_width'],
      embedding_fn = lambda x: self.embed(x),
      output_layer = self.proj_layer,
      maximum_iterations = 80,)

  
  def call(self, inputs, training=True):
    if training:
      source, chars, target_in = inputs
    else:
      source, chars = inputs
    source = tf.cast(source, tf.int32)
    batch_sz = tf.shape(source)[0]

    encoder_o, encoder_s = self.encoder((self.embed(source), self.char_embed(chars)), mask=tf.sign(source), training=training)

    if training:
      self.attn.setup_memory(encoder_o, tf.math.count_nonzero(source, 1))
      attn_state = self.decoder_cell.get_initial_state(batch_size=batch_sz, dtype=tf.float32)
      attn_state = attn_state.clone(cell_state=encoder_s)

      self.proj_layer.pointer.encoder_ids = source
      self.proj_layer.pointer.encoder_out = encoder_o
      self.proj_layer.pointer.is_beam_search = False

      decoder_o, _, _ = self.teach_forcing(
        inputs = self.dropout(self.embed(target_in), training=training),
        initial_state = attn_state,
        sequence_length = tf.math.count_nonzero(target_in, 1, dtype=tf.int32))

      logits_or_ids = decoder_o.rnn_output
    else:
      encoder_o_t = tfa.seq2seq.tile_batch(encoder_o, params['beam_width'])
      encoder_len_t = tfa.seq2seq.tile_batch(tf.math.count_nonzero(source, 1), params['beam_width'])
      encoder_s_t = tfa.seq2seq.tile_batch(encoder_s, params['beam_width'])

      self.attn.setup_memory(encoder_o_t, encoder_len_t)
      attn_state = self.decoder_cell.get_initial_state(batch_size=batch_sz*params['beam_width'], dtype=tf.float32)
      attn_state = attn_state.clone(cell_state=encoder_s_t)

      self.proj_layer.pointer.encoder_ids = tfa.seq2seq.tile_batch(source, params['beam_width'])
      self.proj_layer.pointer.encoder_out = encoder_o_t
      self.proj_layer.pointer.is_beam_search = True

      decoder_o, _, _ = self.beam_search(
        embedding = None,
        start_tokens = tf.tile(tf.constant([1], tf.int32), [batch_sz]),
        end_token = 2,
        initial_state = attn_state,)

      logits_or_ids = decoder_o.predicted_ids[:, :, 0]

    return logits_or_ids
In [12]:
params = {
    'train_path': '../data/train.tsv',
    'test_path': '../data/test.tsv',
    'vocab_tgt_path': '../vocab/target.txt',
    'vocab_char_path': '../vocab/char.txt',
    'model_path': '../model/',
    'dropout_rate': .2,
    'char_embed_size': 100,
    'embed_dim': 300,
    'cnn_filters': 300,
    'cnn_kernel_size': 5,
    'rnn_units': 300,
    'activation': tf.nn.swish,
    'beam_width': 10,
    'init_lr': 1e-4,
    'max_lr': 8e-4,
    'clip_norm': .1,
    'buffer_size': 31279,
    'train_batch_size': 32,
    'eval_batch_size': 128,
    'num_patience': 6,
}
In [13]:
def get_vocab(f_path):
  word2idx = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [14]:
params['tgt2idx'] = get_vocab(params['vocab_tgt_path'])
params['idx2tgt'] = {idx: tgt for tgt, idx in params['tgt2idx'].items()}
params['char2idx'] = get_vocab(params['vocab_char_path'])

model = Model(params)
model.build(input_shape=[[None, None], [None, None, None], [None, None]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])
[('pretrained_embedding:0', TensorShape([8692, 300])),
 ('embedding/embeddings:0', TensorShape([68, 100])),
 ('encoder/conv1d/kernel:0', TensorShape([5, 100, 300])),
 ('encoder/conv1d/bias:0', TensorShape([300])),
 ('encoder/embed_fc/kernel:0', TensorShape([600, 300])),
 ('encoder/embed_fc/bias:0', TensorShape([300])),
 ('encoder/bidirectional/forward_gru/gru_cell_1/kernel:0',
  TensorShape([300, 900])),
 ('encoder/bidirectional/forward_gru/gru_cell_1/recurrent_kernel:0',
  TensorShape([300, 900])),
 ('encoder/bidirectional/forward_gru/gru_cell_1/bias:0', TensorShape([2, 900])),
 ('encoder/bidirectional/backward_gru/gru_cell_2/kernel:0',
  TensorShape([300, 900])),
 ('encoder/bidirectional/backward_gru/gru_cell_2/recurrent_kernel:0',
  TensorShape([300, 900])),
 ('encoder/bidirectional/backward_gru/gru_cell_2/bias:0',
  TensorShape([2, 900])),
 ('encoder/state_fc/kernel:0', TensorShape([600, 300])),
 ('encoder/state_fc/bias:0', TensorShape([300])),
 ('encoder/out_fc/kernel:0', TensorShape([600, 300])),
 ('encoder/out_fc/bias:0', TensorShape([300])),
 ('basic_decoder/decoder/while/attention_wrapper/BahdanauAttention/attention_v:0',
  TensorShape([300])),
 ('basic_decoder/decoder/while/attention_wrapper/BahdanauAttention/kernel:0',
  TensorShape([300, 300])),
 ('BahdanauAttention/memory_layer/kernel:0', TensorShape([300, 300])),
 ('basic_decoder/decoder/while/attention_wrapper/attention_layer/kernel:0',
  TensorShape([600, 300])),
 ('basic_decoder/decoder/while/attention_wrapper/stacked_rnn_cells/gru_cell_3/kernel:0',
  TensorShape([600, 900])),
 ('basic_decoder/decoder/while/attention_wrapper/stacked_rnn_cells/gru_cell_3/recurrent_kernel:0',
  TensorShape([300, 900])),
 ('basic_decoder/decoder/while/attention_wrapper/stacked_rnn_cells/gru_cell_3/bias:0',
  TensorShape([2, 900])),
 ('basic_decoder/decoder/while/output_proj/tied_dense/bias:0',
  TensorShape([8692])),
 ('basic_decoder/decoder/while/output_proj/dense/kernel:0',
  TensorShape([300, 1]))]
In [15]:
decay_lr = tfa.optimizers.Triangular2CyclicalLearningRate(
  initial_learning_rate = params['init_lr'],
  maximal_learning_rate = params['max_lr'],
  step_size = 4*params['buffer_size']//params['train_batch_size'],)
optim = tf.optimizers.Adam(params['init_lr'])
global_step = 0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.propagate = False
logger.setLevel(logging.INFO)

best_acc = .0
count = 0
In [16]:
def unit_test(model, params):
  test_str = ['what', 'times', 'are', 'the', 'nutcracker', 'show', 'playing', 'near', 'me']

  max_char_len = 10
  chars = []
  for w in test_str:
    temp = []
    for c in list(w):
      temp.append(get_idx(params['char2idx'], c))
    if len(temp) < max_char_len:
      temp += [0] * (max_char_len - len(temp))
    else:
      temp = temp[:max_char_len]
    chars.append(temp)

  test_arr = tf.convert_to_tensor([[params['tgt2idx'][w] for w in test_str]])
  test_char = tf.convert_to_tensor([chars])
  generated = model(inputs=(test_arr, test_char), training=False)

  print('-'*12)
  print('unit test')
  print('utterance:', ' '.join(test_str))
  parsed = ' '.join([params['idx2tgt'][idx] for idx in generated[0].numpy() if (idx != 0 and idx != 2)])
  print('parsed:', parsed)
  print()
  try:
    nltk.tree.Tree.fromstring(parsed.replace('[ ', '(').replace(' ]', ')')).pretty_print()
  except:
    pass
  print('-'*12)
In [17]:
def cross_entropy_loss(logits, labels, vocab_size, smoothing):
  soft_targets = tf.one_hot(tf.cast(labels, tf.int32), depth=vocab_size)
  soft_targets = ((1-smoothing) * soft_targets) + (smoothing / vocab_size)

  logits = tf.math.minimum(1., logits + 1e-6)
  log_probs = tf.math.log(logits)
  xentropy = - tf.reduce_sum(soft_targets * log_probs, axis=-1)

  weights = tf.cast(tf.math.not_equal(labels, 0), tf.float32)
  xentropy *= weights
  return tf.reduce_sum(xentropy) / tf.reduce_sum(weights)
In [18]:
unit_test(model, params)

while True:
  # TRAINING
  is_training = True
  for i, (source, chars, target_in, target_out) in enumerate(dataset(is_training=is_training, params=params)):
    with tf.GradientTape() as tape:
      logits_or_ids = model((source, chars, target_in), training=is_training)
      loss = cross_entropy_loss(logits_or_ids, target_out, len(params['tgt2idx'])+1, .2)  

    variables = model.trainable_variables
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    optim.apply_gradients(zip(grads, variables))
    
    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
        global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    
    global_step += 1

  # EVALUATION
  is_training = False
  unit_test(model, params)
  m = tf.keras.metrics.Mean()
  
  parse_fn = lambda x: [e for e in x if (e != 0 and e != 2)]

  for i, (source, chars, target_in, target_out) in enumerate(dataset(is_training=is_training, params=params)):
    generated = model(inputs=(source, chars), training=is_training)
    for pred, tgt in zip(generated.numpy(), target_out.numpy()):
      matched = np.array_equal(parse_fn(pred), parse_fn(tgt))
      m.update_state(int(matched))
  
  acc = m.result().numpy()
  logger.info("Evaluation: Testing EM: {:.3f}".format(acc))

  if acc > best_acc:
    best_acc = acc
    count = 0
    model.save_weights('../model/gru_pointer_clr')
  else:
    count += 1
  logger.info("Best EM: {:.3f}".format(best_acc))

  if count == params['num_patience']:
    print(params['num_patience'], "times not improve the best result, therefore stop training")
    break
/usr/local/lib/python3.6/dist-packages/tensorflow_addons/utils/resource_loader.py:103: UserWarning: You are currently using TensorFlow 2.3.0 and trying to load a custom op (custom_ops/seq2seq/_beam_search_ops.so).
TensorFlow Addons has compiled its custom ops against TensorFlow 2.2.0, and there are no compatibility guarantees between the two versions. 
This means that you might get segfaults when loading the custom op, or other kind of low-level errors.
 If you do, do not file an issue on Github. This is a known limitation.

It might help you to fallback to pure Python ops with TF_ADDONS_PY_OPS . To do that, see https://github.com/tensorflow/addons#gpucpu-custom-ops 

You can also change the TensorFlow version installed on your system. You would need a TensorFlow version equal to or above 2.2.0 and strictly below 2.3.0.
 Note that nightly versions of TensorFlow, as well as non-pip TensorFlow like `conda install tensorflow` or compiled from source are not supported.

The last solution is to find the TensorFlow Addons version that has custom ops compatible with the TensorFlow installed on your system. To do that, refer to the readme: https://github.com/tensorflow/addons
  UserWarning,
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: me me playing me me playing me me me me the me me playing me me me me the me me playing me me me me the me me playing me me me me the me me playing me me me me the me me playing me me me me the me me playing me me me me the me me playing me me me me the me me playing me me me me the me me playing me nutcracker

------------
Reading ../data/train.tsv
INFO:tensorflow:Step 0 | Loss: 7.7416 | Spent: 24.9 secs | LR: 0.000100
INFO:tensorflow:Step 50 | Loss: 4.8253 | Spent: 34.5 secs | LR: 0.000109
INFO:tensorflow:Step 100 | Loss: 4.2872 | Spent: 34.2 secs | LR: 0.000118
INFO:tensorflow:Step 150 | Loss: 3.8141 | Spent: 35.3 secs | LR: 0.000127
INFO:tensorflow:Step 200 | Loss: 3.8106 | Spent: 33.8 secs | LR: 0.000136
INFO:tensorflow:Step 250 | Loss: 3.7581 | Spent: 33.0 secs | LR: 0.000145
INFO:tensorflow:Step 300 | Loss: 3.6028 | Spent: 34.2 secs | LR: 0.000154
INFO:tensorflow:Step 350 | Loss: 3.3973 | Spent: 33.6 secs | LR: 0.000163
INFO:tensorflow:Step 400 | Loss: 3.6943 | Spent: 34.7 secs | LR: 0.000172
INFO:tensorflow:Step 450 | Loss: 3.3863 | Spent: 34.5 secs | LR: 0.000181
INFO:tensorflow:Step 500 | Loss: 3.3324 | Spent: 33.8 secs | LR: 0.000190
INFO:tensorflow:Step 550 | Loss: 3.3545 | Spent: 34.1 secs | LR: 0.000198
INFO:tensorflow:Step 600 | Loss: 3.2589 | Spent: 36.7 secs | LR: 0.000207
INFO:tensorflow:Step 650 | Loss: 3.1180 | Spent: 32.4 secs | LR: 0.000216
INFO:tensorflow:Step 700 | Loss: 3.0595 | Spent: 33.1 secs | LR: 0.000225
INFO:tensorflow:Step 750 | Loss: 3.1210 | Spent: 33.1 secs | LR: 0.000234
INFO:tensorflow:Step 800 | Loss: 3.0119 | Spent: 31.8 secs | LR: 0.000243
INFO:tensorflow:Step 850 | Loss: 2.8718 | Spent: 33.3 secs | LR: 0.000252
INFO:tensorflow:Step 900 | Loss: 2.9508 | Spent: 33.1 secs | LR: 0.000261
INFO:tensorflow:Step 950 | Loss: 2.9321 | Spent: 33.9 secs | LR: 0.000270
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_info_traffic what times are the nutcracker show playing near me ]

                   in:get_info_traf                      
                         fic                             
  ________________________|____________________________   
what times are the    nutcracker    show playing near  me

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.062
INFO:tensorflow:Best EM: 0.062
Reading ../data/train.tsv
INFO:tensorflow:Step 1000 | Loss: 2.8980 | Spent: 182.6 secs | LR: 0.000279
INFO:tensorflow:Step 1050 | Loss: 2.8179 | Spent: 34.2 secs | LR: 0.000288
INFO:tensorflow:Step 1100 | Loss: 2.8210 | Spent: 33.7 secs | LR: 0.000297
INFO:tensorflow:Step 1150 | Loss: 2.7334 | Spent: 31.5 secs | LR: 0.000306
INFO:tensorflow:Step 1200 | Loss: 2.7423 | Spent: 33.3 secs | LR: 0.000315
INFO:tensorflow:Step 1250 | Loss: 2.7699 | Spent: 33.2 secs | LR: 0.000324
INFO:tensorflow:Step 1300 | Loss: 2.6938 | Spent: 34.4 secs | LR: 0.000333
INFO:tensorflow:Step 1350 | Loss: 2.7225 | Spent: 32.8 secs | LR: 0.000342
INFO:tensorflow:Step 1400 | Loss: 2.7560 | Spent: 33.1 secs | LR: 0.000351
INFO:tensorflow:Step 1450 | Loss: 2.6133 | Spent: 33.4 secs | LR: 0.000360
INFO:tensorflow:Step 1500 | Loss: 2.6016 | Spent: 34.1 secs | LR: 0.000369
INFO:tensorflow:Step 1550 | Loss: 2.6027 | Spent: 35.3 secs | LR: 0.000378
INFO:tensorflow:Step 1600 | Loss: 2.6171 | Spent: 33.9 secs | LR: 0.000387
INFO:tensorflow:Step 1650 | Loss: 2.5584 | Spent: 33.8 secs | LR: 0.000395
INFO:tensorflow:Step 1700 | Loss: 2.6161 | Spent: 32.7 secs | LR: 0.000404
INFO:tensorflow:Step 1750 | Loss: 2.5879 | Spent: 33.9 secs | LR: 0.000413
INFO:tensorflow:Step 1800 | Loss: 2.5656 | Spent: 32.4 secs | LR: 0.000422
INFO:tensorflow:Step 1850 | Loss: 2.5577 | Spent: 32.5 secs | LR: 0.000431
INFO:tensorflow:Step 1900 | Loss: 2.5454 | Spent: 33.6 secs | LR: 0.000440
INFO:tensorflow:Step 1950 | Loss: 2.5222 | Spent: 34.0 secs | LR: 0.000449
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are the nutcracker show playing near me ]

                   in:get_event                      
  ______________________|__________________________   
what times are the  nutcracker  show playing near  me

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.400
INFO:tensorflow:Best EM: 0.400
Reading ../data/train.tsv
INFO:tensorflow:Step 2000 | Loss: 2.5129 | Spent: 178.3 secs | LR: 0.000458
INFO:tensorflow:Step 2050 | Loss: 2.5075 | Spent: 34.1 secs | LR: 0.000467
INFO:tensorflow:Step 2100 | Loss: 2.5446 | Spent: 34.1 secs | LR: 0.000476
INFO:tensorflow:Step 2150 | Loss: 2.5239 | Spent: 33.5 secs | LR: 0.000485
INFO:tensorflow:Step 2200 | Loss: 2.4518 | Spent: 34.5 secs | LR: 0.000494
INFO:tensorflow:Step 2250 | Loss: 2.4719 | Spent: 32.5 secs | LR: 0.000503
INFO:tensorflow:Step 2300 | Loss: 2.4971 | Spent: 33.3 secs | LR: 0.000512
INFO:tensorflow:Step 2350 | Loss: 2.4734 | Spent: 33.5 secs | LR: 0.000521
INFO:tensorflow:Step 2400 | Loss: 2.4557 | Spent: 33.8 secs | LR: 0.000530
INFO:tensorflow:Step 2450 | Loss: 2.4482 | Spent: 34.5 secs | LR: 0.000539
INFO:tensorflow:Step 2500 | Loss: 2.5204 | Spent: 34.4 secs | LR: 0.000548
INFO:tensorflow:Step 2550 | Loss: 2.4373 | Spent: 33.5 secs | LR: 0.000557
INFO:tensorflow:Step 2600 | Loss: 2.4355 | Spent: 32.6 secs | LR: 0.000566
INFO:tensorflow:Step 2650 | Loss: 2.4644 | Spent: 32.8 secs | LR: 0.000575
INFO:tensorflow:Step 2700 | Loss: 2.4408 | Spent: 33.4 secs | LR: 0.000583
INFO:tensorflow:Step 2750 | Loss: 2.4676 | Spent: 33.7 secs | LR: 0.000592
INFO:tensorflow:Step 2800 | Loss: 2.4346 | Spent: 33.3 secs | LR: 0.000601
INFO:tensorflow:Step 2850 | Loss: 2.4346 | Spent: 33.0 secs | LR: 0.000610
INFO:tensorflow:Step 2900 | Loss: 2.3983 | Spent: 32.4 secs | LR: 0.000619
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

               in:get_event                                                                        
  __________________|_____________________________________________________                          
 |     |    |                      |                                 sl:location                   
 |     |    |                      |                                      |                         
 |     |    |                      |                               in:get_location                 
 |     |    |                      |                      ________________|_______________          
 |     |    |               sl:category_even      sl:search_radius                 sl:location_user
 |     |    |                      t                     |                                |        
 |     |    |        ______________|__________           |                                |         
what times are     the         nutcracker    show       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.642
INFO:tensorflow:Best EM: 0.642
Reading ../data/train.tsv
INFO:tensorflow:Step 2950 | Loss: 2.4046 | Spent: 199.9 secs | LR: 0.000628
INFO:tensorflow:Step 3000 | Loss: 2.4964 | Spent: 33.8 secs | LR: 0.000637
INFO:tensorflow:Step 3050 | Loss: 2.4143 | Spent: 33.1 secs | LR: 0.000646
INFO:tensorflow:Step 3100 | Loss: 2.4216 | Spent: 33.6 secs | LR: 0.000655
INFO:tensorflow:Step 3150 | Loss: 2.4111 | Spent: 33.5 secs | LR: 0.000664
INFO:tensorflow:Step 3200 | Loss: 2.4456 | Spent: 33.8 secs | LR: 0.000673
INFO:tensorflow:Step 3250 | Loss: 2.3939 | Spent: 32.9 secs | LR: 0.000682
INFO:tensorflow:Step 3300 | Loss: 2.4151 | Spent: 33.1 secs | LR: 0.000691
INFO:tensorflow:Step 3350 | Loss: 2.4030 | Spent: 33.9 secs | LR: 0.000700
INFO:tensorflow:Step 3400 | Loss: 2.3912 | Spent: 33.2 secs | LR: 0.000709
INFO:tensorflow:Step 3450 | Loss: 2.3995 | Spent: 34.0 secs | LR: 0.000718
INFO:tensorflow:Step 3500 | Loss: 2.3969 | Spent: 33.1 secs | LR: 0.000727
INFO:tensorflow:Step 3550 | Loss: 2.3940 | Spent: 34.5 secs | LR: 0.000736
INFO:tensorflow:Step 3600 | Loss: 2.4102 | Spent: 32.7 secs | LR: 0.000745
INFO:tensorflow:Step 3650 | Loss: 2.4161 | Spent: 33.8 secs | LR: 0.000754
INFO:tensorflow:Step 3700 | Loss: 2.4295 | Spent: 33.4 secs | LR: 0.000763
INFO:tensorflow:Step 3750 | Loss: 2.3964 | Spent: 34.0 secs | LR: 0.000772
INFO:tensorflow:Step 3800 | Loss: 2.4307 | Spent: 31.6 secs | LR: 0.000780
INFO:tensorflow:Step 3850 | Loss: 2.3874 | Spent: 33.2 secs | LR: 0.000789
INFO:tensorflow:Step 3900 | Loss: 2.3962 | Spent: 32.6 secs | LR: 0.000798
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.717
INFO:tensorflow:Best EM: 0.717
Reading ../data/train.tsv
INFO:tensorflow:Step 3950 | Loss: 2.4057 | Spent: 199.7 secs | LR: 0.000793
INFO:tensorflow:Step 4000 | Loss: 2.4016 | Spent: 32.1 secs | LR: 0.000784
INFO:tensorflow:Step 4050 | Loss: 2.3742 | Spent: 32.9 secs | LR: 0.000775
INFO:tensorflow:Step 4100 | Loss: 2.3768 | Spent: 32.6 secs | LR: 0.000766
INFO:tensorflow:Step 4150 | Loss: 2.3707 | Spent: 33.0 secs | LR: 0.000757
INFO:tensorflow:Step 4200 | Loss: 2.3847 | Spent: 33.7 secs | LR: 0.000748
INFO:tensorflow:Step 4250 | Loss: 2.3967 | Spent: 32.6 secs | LR: 0.000739
INFO:tensorflow:Step 4300 | Loss: 2.3694 | Spent: 34.1 secs | LR: 0.000730
INFO:tensorflow:Step 4350 | Loss: 2.3865 | Spent: 34.6 secs | LR: 0.000721
INFO:tensorflow:Step 4400 | Loss: 2.3703 | Spent: 32.4 secs | LR: 0.000712
INFO:tensorflow:Step 4450 | Loss: 2.3595 | Spent: 34.3 secs | LR: 0.000703
INFO:tensorflow:Step 4500 | Loss: 2.3635 | Spent: 32.7 secs | LR: 0.000694
INFO:tensorflow:Step 4550 | Loss: 2.3872 | Spent: 32.9 secs | LR: 0.000685
INFO:tensorflow:Step 4600 | Loss: 2.3988 | Spent: 32.0 secs | LR: 0.000676
INFO:tensorflow:Step 4650 | Loss: 2.3585 | Spent: 34.7 secs | LR: 0.000667
INFO:tensorflow:Step 4700 | Loss: 2.3818 | Spent: 33.7 secs | LR: 0.000658
INFO:tensorflow:Step 4750 | Loss: 2.3762 | Spent: 32.4 secs | LR: 0.000649
INFO:tensorflow:Step 4800 | Loss: 2.3663 | Spent: 33.4 secs | LR: 0.000640
INFO:tensorflow:Step 4850 | Loss: 2.3521 | Spent: 31.6 secs | LR: 0.000631
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.705
INFO:tensorflow:Best EM: 0.717
Reading ../data/train.tsv
INFO:tensorflow:Step 4900 | Loss: 2.3448 | Spent: 188.2 secs | LR: 0.000623
INFO:tensorflow:Step 4950 | Loss: 2.3538 | Spent: 34.9 secs | LR: 0.000614
INFO:tensorflow:Step 5000 | Loss: 2.3525 | Spent: 34.3 secs | LR: 0.000605
INFO:tensorflow:Step 5050 | Loss: 2.3519 | Spent: 32.8 secs | LR: 0.000596
INFO:tensorflow:Step 5100 | Loss: 2.3712 | Spent: 33.8 secs | LR: 0.000587
INFO:tensorflow:Step 5150 | Loss: 2.3444 | Spent: 34.2 secs | LR: 0.000578
INFO:tensorflow:Step 5200 | Loss: 2.3441 | Spent: 32.3 secs | LR: 0.000569
INFO:tensorflow:Step 5250 | Loss: 2.3660 | Spent: 34.0 secs | LR: 0.000560
INFO:tensorflow:Step 5300 | Loss: 2.3442 | Spent: 32.9 secs | LR: 0.000551
INFO:tensorflow:Step 5350 | Loss: 2.3499 | Spent: 33.5 secs | LR: 0.000542
INFO:tensorflow:Step 5400 | Loss: 2.3522 | Spent: 32.7 secs | LR: 0.000533
INFO:tensorflow:Step 5450 | Loss: 2.3846 | Spent: 33.0 secs | LR: 0.000524
INFO:tensorflow:Step 5500 | Loss: 2.3573 | Spent: 35.3 secs | LR: 0.000515
INFO:tensorflow:Step 5550 | Loss: 2.3393 | Spent: 33.2 secs | LR: 0.000506
INFO:tensorflow:Step 5600 | Loss: 2.3630 | Spent: 34.1 secs | LR: 0.000497
INFO:tensorflow:Step 5650 | Loss: 2.3528 | Spent: 32.5 secs | LR: 0.000488
INFO:tensorflow:Step 5700 | Loss: 2.3575 | Spent: 32.5 secs | LR: 0.000479
INFO:tensorflow:Step 5750 | Loss: 2.3511 | Spent: 33.4 secs | LR: 0.000470
INFO:tensorflow:Step 5800 | Loss: 2.3417 | Spent: 33.2 secs | LR: 0.000461
INFO:tensorflow:Step 5850 | Loss: 2.3477 | Spent: 32.6 secs | LR: 0.000452
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.776
INFO:tensorflow:Best EM: 0.776
Reading ../data/train.tsv
INFO:tensorflow:Step 5900 | Loss: 2.3445 | Spent: 195.0 secs | LR: 0.000443
INFO:tensorflow:Step 5950 | Loss: 2.3299 | Spent: 33.7 secs | LR: 0.000435
INFO:tensorflow:Step 6000 | Loss: 2.3392 | Spent: 33.9 secs | LR: 0.000426
INFO:tensorflow:Step 6050 | Loss: 2.3294 | Spent: 32.8 secs | LR: 0.000417
INFO:tensorflow:Step 6100 | Loss: 2.3322 | Spent: 31.8 secs | LR: 0.000408
INFO:tensorflow:Step 6150 | Loss: 2.3393 | Spent: 33.1 secs | LR: 0.000399
INFO:tensorflow:Step 6200 | Loss: 2.3494 | Spent: 34.4 secs | LR: 0.000390
INFO:tensorflow:Step 6250 | Loss: 2.3399 | Spent: 33.4 secs | LR: 0.000381
INFO:tensorflow:Step 6300 | Loss: 2.3393 | Spent: 32.9 secs | LR: 0.000372
INFO:tensorflow:Step 6350 | Loss: 2.3410 | Spent: 34.3 secs | LR: 0.000363
INFO:tensorflow:Step 6400 | Loss: 2.3583 | Spent: 33.1 secs | LR: 0.000354
INFO:tensorflow:Step 6450 | Loss: 2.3276 | Spent: 33.0 secs | LR: 0.000345
INFO:tensorflow:Step 6500 | Loss: 2.3428 | Spent: 33.0 secs | LR: 0.000336
INFO:tensorflow:Step 6550 | Loss: 2.3366 | Spent: 32.8 secs | LR: 0.000327
INFO:tensorflow:Step 6600 | Loss: 2.3506 | Spent: 35.2 secs | LR: 0.000318
INFO:tensorflow:Step 6650 | Loss: 2.3456 | Spent: 33.9 secs | LR: 0.000309
INFO:tensorflow:Step 6700 | Loss: 2.3416 | Spent: 34.7 secs | LR: 0.000300
INFO:tensorflow:Step 6750 | Loss: 2.3427 | Spent: 32.3 secs | LR: 0.000291
INFO:tensorflow:Step 6800 | Loss: 2.3398 | Spent: 31.8 secs | LR: 0.000282
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.794
INFO:tensorflow:Best EM: 0.794
Reading ../data/train.tsv
INFO:tensorflow:Step 6850 | Loss: 2.3398 | Spent: 196.9 secs | LR: 0.000273
INFO:tensorflow:Step 6900 | Loss: 2.3372 | Spent: 34.0 secs | LR: 0.000264
INFO:tensorflow:Step 6950 | Loss: 2.3293 | Spent: 32.9 secs | LR: 0.000255
INFO:tensorflow:Step 7000 | Loss: 2.3349 | Spent: 33.3 secs | LR: 0.000246
INFO:tensorflow:Step 7050 | Loss: 2.3349 | Spent: 34.0 secs | LR: 0.000238
INFO:tensorflow:Step 7100 | Loss: 2.3279 | Spent: 34.4 secs | LR: 0.000229
INFO:tensorflow:Step 7150 | Loss: 2.3347 | Spent: 32.7 secs | LR: 0.000220
INFO:tensorflow:Step 7200 | Loss: 2.3358 | Spent: 33.5 secs | LR: 0.000211
INFO:tensorflow:Step 7250 | Loss: 2.3271 | Spent: 33.1 secs | LR: 0.000202
INFO:tensorflow:Step 7300 | Loss: 2.3288 | Spent: 33.7 secs | LR: 0.000193
INFO:tensorflow:Step 7350 | Loss: 2.3230 | Spent: 33.8 secs | LR: 0.000184
INFO:tensorflow:Step 7400 | Loss: 2.3247 | Spent: 33.6 secs | LR: 0.000175
INFO:tensorflow:Step 7450 | Loss: 2.3301 | Spent: 33.4 secs | LR: 0.000166
INFO:tensorflow:Step 7500 | Loss: 2.3213 | Spent: 32.2 secs | LR: 0.000157
INFO:tensorflow:Step 7550 | Loss: 2.3346 | Spent: 32.7 secs | LR: 0.000148
INFO:tensorflow:Step 7600 | Loss: 2.3224 | Spent: 34.8 secs | LR: 0.000139
INFO:tensorflow:Step 7650 | Loss: 2.3328 | Spent: 33.4 secs | LR: 0.000130
INFO:tensorflow:Step 7700 | Loss: 2.3674 | Spent: 32.8 secs | LR: 0.000121
INFO:tensorflow:Step 7750 | Loss: 2.3222 | Spent: 33.6 secs | LR: 0.000112
INFO:tensorflow:Step 7800 | Loss: 2.3187 | Spent: 34.6 secs | LR: 0.000103
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.807
INFO:tensorflow:Best EM: 0.807
Reading ../data/train.tsv
INFO:tensorflow:Step 7850 | Loss: 2.3231 | Spent: 196.9 secs | LR: 0.000103
INFO:tensorflow:Step 7900 | Loss: 2.3257 | Spent: 33.1 secs | LR: 0.000107
INFO:tensorflow:Step 7950 | Loss: 2.3311 | Spent: 32.1 secs | LR: 0.000112
INFO:tensorflow:Step 8000 | Loss: 2.3192 | Spent: 35.8 secs | LR: 0.000116
INFO:tensorflow:Step 8050 | Loss: 2.3242 | Spent: 33.5 secs | LR: 0.000121
INFO:tensorflow:Step 8100 | Loss: 2.3303 | Spent: 33.2 secs | LR: 0.000125
INFO:tensorflow:Step 8150 | Loss: 2.3336 | Spent: 33.2 secs | LR: 0.000130
INFO:tensorflow:Step 8200 | Loss: 2.3291 | Spent: 34.0 secs | LR: 0.000134
INFO:tensorflow:Step 8250 | Loss: 2.3261 | Spent: 33.4 secs | LR: 0.000139
INFO:tensorflow:Step 8300 | Loss: 2.3244 | Spent: 34.2 secs | LR: 0.000143
INFO:tensorflow:Step 8350 | Loss: 2.3366 | Spent: 33.8 secs | LR: 0.000148
INFO:tensorflow:Step 8400 | Loss: 2.3332 | Spent: 32.7 secs | LR: 0.000152
INFO:tensorflow:Step 8450 | Loss: 2.3256 | Spent: 36.5 secs | LR: 0.000157
INFO:tensorflow:Step 8500 | Loss: 2.3401 | Spent: 33.8 secs | LR: 0.000161
INFO:tensorflow:Step 8550 | Loss: 2.3235 | Spent: 32.5 secs | LR: 0.000166
INFO:tensorflow:Step 8600 | Loss: 2.3224 | Spent: 33.3 secs | LR: 0.000170
INFO:tensorflow:Step 8650 | Loss: 2.3341 | Spent: 33.6 secs | LR: 0.000174
INFO:tensorflow:Step 8700 | Loss: 2.3294 | Spent: 34.1 secs | LR: 0.000179
INFO:tensorflow:Step 8750 | Loss: 2.3232 | Spent: 34.6 secs | LR: 0.000183
INFO:tensorflow:Step 8800 | Loss: 2.3269 | Spent: 32.5 secs | LR: 0.000188
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.806
INFO:tensorflow:Best EM: 0.807
Reading ../data/train.tsv
INFO:tensorflow:Step 8850 | Loss: 2.3206 | Spent: 195.2 secs | LR: 0.000192
INFO:tensorflow:Step 8900 | Loss: 2.3215 | Spent: 33.8 secs | LR: 0.000197
INFO:tensorflow:Step 8950 | Loss: 2.3413 | Spent: 34.0 secs | LR: 0.000201
INFO:tensorflow:Step 9000 | Loss: 2.3381 | Spent: 33.1 secs | LR: 0.000206
INFO:tensorflow:Step 9050 | Loss: 2.3212 | Spent: 34.4 secs | LR: 0.000210
INFO:tensorflow:Step 9100 | Loss: 2.3228 | Spent: 33.3 secs | LR: 0.000215
INFO:tensorflow:Step 9150 | Loss: 2.3210 | Spent: 35.1 secs | LR: 0.000219
INFO:tensorflow:Step 9200 | Loss: 2.3395 | Spent: 31.6 secs | LR: 0.000224
INFO:tensorflow:Step 9250 | Loss: 2.3342 | Spent: 33.3 secs | LR: 0.000228
INFO:tensorflow:Step 9300 | Loss: 2.3317 | Spent: 33.7 secs | LR: 0.000233
INFO:tensorflow:Step 9350 | Loss: 2.3260 | Spent: 33.0 secs | LR: 0.000237
INFO:tensorflow:Step 9400 | Loss: 2.3309 | Spent: 32.2 secs | LR: 0.000242
INFO:tensorflow:Step 9450 | Loss: 2.3191 | Spent: 33.3 secs | LR: 0.000246
INFO:tensorflow:Step 9500 | Loss: 2.3204 | Spent: 33.7 secs | LR: 0.000251
INFO:tensorflow:Step 9550 | Loss: 2.3517 | Spent: 33.7 secs | LR: 0.000255
INFO:tensorflow:Step 9600 | Loss: 2.3237 | Spent: 34.1 secs | LR: 0.000260
INFO:tensorflow:Step 9650 | Loss: 2.3431 | Spent: 34.0 secs | LR: 0.000264
INFO:tensorflow:Step 9700 | Loss: 2.3197 | Spent: 32.2 secs | LR: 0.000269
INFO:tensorflow:Step 9750 | Loss: 2.3216 | Spent: 34.5 secs | LR: 0.000273
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.784
INFO:tensorflow:Best EM: 0.807
Reading ../data/train.tsv
INFO:tensorflow:Step 9800 | Loss: 2.3330 | Spent: 193.5 secs | LR: 0.000277
INFO:tensorflow:Step 9850 | Loss: 2.3244 | Spent: 33.5 secs | LR: 0.000282
INFO:tensorflow:Step 9900 | Loss: 2.3201 | Spent: 32.9 secs | LR: 0.000286
INFO:tensorflow:Step 9950 | Loss: 2.3386 | Spent: 34.2 secs | LR: 0.000291
INFO:tensorflow:Step 10000 | Loss: 2.3196 | Spent: 32.1 secs | LR: 0.000295
INFO:tensorflow:Step 10050 | Loss: 2.3301 | Spent: 34.1 secs | LR: 0.000300
INFO:tensorflow:Step 10100 | Loss: 2.3287 | Spent: 33.0 secs | LR: 0.000304
INFO:tensorflow:Step 10150 | Loss: 2.3349 | Spent: 34.1 secs | LR: 0.000309
INFO:tensorflow:Step 10200 | Loss: 2.3238 | Spent: 32.8 secs | LR: 0.000313
INFO:tensorflow:Step 10250 | Loss: 2.3273 | Spent: 33.4 secs | LR: 0.000318
INFO:tensorflow:Step 10300 | Loss: 2.3281 | Spent: 33.3 secs | LR: 0.000322
INFO:tensorflow:Step 10350 | Loss: 2.3220 | Spent: 34.4 secs | LR: 0.000327
INFO:tensorflow:Step 10400 | Loss: 2.3391 | Spent: 34.0 secs | LR: 0.000331
INFO:tensorflow:Step 10450 | Loss: 2.3302 | Spent: 34.7 secs | LR: 0.000336
INFO:tensorflow:Step 10500 | Loss: 2.3257 | Spent: 34.2 secs | LR: 0.000340
INFO:tensorflow:Step 10550 | Loss: 2.3354 | Spent: 33.1 secs | LR: 0.000345
INFO:tensorflow:Step 10600 | Loss: 2.3408 | Spent: 33.2 secs | LR: 0.000349
INFO:tensorflow:Step 10650 | Loss: 2.3266 | Spent: 32.9 secs | LR: 0.000354
INFO:tensorflow:Step 10700 | Loss: 2.3415 | Spent: 32.8 secs | LR: 0.000358
INFO:tensorflow:Step 10750 | Loss: 2.3379 | Spent: 34.2 secs | LR: 0.000363
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.794
INFO:tensorflow:Best EM: 0.807
Reading ../data/train.tsv
INFO:tensorflow:Step 10800 | Loss: 2.3283 | Spent: 199.9 secs | LR: 0.000367
INFO:tensorflow:Step 10850 | Loss: 2.3278 | Spent: 32.8 secs | LR: 0.000371
INFO:tensorflow:Step 10900 | Loss: 2.3280 | Spent: 33.5 secs | LR: 0.000376
INFO:tensorflow:Step 10950 | Loss: 2.3265 | Spent: 33.8 secs | LR: 0.000380
INFO:tensorflow:Step 11000 | Loss: 2.3204 | Spent: 33.0 secs | LR: 0.000385
INFO:tensorflow:Step 11050 | Loss: 2.3246 | Spent: 33.2 secs | LR: 0.000389
INFO:tensorflow:Step 11100 | Loss: 2.3277 | Spent: 33.7 secs | LR: 0.000394
INFO:tensorflow:Step 11150 | Loss: 2.3321 | Spent: 34.0 secs | LR: 0.000398
INFO:tensorflow:Step 11200 | Loss: 2.3379 | Spent: 34.6 secs | LR: 0.000403
INFO:tensorflow:Step 11250 | Loss: 2.3231 | Spent: 33.0 secs | LR: 0.000407
INFO:tensorflow:Step 11300 | Loss: 2.3401 | Spent: 33.8 secs | LR: 0.000412
INFO:tensorflow:Step 11350 | Loss: 2.3339 | Spent: 34.9 secs | LR: 0.000416
INFO:tensorflow:Step 11400 | Loss: 2.3495 | Spent: 33.5 secs | LR: 0.000421
INFO:tensorflow:Step 11450 | Loss: 2.3350 | Spent: 34.1 secs | LR: 0.000425
INFO:tensorflow:Step 11500 | Loss: 2.3400 | Spent: 32.2 secs | LR: 0.000430
INFO:tensorflow:Step 11550 | Loss: 2.3354 | Spent: 33.4 secs | LR: 0.000434
INFO:tensorflow:Step 11600 | Loss: 2.3390 | Spent: 33.3 secs | LR: 0.000439
INFO:tensorflow:Step 11650 | Loss: 2.3292 | Spent: 33.8 secs | LR: 0.000443
INFO:tensorflow:Step 11700 | Loss: 2.3208 | Spent: 31.6 secs | LR: 0.000448
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.799
INFO:tensorflow:Best EM: 0.807
Reading ../data/train.tsv
INFO:tensorflow:Step 11750 | Loss: 2.3392 | Spent: 198.2 secs | LR: 0.000448
INFO:tensorflow:Step 11800 | Loss: 2.3295 | Spent: 33.4 secs | LR: 0.000443
INFO:tensorflow:Step 11850 | Loss: 2.3285 | Spent: 33.6 secs | LR: 0.000439
INFO:tensorflow:Step 11900 | Loss: 2.3288 | Spent: 32.7 secs | LR: 0.000435
INFO:tensorflow:Step 11950 | Loss: 2.3248 | Spent: 32.7 secs | LR: 0.000430
INFO:tensorflow:Step 12000 | Loss: 2.3242 | Spent: 34.2 secs | LR: 0.000426
INFO:tensorflow:Step 12050 | Loss: 2.3351 | Spent: 34.4 secs | LR: 0.000421
INFO:tensorflow:Step 12100 | Loss: 2.3180 | Spent: 34.8 secs | LR: 0.000417
INFO:tensorflow:Step 12150 | Loss: 2.3239 | Spent: 33.0 secs | LR: 0.000412
INFO:tensorflow:Step 12200 | Loss: 2.3296 | Spent: 34.2 secs | LR: 0.000408
INFO:tensorflow:Step 12250 | Loss: 2.3342 | Spent: 33.7 secs | LR: 0.000403
INFO:tensorflow:Step 12300 | Loss: 2.3265 | Spent: 33.5 secs | LR: 0.000399
INFO:tensorflow:Step 12350 | Loss: 2.3335 | Spent: 32.7 secs | LR: 0.000394
INFO:tensorflow:Step 12400 | Loss: 2.3261 | Spent: 32.2 secs | LR: 0.000390
INFO:tensorflow:Step 12450 | Loss: 2.3229 | Spent: 31.7 secs | LR: 0.000385
INFO:tensorflow:Step 12500 | Loss: 2.3218 | Spent: 33.4 secs | LR: 0.000381
INFO:tensorflow:Step 12550 | Loss: 2.3237 | Spent: 32.9 secs | LR: 0.000376
INFO:tensorflow:Step 12600 | Loss: 2.3627 | Spent: 33.9 secs | LR: 0.000372
INFO:tensorflow:Step 12650 | Loss: 2.3276 | Spent: 33.6 secs | LR: 0.000367
INFO:tensorflow:Step 12700 | Loss: 2.3229 | Spent: 33.2 secs | LR: 0.000363
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show ] playing [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                       in:get_event                                                                        
  __________________________|_____________________________________________________                          
 |     |    |     |                        |                                 sl:location                   
 |     |    |     |                        |                                      |                         
 |     |    |     |                        |                               in:get_location                 
 |     |    |     |                        |                      ________________|_______________          
 |     |    |     |                 sl:category_even      sl:search_radius                 sl:location_user
 |     |    |     |                        t                     |                                |        
 |     |    |     |          ______________|__________           |                                |         
what times are playing     the         nutcracker    show       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.798
INFO:tensorflow:Best EM: 0.807
Reading ../data/train.tsv
INFO:tensorflow:Step 12750 | Loss: 2.3246 | Spent: 195.8 secs | LR: 0.000358
INFO:tensorflow:Step 12800 | Loss: 2.3260 | Spent: 33.9 secs | LR: 0.000354
INFO:tensorflow:Step 12850 | Loss: 2.3232 | Spent: 33.4 secs | LR: 0.000349
INFO:tensorflow:Step 12900 | Loss: 2.3224 | Spent: 33.0 secs | LR: 0.000345
INFO:tensorflow:Step 12950 | Loss: 2.3206 | Spent: 35.0 secs | LR: 0.000340
INFO:tensorflow:Step 13000 | Loss: 2.3311 | Spent: 32.3 secs | LR: 0.000336
INFO:tensorflow:Step 13050 | Loss: 2.3181 | Spent: 33.4 secs | LR: 0.000332
INFO:tensorflow:Step 13100 | Loss: 2.3179 | Spent: 32.1 secs | LR: 0.000327
INFO:tensorflow:Step 13150 | Loss: 2.3268 | Spent: 33.5 secs | LR: 0.000323
INFO:tensorflow:Step 13200 | Loss: 2.3325 | Spent: 33.7 secs | LR: 0.000318
INFO:tensorflow:Step 13250 | Loss: 2.3214 | Spent: 32.6 secs | LR: 0.000314
INFO:tensorflow:Step 13300 | Loss: 2.3170 | Spent: 34.2 secs | LR: 0.000309
INFO:tensorflow:Step 13350 | Loss: 2.3222 | Spent: 32.2 secs | LR: 0.000305
INFO:tensorflow:Step 13400 | Loss: 2.3174 | Spent: 33.6 secs | LR: 0.000300
INFO:tensorflow:Step 13450 | Loss: 2.3226 | Spent: 33.0 secs | LR: 0.000296
INFO:tensorflow:Step 13500 | Loss: 2.3196 | Spent: 32.3 secs | LR: 0.000291
INFO:tensorflow:Step 13550 | Loss: 2.3300 | Spent: 33.7 secs | LR: 0.000287
INFO:tensorflow:Step 13600 | Loss: 2.3301 | Spent: 32.9 secs | LR: 0.000282
INFO:tensorflow:Step 13650 | Loss: 2.3413 | Spent: 32.9 secs | LR: 0.000278
------------
unit test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing EM: 0.798
INFO:tensorflow:Best EM: 0.807
6 times not improve the best result, therefore stop training