In [1]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/spoken_language_understanding/atis/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
%tensorflow_version 1.x
TensorFlow 1.x selected.
In [3]:
import tensorflow as tf
import tensorflow_hub as hub

import logging
import time
import numpy as np

from sklearn.metrics import classification_report, f1_score, accuracy_score
from pathlib import Path

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 1.15.2
GPU Enabled: True
In [0]:
def get_vocab(vocab_path):
  word2idx = {}
  with open(vocab_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
def data_generator(f_path, params):
  print('Reading', f_path)
  with open(f_path) as f:
    for line in f:
      line = line.rstrip()
      text, slot_intent = line.split('\t')
      words = text.split()[1:-1]
      slot_intent = slot_intent.split()
      slots, intent = slot_intent[1:-1], slot_intent[-1]
      assert len(words) == len(slots)
      
      yield (words, (intent, slots))
In [0]:
def dataset(is_training, params):
  _shapes = ([None], ((), [None]))
  _types = (tf.string, (tf.string, tf.string))
  _pads = ('<pad>', ('_', 'O'))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(1, _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
def clr(step,
        initial_learning_rate,
        maximal_learning_rate,
        step_size,
        scale_fn,
        scale_mode,):
  step = tf.cast(step, tf.float32)
  
  initial_learning_rate = tf.convert_to_tensor(
    initial_learning_rate, name='initial_learning_rate')
  dtype = initial_learning_rate.dtype
  maximal_learning_rate = tf.cast(maximal_learning_rate, dtype)
  step_size = tf.cast(step_size, dtype)
  cycle = tf.floor(1 + step / (2 * step_size))
  x = tf.abs(step / step_size - 2 * cycle + 1)

  mode_step = cycle if scale_mode == 'cycle' else step

  return initial_learning_rate + (
    maximal_learning_rate - initial_learning_rate) * tf.maximum(
      tf.cast(0, dtype), (1 - x)) * scale_fn(mode_step)
In [0]:
def model_fn(features, labels, mode, params):
  is_training = (mode == tf.estimator.ModeKeys.TRAIN)
  
  vocab = tf.contrib.lookup.index_table_from_file(
    params['word_path'], num_oov_buckets=1)
  words = vocab.lookup(features)
  seq_len = tf.count_nonzero(words, 1, dtype=tf.int32)
  
  embedding = np.load(params['vocab_path'])
  embedding = tf.Variable(embedding, name='embedding', dtype=tf.float32)
  x = tf.nn.embedding_lookup(embedding, words)
  
  elmo = hub.Module("https://tfhub.dev/google/elmo/2", trainable=False)
  e = elmo(inputs={'tokens':features, 'sequence_len':seq_len}, signature='tokens', as_dict=True)['lstm_outputs1']
  
  x = tf.concat((x, e), -1)
  x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
  x = tf.layers.dense(x, params['rnn_units'], tf.nn.elu)
  x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
  
  cell_fw = tf.nn.rnn_cell.GRUCell(params['rnn_units'], kernel_initializer=tf.orthogonal_initializer())
  cell_bw = tf.nn.rnn_cell.GRUCell(params['rnn_units'], kernel_initializer=tf.orthogonal_initializer())
  o, (s_fw, s_bw) = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, x, seq_len, dtype=tf.float32)
  x = tf.concat(o, -1)
  
  y_intent = tf.concat((tf.reduce_max(x, 1), s_fw, s_bw), -1)
  y_intent = tf.layers.dropout(y_intent, params['dropout_rate'], training=is_training)
  y_intent = tf.layers.dense(y_intent, params['rnn_units'], tf.nn.elu)
  y_intent = tf.layers.dense(y_intent, params['intent_size'])
  y_slots = tf.layers.dense(x, params['slot_size'])
  
  
  if labels is not None:
    intent, slots = labels
    
    vocab = tf.contrib.lookup.index_table_from_file(
      params['intent_path'], num_oov_buckets=1)
    intent = vocab.lookup(intent)
    vocab = tf.contrib.lookup.index_table_from_file(
      params['slot_path'], num_oov_buckets=1)
    slots = vocab.lookup(slots)
    
    loss_intent = tf.losses.softmax_cross_entropy(
      onehot_labels = tf.one_hot(intent, len(params['intent2idx'])+1),
      logits = y_intent,
      label_smoothing = .2)
    
    weights = tf.cast(tf.sign(slots), tf.float32)
    padding = tf.fill(tf.shape(weights), 1e-2)
    weights = tf.where(tf.equal(weights, 0.), padding, weights)
    loss_slots = tf.compat.v1.losses.softmax_cross_entropy(
      onehot_labels = tf.one_hot(slots, len(params['slot2idx'])+1),
      logits = y_slots,
      weights = tf.cast(weights, tf.float32),
      label_smoothing = .2)
    
    loss_op = loss_intent + loss_slots

  
  if mode == tf.estimator.ModeKeys.TRAIN:
    variables = tf.trainable_variables()
    
    grads = tf.gradients(loss_op, variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    
    global_step=tf.train.get_or_create_global_step()
    decay_lr = clr(
      step = global_step,
      initial_learning_rate = 1e-4,
      maximal_learning_rate = 8e-4,
      step_size = 8 * params['num_samples'] // params['batch_size'],
      scale_fn=lambda x: 1 / (2.0 ** (x - 1)),
      scale_mode = 'cycle',)
    hook = tf.train.LoggingTensorHook({'lr': decay_lr}, every_n_iter=100)
    
    optim = tf.train.AdamOptimizer(decay_lr)
    train_op = optim.apply_gradients(
      zip(grads, variables), global_step=global_step)
    
    return tf.estimator.EstimatorSpec(mode=mode,
                                      loss=loss_op,
                                      train_op=train_op,
                                      training_hooks=[hook],)
  
  
  if mode == tf.estimator.ModeKeys.EVAL:
    return tf.estimator.EstimatorSpec(mode=mode,
                                      loss=loss_op)
  
  
  if mode == tf.estimator.ModeKeys.PREDICT:
    return tf.estimator.EstimatorSpec(mode,
      predictions={'intent': tf.argmax(y_intent, -1),
                   'slots': tf.argmax(y_slots, -1)})
In [0]:
params = {
  'model_dir': '../model/elmo_bigru',
  'log_path': '../log/elmo_bigru.txt',
  'train_path': '../data/atis.train.w-intent.iob',
  'test_path': '../data/atis.test.w-intent.iob',
  'word_path': '../vocab/word.txt',
  'vocab_path': '../vocab/word.npy',
  'intent_path': '../vocab/intent.txt',
  'slot_path': '../vocab/slot.txt',
  'batch_size': 16,
  'num_samples': 4978,
  'rnn_units': 300,
  'dropout_rate': .2,
  'clip_norm': .1,
}
In [0]:
params['word2idx'] = get_vocab(params['word_path'])
params['intent2idx'] = get_vocab(params['intent_path'])
params['slot2idx'] = get_vocab(params['slot_path'])

params['word_size'] = len(params['word2idx']) + 1
params['intent_size'] = len(params['intent2idx']) + 1
params['slot_size'] = len(params['slot2idx']) + 1
In [11]:
# Create directory if not exist
Path(os.path.dirname(params['log_path'])).mkdir(exist_ok=True)
Path(params['model_dir']).mkdir(exist_ok=True, parents=True)

# Logging
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(params['log_path'])
logger.addHandler(fh)
logger.propagate = False

# Create an estimator
config = tf.estimator.RunConfig(
  save_checkpoints_steps = params['num_samples']//params['batch_size'] + 1,
  keep_checkpoint_max=3)

estimator = tf.estimator.Estimator(
  model_fn=model_fn,
  model_dir=params['model_dir'],
  config=config,
  params=params)

slot_best_f1 = .0
intent_acc_with_that = .0
tf.enable_eager_execution()

for n_epoch in range(1, 64+1):
  estimator.train(input_fn=lambda: dataset(is_training=True, params=params))
  
  intent = []
  slots = []
  for w, (i, s) in dataset(is_training=False, params=params):
    intent.append(i.numpy())
    slots.append(s.numpy())
  intent = [i for batch in intent for i in batch]
  intent = [params['intent2idx'].get(str(t, 'utf-8'), len(params['intent2idx'])) for t in intent]
  slots = [j for batch in slots for i in batch for j in i]
  slots = [params['slot2idx'].get(str(s, 'utf-8'), len(params['slot2idx'])) for s in slots]

  predicted = list(estimator.predict(input_fn=lambda: dataset(is_training=False, params=params)))
  y_slots = [j for i in predicted for j in i['slots']]
  y_intent = [i['intent'] for i in predicted]
  
  f1_slots = f1_score(y_true = slots,
                      y_pred = y_slots,
                      labels = list(params['slot2idx'].values()),
                      sample_weight = np.sign(slots),
                      average='micro',)
  
  acc_intent = accuracy_score(intent, y_intent)

  logger.info("Slot F1: {:.3f}, Intent Acc: {:.3f}".format(f1_slots, acc_intent))

  if n_epoch != 1 and n_epoch % 8 == 0:
    logger.info('\n'+classification_report(y_true = intent,
                                          y_pred = y_intent,
                                          labels = list(params['intent2idx'].values()),
                                          target_names = list(params['intent2idx'].keys()),
                                          digits=3))
    logger.info('\n'+classification_report(y_true = slots,
                                          y_pred = y_slots,
                                          labels = list(params['slot2idx'].values()),
                                          target_names = list(params['slot2idx'].keys()),
                                          sample_weight = np.sign(slots),
                                          digits=3))
  
  if f1_slots > slot_best_f1:
    slot_best_f1 = f1_slots
    intent_acc_with_that = acc_intent
    # you can save model here
  logger.info("Best Slot F1: {:.3f}, Intent Acc: {:.3f}".format(slot_best_f1, intent_acc_with_that))
INFO:tensorflow:Using config: {'_model_dir': '../model/elmo_bigru', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 312, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true
graph_options {
  rewrite_options {
    meta_optimizer_iterations: ONE
  }
}
, '_keep_checkpoint_max': 3, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7effbb046390>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.
INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:
The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
  * https://github.com/tensorflow/io (for I/O related ops)
If you depend on functionality not listed there, please file an issue.

WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.
Instructions for updating:
reduction_indices is deprecated, use axis instead
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
WARNING:tensorflow:From <ipython-input-8-2f14ee56ecef>:17: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dropout instead.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/layers/core.py:271: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.__call__` method instead.
WARNING:tensorflow:From <ipython-input-8-2f14ee56ecef>:18: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.Dense instead.
WARNING:tensorflow:From <ipython-input-8-2f14ee56ecef>:21: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.
WARNING:tensorflow:From <ipython-input-8-2f14ee56ecef>:23: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn_cell_impl.py:559: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.add_weight` method instead.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn_cell_impl.py:565: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn_cell_impl.py:575: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn.py:244: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 0 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 3.999641, step = 0
INFO:tensorflow:lr = 1e-04
INFO:tensorflow:global_step/sec: 8.04008
INFO:tensorflow:loss = 2.3888595, step = 100 (12.443 sec)
INFO:tensorflow:lr = 0.00012812372 (12.444 sec)
INFO:tensorflow:global_step/sec: 9.20975
INFO:tensorflow:loss = 1.8700918, step = 200 (10.858 sec)
INFO:tensorflow:lr = 0.0001562475 (10.858 sec)
INFO:tensorflow:global_step/sec: 9.57979
INFO:tensorflow:loss = 2.0352018, step = 300 (10.439 sec)
INFO:tensorflow:lr = 0.00018437125 (10.439 sec)
INFO:tensorflow:Saving checkpoints for 312 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.8097432.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-312
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.710, Intent Acc: 0.896
INFO:tensorflow:Best Slot F1: 0.710, Intent Acc: 0.896
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-312
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/saver.py:1069: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file utilities to get mtimes.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 312 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.0188375, step = 312
INFO:tensorflow:lr = 0.00018774607
INFO:tensorflow:global_step/sec: 8.4952
INFO:tensorflow:loss = 1.6632937, step = 412 (11.775 sec)
INFO:tensorflow:lr = 0.0002158698 (11.777 sec)
INFO:tensorflow:global_step/sec: 9.89137
INFO:tensorflow:loss = 1.660991, step = 512 (10.111 sec)
INFO:tensorflow:lr = 0.00024399356 (10.108 sec)
INFO:tensorflow:global_step/sec: 9.7387
INFO:tensorflow:loss = 1.7954732, step = 612 (10.268 sec)
INFO:tensorflow:lr = 0.00027211732 (10.268 sec)
INFO:tensorflow:Saving checkpoints for 624 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.7237465.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-624
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.869, Intent Acc: 0.950
INFO:tensorflow:Best Slot F1: 0.869, Intent Acc: 0.950
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-624
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 624 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.7340513, step = 624
INFO:tensorflow:lr = 0.00027549214
INFO:tensorflow:global_step/sec: 9.22039
INFO:tensorflow:loss = 1.4264641, step = 724 (10.850 sec)
INFO:tensorflow:lr = 0.00030361587 (10.850 sec)
INFO:tensorflow:global_step/sec: 9.53252
INFO:tensorflow:loss = 1.5037543, step = 824 (10.490 sec)
INFO:tensorflow:lr = 0.00033173963 (10.488 sec)
INFO:tensorflow:global_step/sec: 9.77308
INFO:tensorflow:loss = 1.4711816, step = 924 (10.232 sec)
INFO:tensorflow:lr = 0.00035986342 (10.232 sec)
INFO:tensorflow:Saving checkpoints for 936 into ../model/elmo_bigru/model.ckpt.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/saver.py:963: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
INFO:tensorflow:Loss for final step: 1.4478765.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-936
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.927, Intent Acc: 0.951
INFO:tensorflow:Best Slot F1: 0.927, Intent Acc: 0.951
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-936
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 936 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.5090232, step = 936
INFO:tensorflow:lr = 0.00036323824
INFO:tensorflow:global_step/sec: 8.7178
INFO:tensorflow:loss = 1.4767967, step = 1036 (11.475 sec)
INFO:tensorflow:lr = 0.00039136197 (11.474 sec)
INFO:tensorflow:global_step/sec: 9.85746
INFO:tensorflow:loss = 1.5037491, step = 1136 (10.143 sec)
INFO:tensorflow:lr = 0.00041948573 (10.145 sec)
INFO:tensorflow:global_step/sec: 10.1929
INFO:tensorflow:loss = 1.5372744, step = 1236 (9.810 sec)
INFO:tensorflow:lr = 0.0004476095 (9.810 sec)
INFO:tensorflow:Saving checkpoints for 1248 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.647975.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-1248
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.949, Intent Acc: 0.969
INFO:tensorflow:Best Slot F1: 0.949, Intent Acc: 0.969
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-1248
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 1248 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4843154, step = 1248
INFO:tensorflow:lr = 0.0004509843
INFO:tensorflow:global_step/sec: 8.57005
INFO:tensorflow:loss = 1.3957615, step = 1348 (11.672 sec)
INFO:tensorflow:lr = 0.0004791081 (11.673 sec)
INFO:tensorflow:global_step/sec: 10.0283
INFO:tensorflow:loss = 1.3899381, step = 1448 (9.973 sec)
INFO:tensorflow:lr = 0.0005072318 (9.972 sec)
INFO:tensorflow:global_step/sec: 10.1396
INFO:tensorflow:loss = 1.5094863, step = 1548 (9.862 sec)
INFO:tensorflow:lr = 0.0005353556 (9.862 sec)
INFO:tensorflow:Saving checkpoints for 1560 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4857789.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-1560
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.971
INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.971
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-1560
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 1560 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4448707, step = 1560
INFO:tensorflow:lr = 0.0005387304
INFO:tensorflow:global_step/sec: 8.66366
INFO:tensorflow:loss = 1.352057, step = 1660 (11.546 sec)
INFO:tensorflow:lr = 0.00056685414 (11.546 sec)
INFO:tensorflow:global_step/sec: 9.91993
INFO:tensorflow:loss = 1.5157007, step = 1760 (10.082 sec)
INFO:tensorflow:lr = 0.0005949779 (10.083 sec)
INFO:tensorflow:global_step/sec: 9.45928
INFO:tensorflow:loss = 1.3679348, step = 1860 (10.572 sec)
INFO:tensorflow:lr = 0.00062310166 (10.572 sec)
INFO:tensorflow:Saving checkpoints for 1872 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.7586644.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-1872
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.966
INFO:tensorflow:Best Slot F1: 0.955, Intent Acc: 0.966
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-1872
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 1872 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4636159, step = 1872
INFO:tensorflow:lr = 0.0006264765
INFO:tensorflow:global_step/sec: 8.56964
INFO:tensorflow:loss = 1.526324, step = 1972 (11.671 sec)
INFO:tensorflow:lr = 0.00065460027 (11.671 sec)
INFO:tensorflow:global_step/sec: 9.90571
INFO:tensorflow:loss = 1.4353061, step = 2072 (10.098 sec)
INFO:tensorflow:lr = 0.000682724 (10.098 sec)
INFO:tensorflow:global_step/sec: 9.95821
INFO:tensorflow:loss = 1.4017978, step = 2172 (10.042 sec)
INFO:tensorflow:lr = 0.00071084773 (10.042 sec)
INFO:tensorflow:Saving checkpoints for 2184 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.6374828.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-2184
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.971
INFO:tensorflow:Best Slot F1: 0.955, Intent Acc: 0.971
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-2184
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 2184 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.5233619, step = 2184
INFO:tensorflow:lr = 0.00071422255
INFO:tensorflow:global_step/sec: 8.79258
INFO:tensorflow:loss = 1.4561973, step = 2284 (11.376 sec)
INFO:tensorflow:lr = 0.00074234634 (11.380 sec)
INFO:tensorflow:global_step/sec: 10.1759
INFO:tensorflow:loss = 1.4808575, step = 2384 (9.828 sec)
INFO:tensorflow:lr = 0.00077047007 (9.826 sec)
INFO:tensorflow:global_step/sec: 9.63377
INFO:tensorflow:loss = 1.4015394, step = 2484 (10.380 sec)
INFO:tensorflow:lr = 0.0007985938 (10.379 sec)
INFO:tensorflow:Saving checkpoints for 2496 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.6669892.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-2496
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.957, Intent Acc: 0.971
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.992     0.986       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      0.900     1.000     0.947        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.571     0.727         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.680     0.651     0.652       888
                            weighted avg      0.973     0.976     0.972       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.980     0.999     0.989     704.0
           I-toloc.city_name      0.985     0.989     0.987     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     0.994     0.981     177.0
 B-depart_time.period_of_day      0.992     0.931     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.946     0.614     0.745      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.919     1.000     0.958      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.889     0.857     0.873      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.800     0.414     0.545      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     0.333     0.500       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      0.958     1.000     0.979      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.562     0.429     0.486      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.939     0.969      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.222     0.364       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-depart_time.end_time      1.000     1.000     1.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.958     0.957    3657.0
                   macro avg      0.670     0.635     0.639    3657.0
                weighted avg      0.959     0.958     0.954    3657.0

INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.971
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-2496
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 2496 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.5339439, step = 2496
INFO:tensorflow:lr = 0.00079803134
INFO:tensorflow:global_step/sec: 8.61191
INFO:tensorflow:loss = 1.4880711, step = 2596 (11.614 sec)
INFO:tensorflow:lr = 0.0007699076 (11.614 sec)
INFO:tensorflow:global_step/sec: 10.2188
INFO:tensorflow:loss = 1.4614279, step = 2696 (9.788 sec)
INFO:tensorflow:lr = 0.00074178376 (9.787 sec)
INFO:tensorflow:global_step/sec: 9.64632
INFO:tensorflow:loss = 1.6068125, step = 2796 (10.367 sec)
INFO:tensorflow:lr = 0.00071366003 (10.366 sec)
INFO:tensorflow:Saving checkpoints for 2808 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.6541773.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-2808
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.968
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.968
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-2808
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 2808 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4058206, step = 2808
INFO:tensorflow:lr = 0.0007102852
INFO:tensorflow:global_step/sec: 8.97999
INFO:tensorflow:loss = 1.3525158, step = 2908 (11.140 sec)
INFO:tensorflow:lr = 0.0006821615 (11.140 sec)
INFO:tensorflow:global_step/sec: 9.89318
INFO:tensorflow:loss = 1.3796324, step = 3008 (10.108 sec)
INFO:tensorflow:lr = 0.0006540377 (10.109 sec)
INFO:tensorflow:global_step/sec: 9.68312
INFO:tensorflow:loss = 1.4304125, step = 3108 (10.324 sec)
INFO:tensorflow:lr = 0.00062591396 (10.325 sec)
INFO:tensorflow:Saving checkpoints for 3120 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5020589.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-3120
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.968
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-3120
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 3120 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4006892, step = 3120
INFO:tensorflow:lr = 0.00062253914
INFO:tensorflow:global_step/sec: 9.32344
INFO:tensorflow:loss = 1.4769299, step = 3220 (10.730 sec)
INFO:tensorflow:lr = 0.0005944154 (10.730 sec)
INFO:tensorflow:global_step/sec: 9.95192
INFO:tensorflow:loss = 1.4246507, step = 3320 (10.049 sec)
INFO:tensorflow:lr = 0.0005662916 (10.051 sec)
INFO:tensorflow:global_step/sec: 8.77668
INFO:tensorflow:loss = 1.4651399, step = 3420 (11.392 sec)
INFO:tensorflow:lr = 0.0005381679 (11.390 sec)
INFO:tensorflow:Saving checkpoints for 3432 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5239291.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-3432
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.968
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-3432
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 3432 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3482316, step = 3432
INFO:tensorflow:lr = 0.0005347931
INFO:tensorflow:global_step/sec: 9.33893
INFO:tensorflow:loss = 1.3795862, step = 3532 (10.712 sec)
INFO:tensorflow:lr = 0.00050666934 (10.713 sec)
INFO:tensorflow:global_step/sec: 9.62542
INFO:tensorflow:loss = 1.3636954, step = 3632 (10.387 sec)
INFO:tensorflow:lr = 0.00047854552 (10.388 sec)
INFO:tensorflow:global_step/sec: 10.2902
INFO:tensorflow:loss = 1.4447916, step = 3732 (9.719 sec)
INFO:tensorflow:lr = 0.00045042182 (9.719 sec)
INFO:tensorflow:Saving checkpoints for 3744 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.410874.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-3744
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-3744
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 3744 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3958027, step = 3744
INFO:tensorflow:lr = 0.000447047
INFO:tensorflow:global_step/sec: 7.05144
INFO:tensorflow:loss = 1.4465686, step = 3844 (14.185 sec)
INFO:tensorflow:lr = 0.00041892327 (14.184 sec)
INFO:tensorflow:global_step/sec: 9.31107
INFO:tensorflow:loss = 1.3941537, step = 3944 (10.737 sec)
INFO:tensorflow:lr = 0.00039079945 (10.740 sec)
INFO:tensorflow:global_step/sec: 9.93101
INFO:tensorflow:loss = 1.440351, step = 4044 (10.072 sec)
INFO:tensorflow:lr = 0.00036267572 (10.069 sec)
INFO:tensorflow:Saving checkpoints for 4056 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4304112.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-4056
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.959, Intent Acc: 0.978
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-4056
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 4056 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4213464, step = 4056
INFO:tensorflow:lr = 0.0003593009
INFO:tensorflow:global_step/sec: 9.10164
INFO:tensorflow:loss = 1.4117901, step = 4156 (10.989 sec)
INFO:tensorflow:lr = 0.00033117717 (10.990 sec)
INFO:tensorflow:global_step/sec: 9.89487
INFO:tensorflow:loss = 1.3885187, step = 4256 (10.107 sec)
INFO:tensorflow:lr = 0.00030305336 (10.110 sec)
INFO:tensorflow:global_step/sec: 9.70224
INFO:tensorflow:loss = 1.2804167, step = 4356 (10.307 sec)
INFO:tensorflow:lr = 0.00027492965 (10.305 sec)
INFO:tensorflow:Saving checkpoints for 4368 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.447479.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-4368
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-4368
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 4368 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4394464, step = 4368
INFO:tensorflow:lr = 0.00027155483
INFO:tensorflow:global_step/sec: 9.45474
INFO:tensorflow:loss = 1.537283, step = 4468 (10.580 sec)
INFO:tensorflow:lr = 0.0002434311 (10.577 sec)
INFO:tensorflow:global_step/sec: 9.78966
INFO:tensorflow:loss = 1.4285619, step = 4568 (10.216 sec)
INFO:tensorflow:lr = 0.00021530729 (10.216 sec)
INFO:tensorflow:global_step/sec: 10.0028
INFO:tensorflow:loss = 1.428333, step = 4668 (9.996 sec)
INFO:tensorflow:lr = 0.00018718356 (9.997 sec)
INFO:tensorflow:Saving checkpoints for 4680 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.635045.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-4680
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-4680
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 4680 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3047377, step = 4680
INFO:tensorflow:lr = 0.00018380873
INFO:tensorflow:global_step/sec: 8.67285
INFO:tensorflow:loss = 1.4502094, step = 4780 (11.534 sec)
INFO:tensorflow:lr = 0.00015568502 (11.533 sec)
INFO:tensorflow:global_step/sec: 9.28242
INFO:tensorflow:loss = 1.3941047, step = 4880 (10.774 sec)
INFO:tensorflow:lr = 0.00012756122 (10.773 sec)
INFO:tensorflow:global_step/sec: 8.27923
INFO:tensorflow:loss = 1.3588406, step = 4980 (12.077 sec)
INFO:tensorflow:lr = 0.00010028121 (12.078 sec)
INFO:tensorflow:Saving checkpoints for 4992 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4580553.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-4992
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.978
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.978     0.983     0.980       888
                               macro avg      0.683     0.678     0.670       888
                            weighted avg      0.979     0.983     0.980       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.996     0.984     716.0
         B-fromloc.city_name      0.985     1.000     0.992     704.0
           I-toloc.city_name      0.957     0.996     0.976     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.892     0.579     0.702      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     1.000     1.000      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.433     0.605      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     1.000     1.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.960    3657.0
                   macro avg      0.701     0.676     0.677    3657.0
                weighted avg      0.964     0.961     0.957    3657.0

INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-4992
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 4992 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3538101, step = 4992
INFO:tensorflow:lr = 0.00010196867
INFO:tensorflow:global_step/sec: 9.37402
INFO:tensorflow:loss = 1.3983464, step = 5092 (10.672 sec)
INFO:tensorflow:lr = 0.00011603057 (10.674 sec)
INFO:tensorflow:global_step/sec: 9.80098
INFO:tensorflow:loss = 1.5250366, step = 5192 (10.203 sec)
INFO:tensorflow:lr = 0.00013009239 (10.202 sec)
INFO:tensorflow:global_step/sec: 9.83411
INFO:tensorflow:loss = 1.4319367, step = 5292 (10.169 sec)
INFO:tensorflow:lr = 0.0001441543 (10.168 sec)
INFO:tensorflow:Saving checkpoints for 5304 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5401801.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-5304
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.978
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-5304
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 5304 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3343503, step = 5304
INFO:tensorflow:lr = 0.00014584175
INFO:tensorflow:global_step/sec: 9.25774
INFO:tensorflow:loss = 1.3968102, step = 5404 (10.806 sec)
INFO:tensorflow:lr = 0.00015990357 (10.807 sec)
INFO:tensorflow:global_step/sec: 9.69502
INFO:tensorflow:loss = 1.396074, step = 5504 (10.315 sec)
INFO:tensorflow:lr = 0.00017396547 (10.313 sec)
INFO:tensorflow:global_step/sec: 9.34137
INFO:tensorflow:loss = 1.3761456, step = 5604 (10.969 sec)
INFO:tensorflow:lr = 0.00018802729 (10.972 sec)
INFO:tensorflow:Saving checkpoints for 5616 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4086791.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-5616
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.978
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-5616
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 5616 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4908104, step = 5616
INFO:tensorflow:lr = 0.00018971474
INFO:tensorflow:global_step/sec: 9.30127
INFO:tensorflow:loss = 1.339049, step = 5716 (10.753 sec)
INFO:tensorflow:lr = 0.00020377665 (10.753 sec)
INFO:tensorflow:global_step/sec: 9.66277
INFO:tensorflow:loss = 1.4201441, step = 5816 (25.284 sec)
INFO:tensorflow:lr = 0.00021783847 (25.289 sec)
INFO:tensorflow:global_step/sec: 3.90206
INFO:tensorflow:loss = 1.4243789, step = 5916 (10.696 sec)
INFO:tensorflow:lr = 0.00023190037 (10.693 sec)
INFO:tensorflow:Saving checkpoints for 5928 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5755584.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-5928
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.978
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-5928
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 5928 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4490582, step = 5928
INFO:tensorflow:lr = 0.00023358784
INFO:tensorflow:global_step/sec: 9.30921
INFO:tensorflow:loss = 1.3139141, step = 6028 (10.745 sec)
INFO:tensorflow:lr = 0.00024764964 (10.747 sec)
INFO:tensorflow:global_step/sec: 9.20629
INFO:tensorflow:loss = 1.3624984, step = 6128 (10.862 sec)
INFO:tensorflow:lr = 0.00026171157 (10.861 sec)
INFO:tensorflow:global_step/sec: 9.86758
INFO:tensorflow:loss = 1.357186, step = 6228 (10.133 sec)
INFO:tensorflow:lr = 0.00027577337 (10.134 sec)
INFO:tensorflow:Saving checkpoints for 6240 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.6436534.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-6240
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.978
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-6240
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 6240 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4237194, step = 6240
INFO:tensorflow:lr = 0.00027746081
INFO:tensorflow:global_step/sec: 9.35895
INFO:tensorflow:loss = 1.426637, step = 6340 (10.687 sec)
INFO:tensorflow:lr = 0.00029152274 (10.686 sec)
INFO:tensorflow:global_step/sec: 9.75338
INFO:tensorflow:loss = 1.4954264, step = 6440 (10.254 sec)
INFO:tensorflow:lr = 0.00030558454 (10.253 sec)
INFO:tensorflow:global_step/sec: 9.92346
INFO:tensorflow:loss = 1.4098217, step = 6540 (10.076 sec)
INFO:tensorflow:lr = 0.00031964647 (10.076 sec)
INFO:tensorflow:Saving checkpoints for 6552 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.6194656.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-6552
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.978
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-6552
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 6552 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.342941, step = 6552
INFO:tensorflow:lr = 0.0003213339
INFO:tensorflow:global_step/sec: 8.99005
INFO:tensorflow:loss = 1.4306874, step = 6652 (11.126 sec)
INFO:tensorflow:lr = 0.00033539574 (11.126 sec)
INFO:tensorflow:global_step/sec: 9.93931
INFO:tensorflow:loss = 1.4569489, step = 6752 (10.063 sec)
INFO:tensorflow:lr = 0.00034945764 (10.062 sec)
INFO:tensorflow:global_step/sec: 7.59812
INFO:tensorflow:loss = 1.3535345, step = 6852 (13.163 sec)
INFO:tensorflow:lr = 0.00036351947 (13.163 sec)
INFO:tensorflow:Saving checkpoints for 6864 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.3413649.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-6864
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-6864
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 6864 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4585054, step = 6864
INFO:tensorflow:lr = 0.0003652069
INFO:tensorflow:global_step/sec: 9.2698
INFO:tensorflow:loss = 1.406852, step = 6964 (10.790 sec)
INFO:tensorflow:lr = 0.0003792688 (10.789 sec)
INFO:tensorflow:global_step/sec: 9.81532
INFO:tensorflow:loss = 1.3885676, step = 7064 (10.190 sec)
INFO:tensorflow:lr = 0.00039333064 (10.190 sec)
INFO:tensorflow:global_step/sec: 10.0539
INFO:tensorflow:loss = 1.4385337, step = 7164 (9.946 sec)
INFO:tensorflow:lr = 0.00040739254 (9.946 sec)
INFO:tensorflow:Saving checkpoints for 7176 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4175079.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-7176
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-7176
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 7176 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3749772, step = 7176
INFO:tensorflow:lr = 0.00040908
INFO:tensorflow:global_step/sec: 9.07259
INFO:tensorflow:loss = 1.4185073, step = 7276 (11.026 sec)
INFO:tensorflow:lr = 0.0004231418 (11.025 sec)
INFO:tensorflow:global_step/sec: 10.1395
INFO:tensorflow:loss = 1.3918288, step = 7376 (9.863 sec)
INFO:tensorflow:lr = 0.0004372037 (9.862 sec)
INFO:tensorflow:global_step/sec: 9.85949
INFO:tensorflow:loss = 1.4393605, step = 7476 (10.143 sec)
INFO:tensorflow:lr = 0.00044873444 (10.142 sec)
INFO:tensorflow:Saving checkpoints for 7488 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.530221.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-7488
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.976
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.991     0.987       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.976     0.982     0.979       888
                               macro avg      0.682     0.678     0.669       888
                            weighted avg      0.978     0.982     0.978       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.996     0.983     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.957     0.996     0.976     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.914     0.561     0.696      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     1.000     1.000      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.933     0.467     0.622      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     1.000     1.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.960    3657.0
                   macro avg      0.702     0.676     0.677    3657.0
                weighted avg      0.963     0.961     0.957    3657.0

INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-7488
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 7488 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4039026, step = 7488
INFO:tensorflow:lr = 0.000447047
INFO:tensorflow:global_step/sec: 8.82955
INFO:tensorflow:loss = 1.2839608, step = 7588 (11.327 sec)
INFO:tensorflow:lr = 0.00043298508 (11.327 sec)
INFO:tensorflow:global_step/sec: 10.6101
INFO:tensorflow:loss = 1.3488975, step = 7688 (9.427 sec)
INFO:tensorflow:lr = 0.00041892327 (9.428 sec)
INFO:tensorflow:global_step/sec: 9.73681
INFO:tensorflow:loss = 1.4587625, step = 7788 (10.278 sec)
INFO:tensorflow:lr = 0.00040486135 (10.277 sec)
INFO:tensorflow:Saving checkpoints for 7800 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4976358.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-7800
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.978
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-7800
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 7800 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4646809, step = 7800
INFO:tensorflow:lr = 0.0004031739
INFO:tensorflow:global_step/sec: 9.05848
INFO:tensorflow:loss = 1.3482566, step = 7900 (11.044 sec)
INFO:tensorflow:lr = 0.00038911207 (11.044 sec)
INFO:tensorflow:global_step/sec: 9.75261
INFO:tensorflow:loss = 1.4880786, step = 8000 (10.251 sec)
INFO:tensorflow:lr = 0.00037505018 (10.251 sec)
INFO:tensorflow:global_step/sec: 7.06527
INFO:tensorflow:loss = 1.359579, step = 8100 (14.157 sec)
INFO:tensorflow:lr = 0.00036098837 (14.157 sec)
INFO:tensorflow:Saving checkpoints for 8112 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5920289.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-8112
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-8112
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 8112 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4246266, step = 8112
INFO:tensorflow:lr = 0.0003593009
INFO:tensorflow:global_step/sec: 9.17504
INFO:tensorflow:loss = 1.3861287, step = 8212 (10.903 sec)
INFO:tensorflow:lr = 0.000345239 (10.903 sec)
INFO:tensorflow:global_step/sec: 10.2916
INFO:tensorflow:loss = 1.4733317, step = 8312 (9.716 sec)
INFO:tensorflow:lr = 0.00033117717 (9.718 sec)
INFO:tensorflow:global_step/sec: 9.69568
INFO:tensorflow:loss = 1.386755, step = 8412 (10.314 sec)
INFO:tensorflow:lr = 0.00031711528 (10.313 sec)
INFO:tensorflow:Saving checkpoints for 8424 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5367917.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-8424
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-8424
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 8424 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3304763, step = 8424
INFO:tensorflow:lr = 0.0003154278
INFO:tensorflow:global_step/sec: 9.11365
INFO:tensorflow:loss = 1.3352221, step = 8524 (10.976 sec)
INFO:tensorflow:lr = 0.000301366 (10.977 sec)
INFO:tensorflow:global_step/sec: 9.95781
INFO:tensorflow:loss = 1.2977567, step = 8624 (10.043 sec)
INFO:tensorflow:lr = 0.0002873041 (10.042 sec)
INFO:tensorflow:global_step/sec: 10.3288
INFO:tensorflow:loss = 1.4254208, step = 8724 (9.683 sec)
INFO:tensorflow:lr = 0.00027324227 (9.683 sec)
INFO:tensorflow:Saving checkpoints for 8736 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.7526174.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-8736
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-8736
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 8736 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4905977, step = 8736
INFO:tensorflow:lr = 0.00027155483
INFO:tensorflow:global_step/sec: 9.05945
INFO:tensorflow:loss = 1.4527041, step = 8836 (11.043 sec)
INFO:tensorflow:lr = 0.0002574929 (11.044 sec)
INFO:tensorflow:global_step/sec: 9.98964
INFO:tensorflow:loss = 1.3823752, step = 8936 (10.007 sec)
INFO:tensorflow:lr = 0.0002434311 (10.009 sec)
INFO:tensorflow:global_step/sec: 9.92079
INFO:tensorflow:loss = 1.3885853, step = 9036 (10.083 sec)
INFO:tensorflow:lr = 0.00022936918 (10.080 sec)
INFO:tensorflow:Saving checkpoints for 9048 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5270206.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-9048
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-9048
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 9048 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4371688, step = 9048
INFO:tensorflow:lr = 0.00022768174
INFO:tensorflow:global_step/sec: 8.95421
INFO:tensorflow:loss = 1.462599, step = 9148 (11.173 sec)
INFO:tensorflow:lr = 0.00021361992 (11.171 sec)
INFO:tensorflow:global_step/sec: 10.1543
INFO:tensorflow:loss = 1.3865968, step = 9248 (9.848 sec)
INFO:tensorflow:lr = 0.00019955801 (9.849 sec)
INFO:tensorflow:global_step/sec: 9.88997
INFO:tensorflow:loss = 1.3967034, step = 9348 (10.110 sec)
INFO:tensorflow:lr = 0.0001854962 (10.112 sec)
INFO:tensorflow:Saving checkpoints for 9360 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5900289.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-9360
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-9360
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 9360 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4742596, step = 9360
INFO:tensorflow:lr = 0.00018380873
INFO:tensorflow:global_step/sec: 9.31627
INFO:tensorflow:loss = 1.4506983, step = 9460 (10.735 sec)
INFO:tensorflow:lr = 0.00016974684 (10.736 sec)
INFO:tensorflow:global_step/sec: 9.88915
INFO:tensorflow:loss = 1.3342823, step = 9560 (10.115 sec)
INFO:tensorflow:lr = 0.00015568502 (10.113 sec)
INFO:tensorflow:global_step/sec: 10.0048
INFO:tensorflow:loss = 1.3753715, step = 9660 (9.995 sec)
INFO:tensorflow:lr = 0.00014162311 (9.997 sec)
INFO:tensorflow:Saving checkpoints for 9672 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5192186.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-9672
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-9672
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 9672 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3269039, step = 9672
INFO:tensorflow:lr = 0.00013993566
INFO:tensorflow:global_step/sec: 7.33744
INFO:tensorflow:loss = 1.3606923, step = 9772 (13.637 sec)
INFO:tensorflow:lr = 0.00012587383 (13.633 sec)
INFO:tensorflow:global_step/sec: 9.91745
INFO:tensorflow:loss = 1.3477261, step = 9872 (10.079 sec)
INFO:tensorflow:lr = 0.00011181193 (10.079 sec)
INFO:tensorflow:global_step/sec: 9.86229
INFO:tensorflow:loss = 1.3161685, step = 9972 (10.139 sec)
INFO:tensorflow:lr = 0.00010112494 (10.139 sec)
INFO:tensorflow:Saving checkpoints for 9984 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4549196.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-9984
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.976
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.991     0.987       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.976     0.982     0.979       888
                               macro avg      0.682     0.678     0.669       888
                            weighted avg      0.978     0.982     0.978       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.996     0.983     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.957     0.996     0.976     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.889     0.561     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     1.000     1.000      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.400     0.571      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     1.000     1.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.961     0.960    3657.0
                   macro avg      0.702     0.675     0.676    3657.0
                weighted avg      0.963     0.961     0.957    3657.0

INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-9984
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 9984 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4513614, step = 9984
INFO:tensorflow:lr = 0.00010196867
INFO:tensorflow:global_step/sec: 8.97816
INFO:tensorflow:loss = 1.389141, step = 10084 (11.142 sec)
INFO:tensorflow:lr = 0.00010899962 (11.142 sec)
INFO:tensorflow:global_step/sec: 9.92417
INFO:tensorflow:loss = 1.4163504, step = 10184 (10.077 sec)
INFO:tensorflow:lr = 0.00011603057 (10.076 sec)
INFO:tensorflow:global_step/sec: 9.98561
INFO:tensorflow:loss = 1.4134475, step = 10284 (10.013 sec)
INFO:tensorflow:lr = 0.00012306144 (10.016 sec)
INFO:tensorflow:Saving checkpoints for 10296 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.3089747.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-10296
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-10296
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 10296 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4251472, step = 10296
INFO:tensorflow:lr = 0.00012390516
INFO:tensorflow:global_step/sec: 9.19652
INFO:tensorflow:loss = 1.4969662, step = 10396 (10.878 sec)
INFO:tensorflow:lr = 0.00013093613 (10.879 sec)
INFO:tensorflow:global_step/sec: 9.81867
INFO:tensorflow:loss = 1.5439547, step = 10496 (10.184 sec)
INFO:tensorflow:lr = 0.00013796707 (10.184 sec)
INFO:tensorflow:global_step/sec: 10.0368
INFO:tensorflow:loss = 1.2855426, step = 10596 (9.965 sec)
INFO:tensorflow:lr = 0.00014499802 (9.964 sec)
INFO:tensorflow:Saving checkpoints for 10608 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4862626.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-10608
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-10608
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 10608 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4431376, step = 10608
INFO:tensorflow:lr = 0.00014584175
INFO:tensorflow:global_step/sec: 8.93785
INFO:tensorflow:loss = 1.3635898, step = 10708 (11.190 sec)
INFO:tensorflow:lr = 0.00015287261 (11.191 sec)
INFO:tensorflow:global_step/sec: 9.73569
INFO:tensorflow:loss = 1.4345067, step = 10808 (10.272 sec)
INFO:tensorflow:lr = 0.00015990357 (10.270 sec)
INFO:tensorflow:global_step/sec: 9.44245
INFO:tensorflow:loss = 1.4790363, step = 10908 (10.592 sec)
INFO:tensorflow:lr = 0.00016693452 (10.592 sec)
INFO:tensorflow:Saving checkpoints for 10920 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4832919.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-10920
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-10920
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 10920 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.2415458, step = 10920
INFO:tensorflow:lr = 0.00016777826
INFO:tensorflow:global_step/sec: 9.31244
INFO:tensorflow:loss = 1.4477246, step = 11020 (10.743 sec)
INFO:tensorflow:lr = 0.0001748092 (10.743 sec)
INFO:tensorflow:global_step/sec: 10.2044
INFO:tensorflow:loss = 1.3923736, step = 11120 (9.801 sec)
INFO:tensorflow:lr = 0.00018184006 (9.803 sec)
INFO:tensorflow:global_step/sec: 9.69639
INFO:tensorflow:loss = 1.414816, step = 11220 (10.311 sec)
INFO:tensorflow:lr = 0.00018887102 (10.309 sec)
INFO:tensorflow:Saving checkpoints for 11232 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4031807.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-11232
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-11232
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 11232 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4679098, step = 11232
INFO:tensorflow:lr = 0.00018971474
INFO:tensorflow:global_step/sec: 8.97691
INFO:tensorflow:loss = 1.4135969, step = 11332 (11.143 sec)
INFO:tensorflow:lr = 0.0001967457 (11.143 sec)
INFO:tensorflow:global_step/sec: 9.87558
INFO:tensorflow:loss = 1.4685216, step = 11432 (10.124 sec)
INFO:tensorflow:lr = 0.00020377665 (10.126 sec)
INFO:tensorflow:global_step/sec: 9.98601
INFO:tensorflow:loss = 1.319553, step = 11532 (10.017 sec)
INFO:tensorflow:lr = 0.0002108076 (10.016 sec)
INFO:tensorflow:Saving checkpoints for 11544 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5751863.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-11544
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-11544
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 11544 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4138879, step = 11544
INFO:tensorflow:lr = 0.00021165125
INFO:tensorflow:global_step/sec: 8.30702
INFO:tensorflow:loss = 1.4211999, step = 11644 (12.042 sec)
INFO:tensorflow:lr = 0.0002186822 (12.041 sec)
INFO:tensorflow:global_step/sec: 10.0729
INFO:tensorflow:loss = 1.4204689, step = 11744 (9.929 sec)
INFO:tensorflow:lr = 0.00022571316 (9.928 sec)
INFO:tensorflow:global_step/sec: 9.79003
INFO:tensorflow:loss = 1.3147762, step = 11844 (10.211 sec)
INFO:tensorflow:lr = 0.00023274409 (10.212 sec)
INFO:tensorflow:Saving checkpoints for 11856 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5654552.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-11856
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-11856
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 11856 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.270679, step = 11856
INFO:tensorflow:lr = 0.00023358784
INFO:tensorflow:global_step/sec: 9.07161
INFO:tensorflow:loss = 1.3487439, step = 11956 (11.028 sec)
INFO:tensorflow:lr = 0.00024061871 (11.026 sec)
INFO:tensorflow:global_step/sec: 10.1068
INFO:tensorflow:loss = 1.4413215, step = 12056 (9.893 sec)
INFO:tensorflow:lr = 0.00024764964 (9.893 sec)
INFO:tensorflow:global_step/sec: 9.67451
INFO:tensorflow:loss = 1.4958509, step = 12156 (10.337 sec)
INFO:tensorflow:lr = 0.0002546806 (10.338 sec)
INFO:tensorflow:Saving checkpoints for 12168 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.3686566.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-12168
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-12168
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 12168 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3869534, step = 12168
INFO:tensorflow:lr = 0.00025552433
INFO:tensorflow:global_step/sec: 9.16238
INFO:tensorflow:loss = 1.4068757, step = 12268 (10.916 sec)
INFO:tensorflow:lr = 0.0002625553 (10.917 sec)
INFO:tensorflow:global_step/sec: 9.53437
INFO:tensorflow:loss = 1.5716747, step = 12368 (10.491 sec)
INFO:tensorflow:lr = 0.00026958616 (10.491 sec)
INFO:tensorflow:global_step/sec: 9.78193
INFO:tensorflow:loss = 1.415336, step = 12468 (10.227 sec)
INFO:tensorflow:lr = 0.00027338287 (10.227 sec)
INFO:tensorflow:Saving checkpoints for 12480 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5568521.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-12480
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.974
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.989     0.986       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      0.833     0.833     0.833         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.672     0.672     0.662       888
                            weighted avg      0.976     0.980     0.976       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.981     1.000     0.990     704.0
           I-toloc.city_name      0.960     0.996     0.978     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.892     0.579     0.702      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     1.000     1.000      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.467     0.636      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     1.000     1.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     1.000     1.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.961    3657.0
                   macro avg      0.702     0.678     0.679    3657.0
                weighted avg      0.964     0.961     0.958    3657.0

INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-12480
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 12480 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4291271, step = 12480
INFO:tensorflow:lr = 0.00027253915
INFO:tensorflow:global_step/sec: 9.00107
INFO:tensorflow:loss = 1.4653343, step = 12580 (11.114 sec)
INFO:tensorflow:lr = 0.0002655082 (11.115 sec)
INFO:tensorflow:global_step/sec: 9.30082
INFO:tensorflow:loss = 1.2840335, step = 12680 (10.752 sec)
INFO:tensorflow:lr = 0.00025847726 (10.753 sec)
INFO:tensorflow:global_step/sec: 9.9105
INFO:tensorflow:loss = 1.403368, step = 12780 (10.090 sec)
INFO:tensorflow:lr = 0.0002514463 (10.090 sec)
INFO:tensorflow:Saving checkpoints for 12792 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4869626.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-12792
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-12792
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 12792 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3180772, step = 12792
INFO:tensorflow:lr = 0.00025060266
INFO:tensorflow:global_step/sec: 8.91387
INFO:tensorflow:loss = 1.4678496, step = 12892 (11.220 sec)
INFO:tensorflow:lr = 0.0002435717 (11.221 sec)
INFO:tensorflow:global_step/sec: 9.67882
INFO:tensorflow:loss = 1.3325438, step = 12992 (10.334 sec)
INFO:tensorflow:lr = 0.00023654074 (10.335 sec)
INFO:tensorflow:global_step/sec: 10.0386
INFO:tensorflow:loss = 1.3150475, step = 13092 (9.962 sec)
INFO:tensorflow:lr = 0.00022950981 (9.961 sec)
INFO:tensorflow:Saving checkpoints for 13104 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.7593229.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-13104
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-13104
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 13104 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.45171, step = 13104
INFO:tensorflow:lr = 0.00022866606
INFO:tensorflow:global_step/sec: 7.02182
INFO:tensorflow:loss = 1.368976, step = 13204 (14.243 sec)
INFO:tensorflow:lr = 0.0002216352 (14.243 sec)
INFO:tensorflow:global_step/sec: 8.83156
INFO:tensorflow:loss = 1.3487523, step = 13304 (11.326 sec)
INFO:tensorflow:lr = 0.00021460425 (11.325 sec)
INFO:tensorflow:global_step/sec: 7.3657
INFO:tensorflow:loss = 1.2702498, step = 13404 (13.582 sec)
INFO:tensorflow:lr = 0.00020757329 (13.582 sec)
INFO:tensorflow:Saving checkpoints for 13416 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.6175996.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-13416
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-13416
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 13416 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.261663, step = 13416
INFO:tensorflow:lr = 0.00020672957
INFO:tensorflow:global_step/sec: 9.07354
INFO:tensorflow:loss = 1.2987804, step = 13516 (11.025 sec)
INFO:tensorflow:lr = 0.00019969861 (11.024 sec)
INFO:tensorflow:global_step/sec: 10.1011
INFO:tensorflow:loss = 1.3888392, step = 13616 (9.900 sec)
INFO:tensorflow:lr = 0.00019266775 (9.901 sec)
INFO:tensorflow:global_step/sec: 10.0105
INFO:tensorflow:loss = 1.3347571, step = 13716 (9.992 sec)
INFO:tensorflow:lr = 0.0001856368 (9.992 sec)
INFO:tensorflow:Saving checkpoints for 13728 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.36664.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-13728
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-13728
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 13728 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4607565, step = 13728
INFO:tensorflow:lr = 0.00018479308
INFO:tensorflow:global_step/sec: 8.792
INFO:tensorflow:loss = 1.2576694, step = 13828 (11.378 sec)
INFO:tensorflow:lr = 0.00017776212 (11.378 sec)
INFO:tensorflow:global_step/sec: 9.94255
INFO:tensorflow:loss = 1.4379426, step = 13928 (10.058 sec)
INFO:tensorflow:lr = 0.00017073116 (10.059 sec)
INFO:tensorflow:global_step/sec: 10.013
INFO:tensorflow:loss = 1.4003817, step = 14028 (9.986 sec)
INFO:tensorflow:lr = 0.00016370023 (9.989 sec)
INFO:tensorflow:Saving checkpoints for 14040 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.6824641.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-14040
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-14040
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 14040 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4655322, step = 14040
INFO:tensorflow:lr = 0.00016285657
INFO:tensorflow:global_step/sec: 8.81451
INFO:tensorflow:loss = 1.3493655, step = 14140 (11.349 sec)
INFO:tensorflow:lr = 0.00015582562 (11.350 sec)
INFO:tensorflow:global_step/sec: 10.3058
INFO:tensorflow:loss = 1.4032513, step = 14240 (9.702 sec)
INFO:tensorflow:lr = 0.00014879467 (9.703 sec)
INFO:tensorflow:global_step/sec: 9.81079
INFO:tensorflow:loss = 1.2707402, step = 14340 (10.194 sec)
INFO:tensorflow:lr = 0.00014176371 (10.194 sec)
INFO:tensorflow:Saving checkpoints for 14352 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4578941.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-14352
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-14352
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 14352 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4214216, step = 14352
INFO:tensorflow:lr = 0.00014091999
INFO:tensorflow:global_step/sec: 9.48444
INFO:tensorflow:loss = 1.4259878, step = 14452 (10.548 sec)
INFO:tensorflow:lr = 0.00013388912 (10.546 sec)
INFO:tensorflow:global_step/sec: 9.72096
INFO:tensorflow:loss = 1.4009758, step = 14552 (10.287 sec)
INFO:tensorflow:lr = 0.00012685817 (10.288 sec)
INFO:tensorflow:global_step/sec: 9.88248
INFO:tensorflow:loss = 1.4998803, step = 14652 (10.118 sec)
INFO:tensorflow:lr = 0.00011982722 (10.117 sec)
INFO:tensorflow:Saving checkpoints for 14664 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4549606.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-14664
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-14664
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 14664 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4097495, step = 14664
INFO:tensorflow:lr = 0.000118983495
INFO:tensorflow:global_step/sec: 9.4595
INFO:tensorflow:loss = 1.4379569, step = 14764 (10.575 sec)
INFO:tensorflow:lr = 0.00011195254 (10.574 sec)
INFO:tensorflow:global_step/sec: 9.66093
INFO:tensorflow:loss = 1.2376904, step = 14864 (10.351 sec)
INFO:tensorflow:lr = 0.00010492167 (10.352 sec)
INFO:tensorflow:global_step/sec: 9.57684
INFO:tensorflow:loss = 1.3348329, step = 14964 (10.442 sec)
INFO:tensorflow:lr = 0.00010105463 (10.442 sec)
INFO:tensorflow:Saving checkpoints for 14976 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5517815.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-14976
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.991     0.987       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.680     0.673     0.664       888
                            weighted avg      0.978     0.981     0.978       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.996     0.984     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.957     0.996     0.976     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.893     0.893     0.893      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     1.000     1.000      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.400     0.571      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.615     0.381     0.471      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.909     0.952      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     1.000     1.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     1.000     1.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.962     0.961    3657.0
                   macro avg      0.702     0.678     0.679    3657.0
                weighted avg      0.963     0.962     0.958    3657.0

INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-14976
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 14976 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3492382, step = 14976
INFO:tensorflow:lr = 0.0001014765
INFO:tensorflow:global_step/sec: 9.11526
INFO:tensorflow:loss = 1.3974228, step = 15076 (10.975 sec)
INFO:tensorflow:lr = 0.000104991974 (10.977 sec)
INFO:tensorflow:global_step/sec: 9.85589
INFO:tensorflow:loss = 1.5027564, step = 15176 (10.146 sec)
INFO:tensorflow:lr = 0.000108507455 (10.145 sec)
INFO:tensorflow:global_step/sec: 9.87238
INFO:tensorflow:loss = 1.3273671, step = 15276 (10.127 sec)
INFO:tensorflow:lr = 0.00011202293 (10.130 sec)
INFO:tensorflow:Saving checkpoints for 15288 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.7847414.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-15288
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-15288
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 15288 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3502831, step = 15288
INFO:tensorflow:lr = 0.00011244475
INFO:tensorflow:global_step/sec: 9.34187
INFO:tensorflow:loss = 1.3784354, step = 15388 (10.707 sec)
INFO:tensorflow:lr = 0.000115960225 (10.708 sec)
INFO:tensorflow:global_step/sec: 9.78782
INFO:tensorflow:loss = 1.4704087, step = 15488 (10.217 sec)
INFO:tensorflow:lr = 0.0001194757 (10.217 sec)
INFO:tensorflow:global_step/sec: 9.8924
INFO:tensorflow:loss = 1.4717734, step = 15588 (10.109 sec)
INFO:tensorflow:lr = 0.00012299117 (10.108 sec)
INFO:tensorflow:Saving checkpoints for 15600 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.6340075.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-15600
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-15600
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 15600 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4139618, step = 15600
INFO:tensorflow:lr = 0.00012341305
INFO:tensorflow:global_step/sec: 8.99061
INFO:tensorflow:loss = 1.3827469, step = 15700 (11.127 sec)
INFO:tensorflow:lr = 0.00012692847 (11.129 sec)
INFO:tensorflow:global_step/sec: 9.8712
INFO:tensorflow:loss = 1.3810418, step = 15800 (10.129 sec)
INFO:tensorflow:lr = 0.00013044395 (10.129 sec)
INFO:tensorflow:global_step/sec: 10.141
INFO:tensorflow:loss = 1.4208789, step = 15900 (9.863 sec)
INFO:tensorflow:lr = 0.00013395943 (9.862 sec)
INFO:tensorflow:Saving checkpoints for 15912 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.7580531.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-15912
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.976
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-15912
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 15912 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3895128, step = 15912
INFO:tensorflow:lr = 0.00013438129
INFO:tensorflow:global_step/sec: 9.07122
INFO:tensorflow:loss = 1.3131869, step = 16012 (11.029 sec)
INFO:tensorflow:lr = 0.00013789677 (11.027 sec)
INFO:tensorflow:global_step/sec: 9.72668
INFO:tensorflow:loss = 1.3209769, step = 16112 (10.281 sec)
INFO:tensorflow:lr = 0.0001414122 (10.281 sec)
INFO:tensorflow:global_step/sec: 10.0731
INFO:tensorflow:loss = 1.3567011, step = 16212 (9.927 sec)
INFO:tensorflow:lr = 0.00014492768 (9.929 sec)
INFO:tensorflow:Saving checkpoints for 16224 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.515823.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-16224
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-16224
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 16224 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3520435, step = 16224
INFO:tensorflow:lr = 0.00014534954
INFO:tensorflow:global_step/sec: 8.93298
INFO:tensorflow:loss = 1.3124169, step = 16324 (11.199 sec)
INFO:tensorflow:lr = 0.00014886502 (11.200 sec)
INFO:tensorflow:global_step/sec: 9.80825
INFO:tensorflow:loss = 1.3431764, step = 16424 (10.195 sec)
INFO:tensorflow:lr = 0.0001523805 (10.194 sec)
INFO:tensorflow:global_step/sec: 10.2541
INFO:tensorflow:loss = 1.3480072, step = 16524 (9.752 sec)
INFO:tensorflow:lr = 0.00015589596 (9.751 sec)
INFO:tensorflow:Saving checkpoints for 16536 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4112446.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-16536
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-16536
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 16536 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.2974615, step = 16536
INFO:tensorflow:lr = 0.0001563178
INFO:tensorflow:global_step/sec: 6.02022
INFO:tensorflow:loss = 1.2766285, step = 16636 (16.615 sec)
INFO:tensorflow:lr = 0.00015983326 (16.617 sec)
INFO:tensorflow:global_step/sec: 8.97792
INFO:tensorflow:loss = 1.4009911, step = 16736 (11.137 sec)
INFO:tensorflow:lr = 0.00016334874 (11.139 sec)
INFO:tensorflow:global_step/sec: 9.94307
INFO:tensorflow:loss = 1.4903059, step = 16836 (10.058 sec)
INFO:tensorflow:lr = 0.00016686422 (10.053 sec)
INFO:tensorflow:Saving checkpoints for 16848 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5365174.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-16848
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-16848
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 16848 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.2568657, step = 16848
INFO:tensorflow:lr = 0.00016728608
INFO:tensorflow:global_step/sec: 9.06341
INFO:tensorflow:loss = 1.3246856, step = 16948 (11.036 sec)
INFO:tensorflow:lr = 0.00017080152 (11.036 sec)
INFO:tensorflow:global_step/sec: 10.0624
INFO:tensorflow:loss = 1.4553119, step = 17048 (9.939 sec)
INFO:tensorflow:lr = 0.00017431699 (9.939 sec)
INFO:tensorflow:global_step/sec: 9.63898
INFO:tensorflow:loss = 1.3572354, step = 17148 (10.373 sec)
INFO:tensorflow:lr = 0.00017783247 (10.374 sec)
INFO:tensorflow:Saving checkpoints for 17160 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4536486.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-17160
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-17160
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 17160 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.4124483, step = 17160
INFO:tensorflow:lr = 0.00017825433
INFO:tensorflow:global_step/sec: 8.89719
INFO:tensorflow:loss = 1.3847319, step = 17260 (11.243 sec)
INFO:tensorflow:lr = 0.0001817698 (11.245 sec)
INFO:tensorflow:global_step/sec: 9.88365
INFO:tensorflow:loss = 1.4380949, step = 17360 (10.119 sec)
INFO:tensorflow:lr = 0.00018528524 (10.117 sec)
INFO:tensorflow:global_step/sec: 10.1202
INFO:tensorflow:loss = 1.4390996, step = 17460 (9.880 sec)
INFO:tensorflow:lr = 0.00018619926 (9.881 sec)
INFO:tensorflow:Saving checkpoints for 17472 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.595715.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-17472
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.975
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.991     0.987       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.680     0.673     0.664       888
                            weighted avg      0.978     0.981     0.978       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.996     0.984     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.957     0.996     0.976     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.892     0.579     0.702      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     1.000     1.000      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.929     0.433     0.591      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.615     0.381     0.471      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.909     0.952      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     1.000     1.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.960    3657.0
                   macro avg      0.702     0.672     0.676    3657.0
                weighted avg      0.963     0.961     0.957    3657.0

INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-17472
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 17472 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3522878, step = 17472
INFO:tensorflow:lr = 0.0001857774
INFO:tensorflow:global_step/sec: 9.19105
INFO:tensorflow:loss = 1.3775727, step = 17572 (10.882 sec)
INFO:tensorflow:lr = 0.00018226192 (10.880 sec)
INFO:tensorflow:global_step/sec: 10.0444
INFO:tensorflow:loss = 1.3149229, step = 17672 (9.956 sec)
INFO:tensorflow:lr = 0.00017874646 (9.958 sec)
INFO:tensorflow:global_step/sec: 10.0279
INFO:tensorflow:loss = 1.6317581, step = 17772 (9.974 sec)
INFO:tensorflow:lr = 0.00017523099 (9.973 sec)
INFO:tensorflow:Saving checkpoints for 17784 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4558718.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-17784
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-17784
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 17784 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3914574, step = 17784
INFO:tensorflow:lr = 0.00017480916
INFO:tensorflow:global_step/sec: 5.87302
INFO:tensorflow:loss = 1.4074838, step = 17884 (17.032 sec)
INFO:tensorflow:lr = 0.00017129368 (17.032 sec)
INFO:tensorflow:global_step/sec: 9.68533
INFO:tensorflow:loss = 1.2983662, step = 17984 (10.322 sec)
INFO:tensorflow:lr = 0.0001677782 (10.322 sec)
INFO:tensorflow:global_step/sec: 9.9224
INFO:tensorflow:loss = 1.4488811, step = 18084 (10.080 sec)
INFO:tensorflow:lr = 0.00016426273 (10.081 sec)
INFO:tensorflow:Saving checkpoints for 18096 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5434359.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-18096
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-18096
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 18096 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3872954, step = 18096
INFO:tensorflow:lr = 0.00016384086
INFO:tensorflow:global_step/sec: 9.07604
INFO:tensorflow:loss = 1.3522475, step = 18196 (11.021 sec)
INFO:tensorflow:lr = 0.00016032543 (11.020 sec)
INFO:tensorflow:global_step/sec: 9.77837
INFO:tensorflow:loss = 1.2364745, step = 18296 (10.226 sec)
INFO:tensorflow:lr = 0.00015680995 (10.227 sec)
INFO:tensorflow:global_step/sec: 9.81604
INFO:tensorflow:loss = 1.4598484, step = 18396 (10.189 sec)
INFO:tensorflow:lr = 0.00015329447 (10.192 sec)
INFO:tensorflow:Saving checkpoints for 18408 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.5730287.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-18408
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-18408
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 18408 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3265864, step = 18408
INFO:tensorflow:lr = 0.00015287261
INFO:tensorflow:global_step/sec: 8.94477
INFO:tensorflow:loss = 1.3522414, step = 18508 (11.183 sec)
INFO:tensorflow:lr = 0.00014935715 (11.184 sec)
INFO:tensorflow:global_step/sec: 9.86858
INFO:tensorflow:loss = 1.350786, step = 18608 (10.134 sec)
INFO:tensorflow:lr = 0.00014584171 (10.133 sec)
INFO:tensorflow:global_step/sec: 10.1399
INFO:tensorflow:loss = 1.3429725, step = 18708 (9.862 sec)
INFO:tensorflow:lr = 0.00014232623 (9.862 sec)
INFO:tensorflow:Saving checkpoints for 18720 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.8129239.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-18720
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-18720
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 18720 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.337751, step = 18720
INFO:tensorflow:lr = 0.00014190437
INFO:tensorflow:global_step/sec: 9.15394
INFO:tensorflow:loss = 1.475024, step = 18820 (10.929 sec)
INFO:tensorflow:lr = 0.00013838889 (10.928 sec)
INFO:tensorflow:global_step/sec: 9.79094
INFO:tensorflow:loss = 1.5030525, step = 18920 (10.214 sec)
INFO:tensorflow:lr = 0.00013487341 (10.213 sec)
INFO:tensorflow:global_step/sec: 9.99954
INFO:tensorflow:loss = 1.4375403, step = 19020 (10.001 sec)
INFO:tensorflow:lr = 0.00013135794 (10.002 sec)
INFO:tensorflow:Saving checkpoints for 19032 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.3993608.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-19032
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-19032
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 19032 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.2996929, step = 19032
INFO:tensorflow:lr = 0.00013093613
INFO:tensorflow:global_step/sec: 9.13035
INFO:tensorflow:loss = 1.3681372, step = 19132 (10.957 sec)
INFO:tensorflow:lr = 0.00012742064 (10.957 sec)
INFO:tensorflow:global_step/sec: 9.75585
INFO:tensorflow:loss = 1.3759432, step = 19232 (10.250 sec)
INFO:tensorflow:lr = 0.00012390516 (10.250 sec)
INFO:tensorflow:global_step/sec: 9.8615
INFO:tensorflow:loss = 1.438529, step = 19332 (10.142 sec)
INFO:tensorflow:lr = 0.00012038969 (10.145 sec)
INFO:tensorflow:Saving checkpoints for 19344 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4564463.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-19344
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-19344
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 19344 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.5058904, step = 19344
INFO:tensorflow:lr = 0.00011996782
INFO:tensorflow:global_step/sec: 9.32666
INFO:tensorflow:loss = 1.3566515, step = 19444 (10.724 sec)
INFO:tensorflow:lr = 0.00011645239 (10.724 sec)
INFO:tensorflow:global_step/sec: 9.8454
INFO:tensorflow:loss = 1.3719909, step = 19544 (10.159 sec)
INFO:tensorflow:lr = 0.00011293692 (10.159 sec)
INFO:tensorflow:global_step/sec: 9.86602
INFO:tensorflow:loss = 1.2837491, step = 19644 (10.135 sec)
INFO:tensorflow:lr = 0.00010942144 (10.135 sec)
INFO:tensorflow:Saving checkpoints for 19656 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.6603131.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-19656
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-19656
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 19656 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 1.3698283, step = 19656
INFO:tensorflow:lr = 0.00010899958
INFO:tensorflow:global_step/sec: 9.0087
INFO:tensorflow:loss = 1.3771712, step = 19756 (11.104 sec)
INFO:tensorflow:lr = 0.0001054841 (11.104 sec)
INFO:tensorflow:global_step/sec: 9.77676
INFO:tensorflow:loss = 1.384309, step = 19856 (17.176 sec)
INFO:tensorflow:lr = 0.00010196867 (17.179 sec)
INFO:tensorflow:global_step/sec: 5.81207
INFO:tensorflow:loss = 1.4715413, step = 19956 (10.258 sec)
INFO:tensorflow:lr = 0.000100773424 (10.255 sec)
INFO:tensorflow:Saving checkpoints for 19968 into ../model/elmo_bigru/model.ckpt.
INFO:tensorflow:Loss for final step: 1.4983729.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru/model.ckpt-19968
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.974
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.989     0.986       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      0.833     0.833     0.833         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.672     0.672     0.661       888
                            weighted avg      0.977     0.980     0.977       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.996     0.984     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.957     0.996     0.976     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.893     0.893     0.893      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     1.000     1.000      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.400     0.571      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.615     0.381     0.471      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.909     0.952      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     1.000     1.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     1.000     1.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.962     0.961    3657.0
                   macro avg      0.702     0.678     0.679    3657.0
                weighted avg      0.963     0.962     0.958    3657.0

INFO:tensorflow:Best Slot F1: 0.961, Intent Acc: 0.975
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))