In [0]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/spoken_language_understanding/atis/main')
In [2]:
%tensorflow_version 1.x
TensorFlow 1.x selected.
In [3]:
import tensorflow as tf
import tensorflow_hub as hub

import logging
import time
import numpy as np

from sklearn.metrics import classification_report, f1_score, accuracy_score
from pathlib import Path

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 1.15.2
GPU Enabled: True
In [0]:
def get_vocab(vocab_path):
  word2idx = {}
  with open(vocab_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
def data_generator(f_path, params):
  print('Reading', f_path)
  with open(f_path) as f:
    for line in f:
      line = line.rstrip()
      text, slot_intent = line.split('\t')
      words = text.split()[1:-1]
      slot_intent = slot_intent.split()
      slots, intent = slot_intent[1:-1], slot_intent[-1]
      assert len(words) == len(slots)
      
      yield (words, (intent, slots))
In [0]:
def dataset(is_training, params):
  _shapes = ([None], ((), [None]))
  _types = (tf.string, (tf.string, tf.string))
  _pads = ('<pad>', ('_', 'O'))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(1, _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
def clr(step,
        initial_learning_rate,
        maximal_learning_rate,
        step_size,
        scale_fn,
        scale_mode,):
  step = tf.cast(step, tf.float32)
  
  initial_learning_rate = tf.convert_to_tensor(
    initial_learning_rate, name='initial_learning_rate')
  dtype = initial_learning_rate.dtype
  maximal_learning_rate = tf.cast(maximal_learning_rate, dtype)
  step_size = tf.cast(step_size, dtype)
  cycle = tf.floor(1 + step / (2 * step_size))
  x = tf.abs(step / step_size - 2 * cycle + 1)

  mode_step = cycle if scale_mode == 'cycle' else step

  return initial_learning_rate + (
    maximal_learning_rate - initial_learning_rate) * tf.maximum(
      tf.cast(0, dtype), (1 - x)) * scale_fn(mode_step)
In [0]:
def model_fn(features, labels, mode, params):
  is_training = (mode == tf.estimator.ModeKeys.TRAIN)
  
  vocab = tf.contrib.lookup.index_table_from_file(
    params['word_path'], num_oov_buckets=1)
  words = vocab.lookup(features)
  seq_len = tf.count_nonzero(words, 1, dtype=tf.int32)
  
  embedding = np.load(params['vocab_path'])
  embedding = tf.Variable(embedding, name='embedding', dtype=tf.float32)
  x = tf.nn.embedding_lookup(embedding, words)
  
  elmo = hub.Module("https://tfhub.dev/google/elmo/2", trainable=False)
  e = elmo(inputs={'tokens':features, 'sequence_len':seq_len}, signature='tokens', as_dict=True)['lstm_outputs1']
  
  x = tf.concat((x, e), -1)
  x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
  x = tf.layers.dense(x, params['rnn_units'], tf.nn.elu)
  x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
  
  cell_fw = tf.nn.rnn_cell.GRUCell(params['rnn_units'], kernel_initializer=tf.orthogonal_initializer())
  cell_bw = tf.nn.rnn_cell.GRUCell(params['rnn_units'], kernel_initializer=tf.orthogonal_initializer())
  o, (s_fw, s_bw) = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, x, seq_len, dtype=tf.float32)
  x = tf.concat(o, -1)
  
  y_intent = tf.concat((tf.reduce_max(x, 1), s_fw, s_bw), -1)
  y_intent = tf.layers.dropout(y_intent, params['dropout_rate'], training=is_training)
  y_intent = tf.layers.dense(y_intent, params['rnn_units'], tf.nn.elu)
  y_intent = tf.layers.dense(y_intent, params['intent_size'])
  y_slots = tf.layers.dense(x, params['slot_size'])

  crf_params = tf.get_variable(
    "transitions", [params['slot_size'], params['slot_size']], dtype=tf.float32)
  pred_ids, _ = tf.contrib.crf.crf_decode(y_slots, crf_params, seq_len)
  
  
  if labels is not None:
    intent, slots = labels
    
    vocab = tf.contrib.lookup.index_table_from_file(
      params['intent_path'], num_oov_buckets=1)
    intent = vocab.lookup(intent)
    vocab = tf.contrib.lookup.index_table_from_file(
      params['slot_path'], num_oov_buckets=1)
    slots = vocab.lookup(slots)
    
    loss_intent = tf.losses.softmax_cross_entropy(
      onehot_labels = tf.one_hot(intent, len(params['intent2idx'])+1),
      logits = y_intent,
      label_smoothing = .2)
    
    log_likelihood, _ = tf.contrib.crf.crf_log_likelihood(
      y_slots, slots, seq_len, crf_params)
    loss_slots = tf.reduce_mean(- log_likelihood)
    
    loss_op = 2. * loss_intent + loss_slots

  
  if mode == tf.estimator.ModeKeys.TRAIN:
    variables = tf.trainable_variables()
    
    grads = tf.gradients(loss_op, variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    
    global_step=tf.train.get_or_create_global_step()
    decay_lr = clr(
      step = global_step,
      initial_learning_rate = 1e-4,
      maximal_learning_rate = 8e-4,
      step_size = 8 * params['num_samples'] // params['batch_size'],
      scale_fn=lambda x: 1 / (2.0 ** (x - 1)),
      scale_mode = 'cycle',)
    hook = tf.train.LoggingTensorHook({'lr': decay_lr, 'loss_intent': loss_intent, 'loss_slots': loss_slots}, every_n_iter=100)
    
    optim = tf.train.AdamOptimizer(decay_lr)
    train_op = optim.apply_gradients(
      zip(grads, variables), global_step=global_step)
    
    return tf.estimator.EstimatorSpec(mode=mode,
                                      loss=loss_op,
                                      train_op=train_op,
                                      training_hooks=[hook],)
  
  
  if mode == tf.estimator.ModeKeys.EVAL:
    return tf.estimator.EstimatorSpec(mode=mode,
                                      loss=loss_op)
  
  
  if mode == tf.estimator.ModeKeys.PREDICT:
    return tf.estimator.EstimatorSpec(mode,
      predictions={'intent': tf.argmax(y_intent, -1),
                   'slots': pred_ids})
In [0]:
params = {
  'model_dir': '../model/elmo_bigru_crf',
  'log_path': '../log/elmo_bigru_crf.txt',
  'train_path': '../data/atis.train.w-intent.iob',
  'test_path': '../data/atis.test.w-intent.iob',
  'word_path': '../vocab/word.txt',
  'vocab_path': '../vocab/word.npy',
  'intent_path': '../vocab/intent.txt',
  'slot_path': '../vocab/slot.txt',
  'batch_size': 16,
  'num_samples': 4978,
  'rnn_units': 300,
  'dropout_rate': .2,
  'clip_norm': 5.,
  'num_patience': 10,
}
In [0]:
params['word2idx'] = get_vocab(params['word_path'])
params['intent2idx'] = get_vocab(params['intent_path'])
params['slot2idx'] = get_vocab(params['slot_path'])

params['word_size'] = len(params['word2idx']) + 1
params['intent_size'] = len(params['intent2idx']) + 1
params['slot_size'] = len(params['slot2idx']) + 1
In [11]:
# Create directory if not exist
Path(os.path.dirname(params['log_path'])).mkdir(exist_ok=True)
Path(params['model_dir']).mkdir(exist_ok=True, parents=True)

# Logging
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(params['log_path'])
logger.addHandler(fh)
logger.propagate = False

# Create an estimator
config = tf.estimator.RunConfig(
  save_checkpoints_steps = params['num_samples']//params['batch_size'] + 1,
  keep_checkpoint_max=3)

estimator = tf.estimator.Estimator(
  model_fn=model_fn,
  model_dir=params['model_dir'],
  config=config,
  params=params)

slot_best_f1 = .0
intent_acc_with_that = .0
tf.enable_eager_execution()
count = 0
n_epoch = 1

while True:
  estimator.train(input_fn=lambda: dataset(is_training=True, params=params))
  
  intent = []
  slots = []
  for w, (i, s) in dataset(is_training=False, params=params):
    intent.append(i.numpy())
    slots.append(s.numpy())
  intent = [i for batch in intent for i in batch]
  intent = [params['intent2idx'].get(str(t, 'utf-8'), len(params['intent2idx'])) for t in intent]
  slots = [j for batch in slots for i in batch for j in i]
  slots = [params['slot2idx'].get(str(s, 'utf-8'), len(params['slot2idx'])) for s in slots]

  predicted = list(estimator.predict(input_fn=lambda: dataset(is_training=False, params=params)))
  y_slots = [j for i in predicted for j in i['slots']]
  y_intent = [i['intent'] for i in predicted]
  
  f1_slots = f1_score(y_true = slots,
                      y_pred = y_slots,
                      labels = list(params['slot2idx'].values()),
                      sample_weight = np.sign(slots),
                      average='micro',)
  
  acc_intent = accuracy_score(intent, y_intent)

  logger.info("Slot F1: {:.3f}, Intent Acc: {:.3f}".format(f1_slots, acc_intent))

  if n_epoch != 1 and n_epoch % 8 == 0:
    logger.info('\n'+classification_report(y_true = intent,
                                          y_pred = y_intent,
                                          labels = list(params['intent2idx'].values()),
                                          target_names = list(params['intent2idx'].keys()),
                                          digits=3))
    logger.info('\n'+classification_report(y_true = slots,
                                          y_pred = y_slots,
                                          labels = list(params['slot2idx'].values()),
                                          target_names = list(params['slot2idx'].keys()),
                                          sample_weight = np.sign(slots),
                                          digits=3))
  
  if f1_slots > slot_best_f1:
    slot_best_f1 = f1_slots
    intent_acc_with_that = acc_intent
    count = 0
  else:
    count += 1
  logger.info("Best Slot F1: {:.3f}, Intent Acc: {:.3f}".format(slot_best_f1, intent_acc_with_that))
  n_epoch += 1

  if count == params['num_patience']:
    print(params['num_patience'], "times not improve the best result, therefore stop training")
    break
INFO:tensorflow:Using config: {'_model_dir': '../model/elmo_bigru_crf', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 312, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true
graph_options {
  rewrite_options {
    meta_optimizer_iterations: ONE
  }
}
, '_keep_checkpoint_max': 3, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f06c017ef28>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.
INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:
The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
  * https://github.com/tensorflow/io (for I/O related ops)
If you depend on functionality not listed there, please file an issue.

WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.
Instructions for updating:
reduction_indices is deprecated, use axis instead
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
WARNING:tensorflow:From <ipython-input-8-600506eb75c4>:17: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dropout instead.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/layers/core.py:271: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.__call__` method instead.
WARNING:tensorflow:From <ipython-input-8-600506eb75c4>:18: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.Dense instead.
WARNING:tensorflow:From <ipython-input-8-600506eb75c4>:21: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.
WARNING:tensorflow:From <ipython-input-8-600506eb75c4>:23: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn_cell_impl.py:559: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.add_weight` method instead.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn_cell_impl.py:565: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn_cell_impl.py:575: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/rnn.py:244: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 0 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 56.96795, step = 0
INFO:tensorflow:loss_intent = 3.2335634, loss_slots = 50.500824, lr = 1e-04
INFO:tensorflow:global_step/sec: 8.43432
INFO:tensorflow:loss = 13.95684, step = 100 (11.859 sec)
INFO:tensorflow:loss_intent = 1.6046515, loss_slots = 10.747537, lr = 0.00012812372 (11.859 sec)
INFO:tensorflow:global_step/sec: 10.6349
INFO:tensorflow:loss = 6.9039392, step = 200 (9.403 sec)
INFO:tensorflow:loss_intent = 1.465177, loss_slots = 3.9735851, lr = 0.0001562475 (9.401 sec)
INFO:tensorflow:global_step/sec: 10.9314
INFO:tensorflow:loss = 5.49319, step = 300 (9.145 sec)
INFO:tensorflow:loss_intent = 1.3888321, loss_slots = 2.7155259, lr = 0.00018437125 (9.146 sec)
INFO:tensorflow:Saving checkpoints for 312 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.5702953.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-312
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.854, Intent Acc: 0.885
INFO:tensorflow:Best Slot F1: 0.854, Intent Acc: 0.885
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-312
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/saver.py:1069: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file utilities to get mtimes.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 312 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 4.390752, step = 312
INFO:tensorflow:loss_intent = 1.2865298, loss_slots = 1.8176922, lr = 0.00018774607
INFO:tensorflow:global_step/sec: 8.7563
INFO:tensorflow:loss = 4.8969536, step = 412 (11.422 sec)
INFO:tensorflow:loss_intent = 1.5164055, loss_slots = 1.8641429, lr = 0.0002158698 (11.422 sec)
INFO:tensorflow:global_step/sec: 10.4827
INFO:tensorflow:loss = 7.6125627, step = 512 (9.542 sec)
INFO:tensorflow:loss_intent = 1.6112323, loss_slots = 4.390098, lr = 0.00024399356 (9.542 sec)
INFO:tensorflow:global_step/sec: 10.7509
INFO:tensorflow:loss = 3.744679, step = 612 (9.299 sec)
INFO:tensorflow:loss_intent = 1.3066208, loss_slots = 1.1314373, lr = 0.00027211732 (9.298 sec)
INFO:tensorflow:Saving checkpoints for 624 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.7811527.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-624
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.937, Intent Acc: 0.937
INFO:tensorflow:Best Slot F1: 0.937, Intent Acc: 0.937
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-624
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 624 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.9057803, step = 624
INFO:tensorflow:loss_intent = 1.1894498, loss_slots = 0.5268807, lr = 0.00027549214
INFO:tensorflow:global_step/sec: 9.04373
INFO:tensorflow:loss = 2.7933965, step = 724 (11.062 sec)
INFO:tensorflow:loss_intent = 1.2393954, loss_slots = 0.31460583, lr = 0.00030361587 (11.060 sec)
INFO:tensorflow:global_step/sec: 11.043
INFO:tensorflow:loss = 3.55625, step = 824 (9.054 sec)
INFO:tensorflow:loss_intent = 1.2385283, loss_slots = 1.0791936, lr = 0.00033173963 (9.054 sec)
INFO:tensorflow:global_step/sec: 10.526
INFO:tensorflow:loss = 3.1039736, step = 924 (9.500 sec)
INFO:tensorflow:loss_intent = 1.3849479, loss_slots = 0.33407784, lr = 0.00035986342 (9.503 sec)
INFO:tensorflow:Saving checkpoints for 936 into ../model/elmo_bigru_crf/model.ckpt.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/saver.py:963: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
INFO:tensorflow:Loss for final step: 2.8349495.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-936
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.949, Intent Acc: 0.947
INFO:tensorflow:Best Slot F1: 0.949, Intent Acc: 0.947
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-936
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 936 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.5901468, step = 936
INFO:tensorflow:loss_intent = 1.1828421, loss_slots = 0.22446251, lr = 0.00036323824
INFO:tensorflow:global_step/sec: 9.31658
INFO:tensorflow:loss = 2.5230055, step = 1036 (10.738 sec)
INFO:tensorflow:loss_intent = 1.1075196, loss_slots = 0.30796623, lr = 0.00039136197 (10.738 sec)
INFO:tensorflow:global_step/sec: 10.5919
INFO:tensorflow:loss = 3.5216908, step = 1136 (9.441 sec)
INFO:tensorflow:loss_intent = 1.1092415, loss_slots = 1.3032079, lr = 0.00041948573 (9.441 sec)
INFO:tensorflow:global_step/sec: 10.9988
INFO:tensorflow:loss = 2.704909, step = 1236 (9.090 sec)
INFO:tensorflow:loss_intent = 1.115282, loss_slots = 0.47434497, lr = 0.0004476095 (9.088 sec)
INFO:tensorflow:Saving checkpoints for 1248 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 4.397087.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-1248
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.951, Intent Acc: 0.952
INFO:tensorflow:Best Slot F1: 0.951, Intent Acc: 0.952
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-1248
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 1248 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.5166287, step = 1248
INFO:tensorflow:loss_intent = 1.1254586, loss_slots = 0.26571155, lr = 0.0004509843
INFO:tensorflow:global_step/sec: 9.07205
INFO:tensorflow:loss = 2.3877375, step = 1348 (11.025 sec)
INFO:tensorflow:loss_intent = 1.1129894, loss_slots = 0.16175866, lr = 0.0004791081 (11.025 sec)
INFO:tensorflow:global_step/sec: 11.1818
INFO:tensorflow:loss = 2.3091016, step = 1448 (8.944 sec)
INFO:tensorflow:loss_intent = 1.1135197, loss_slots = 0.082062244, lr = 0.0005072318 (8.943 sec)
INFO:tensorflow:global_step/sec: 10.7808
INFO:tensorflow:loss = 2.3305216, step = 1548 (9.274 sec)
INFO:tensorflow:loss_intent = 1.0961514, loss_slots = 0.13821888, lr = 0.0005353556 (9.274 sec)
INFO:tensorflow:Saving checkpoints for 1560 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.8573494.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-1560
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.962
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.962
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-1560
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 1560 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.6917725, step = 1560
INFO:tensorflow:loss_intent = 1.114186, loss_slots = 0.46340036, lr = 0.0005387304
INFO:tensorflow:global_step/sec: 9.08823
INFO:tensorflow:loss = 2.4517035, step = 1660 (11.008 sec)
INFO:tensorflow:loss_intent = 1.1136017, loss_slots = 0.22450018, lr = 0.00056685414 (11.008 sec)
INFO:tensorflow:global_step/sec: 11.4135
INFO:tensorflow:loss = 2.5300822, step = 1760 (8.759 sec)
INFO:tensorflow:loss_intent = 1.1150882, loss_slots = 0.29990566, lr = 0.0005949779 (8.759 sec)
INFO:tensorflow:global_step/sec: 10.8866
INFO:tensorflow:loss = 2.25, step = 1860 (9.189 sec)
INFO:tensorflow:loss_intent = 1.1027354, loss_slots = 0.0445292, lr = 0.00062310166 (9.188 sec)
INFO:tensorflow:Saving checkpoints for 1872 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.202364.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-1872
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.964
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.962
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-1872
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 1872 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.2208226, step = 1872
INFO:tensorflow:loss_intent = 1.0988231, loss_slots = 0.023176432, lr = 0.0006264765
INFO:tensorflow:global_step/sec: 9.84477
INFO:tensorflow:loss = 2.2230372, step = 1972 (10.159 sec)
INFO:tensorflow:loss_intent = 1.1085105, loss_slots = 0.0060162544, lr = 0.00065460027 (10.160 sec)
INFO:tensorflow:global_step/sec: 11.2661
INFO:tensorflow:loss = 2.2402506, step = 2072 (8.876 sec)
INFO:tensorflow:loss_intent = 1.1063489, loss_slots = 0.027552843, lr = 0.000682724 (8.876 sec)
INFO:tensorflow:global_step/sec: 11.2488
INFO:tensorflow:loss = 2.359384, step = 2172 (8.893 sec)
INFO:tensorflow:loss_intent = 1.12128, loss_slots = 0.11682415, lr = 0.00071084773 (8.893 sec)
INFO:tensorflow:Saving checkpoints for 2184 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1818657.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-2184
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.954
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.954
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-2184
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 2184 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.2701693, step = 2184
INFO:tensorflow:loss_intent = 1.1035774, loss_slots = 0.06301451, lr = 0.00071422255
INFO:tensorflow:global_step/sec: 9.82304
INFO:tensorflow:loss = 2.7018156, step = 2284 (10.184 sec)
INFO:tensorflow:loss_intent = 1.1157036, loss_slots = 0.47040844, lr = 0.00074234634 (10.184 sec)
INFO:tensorflow:global_step/sec: 10.3361
INFO:tensorflow:loss = 2.384247, step = 2384 (9.673 sec)
INFO:tensorflow:loss_intent = 1.1781604, loss_slots = 0.027926207, lr = 0.00077047007 (9.676 sec)
INFO:tensorflow:global_step/sec: 10.8119
INFO:tensorflow:loss = 2.3738241, step = 2484 (9.249 sec)
INFO:tensorflow:loss_intent = 1.0992565, loss_slots = 0.17531109, lr = 0.0007985938 (9.251 sec)
INFO:tensorflow:Saving checkpoints for 2496 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.182281.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-2496
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.956, Intent Acc: 0.970
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.989     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.500     0.667         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.676     0.650     0.647       888
                            weighted avg      0.972     0.975     0.971       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.990     0.983     716.0
         B-fromloc.city_name      0.982     1.000     0.991     704.0
           I-toloc.city_name      0.956     0.985     0.970     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.947     1.000     0.973     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.914     0.561     0.696      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     1.000     1.000      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.452     0.933     0.609      15.0
      B-fromloc.airport_name      0.500     1.000     0.667      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.875     1.000     0.933      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.400     0.571      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.909     0.952      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      0.750     0.750     0.750       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.680     0.648     0.651    3657.0
                weighted avg      0.962     0.957     0.954    3657.0

INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.954
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-2496
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 2496 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.2403824, step = 2496
INFO:tensorflow:loss_intent = 1.0935999, loss_slots = 0.053182602, lr = 0.00079803134
INFO:tensorflow:global_step/sec: 9.82745
INFO:tensorflow:loss = 2.2059789, step = 2596 (10.180 sec)
INFO:tensorflow:loss_intent = 1.0947886, loss_slots = 0.016401768, lr = 0.0007699076 (10.179 sec)
INFO:tensorflow:global_step/sec: 10.8209
INFO:tensorflow:loss = 2.2106996, step = 2696 (9.241 sec)
INFO:tensorflow:loss_intent = 1.0973401, loss_slots = 0.016019344, lr = 0.00074178376 (9.240 sec)
INFO:tensorflow:global_step/sec: 11.6757
INFO:tensorflow:loss = 2.8307562, step = 2796 (8.565 sec)
INFO:tensorflow:loss_intent = 1.1302485, loss_slots = 0.5702591, lr = 0.00071366003 (8.566 sec)
INFO:tensorflow:Saving checkpoints for 2808 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1825433.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-2808
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.968
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.954
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-2808
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 2808 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.2265084, step = 2808
INFO:tensorflow:loss_intent = 1.0956866, loss_slots = 0.03513527, lr = 0.0007102852
INFO:tensorflow:global_step/sec: 7.39549
INFO:tensorflow:loss = 2.238328, step = 2908 (13.527 sec)
INFO:tensorflow:loss_intent = 1.1138813, loss_slots = 0.010565281, lr = 0.0006821615 (13.525 sec)
INFO:tensorflow:global_step/sec: 9.80867
INFO:tensorflow:loss = 2.1816208, step = 3008 (10.193 sec)
INFO:tensorflow:loss_intent = 1.0875883, loss_slots = 0.006444216, lr = 0.0006540377 (10.193 sec)
INFO:tensorflow:global_step/sec: 11.23
INFO:tensorflow:loss = 2.1957023, step = 3108 (8.903 sec)
INFO:tensorflow:loss_intent = 1.0955695, loss_slots = 0.0045633316, lr = 0.00062591396 (8.904 sec)
INFO:tensorflow:Saving checkpoints for 3120 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1708324.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-3120
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.957, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.954
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-3120
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 3120 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1985755, step = 3120
INFO:tensorflow:loss_intent = 1.0949196, loss_slots = 0.008736372, lr = 0.00062253914
INFO:tensorflow:global_step/sec: 8.83905
INFO:tensorflow:loss = 2.1988928, step = 3220 (11.316 sec)
INFO:tensorflow:loss_intent = 1.0954043, loss_slots = 0.008084297, lr = 0.0005944154 (11.317 sec)
INFO:tensorflow:global_step/sec: 11.4865
INFO:tensorflow:loss = 2.1749911, step = 3320 (8.704 sec)
INFO:tensorflow:loss_intent = 1.0851042, loss_slots = 0.0047826767, lr = 0.0005662916 (8.704 sec)
INFO:tensorflow:global_step/sec: 11.1498
INFO:tensorflow:loss = 2.5186455, step = 3420 (8.970 sec)
INFO:tensorflow:loss_intent = 1.0888999, loss_slots = 0.34084582, lr = 0.0005381679 (8.970 sec)
INFO:tensorflow:Saving checkpoints for 3432 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.2590313.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-3432
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.954
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-3432
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 3432 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.3802414, step = 3432
INFO:tensorflow:loss_intent = 1.0885923, loss_slots = 0.20305681, lr = 0.0005347931
INFO:tensorflow:global_step/sec: 9.20547
INFO:tensorflow:loss = 2.1740003, step = 3532 (10.865 sec)
INFO:tensorflow:loss_intent = 1.0847318, loss_slots = 0.0045366287, lr = 0.00050666934 (10.866 sec)
INFO:tensorflow:global_step/sec: 10.7755
INFO:tensorflow:loss = 2.2771437, step = 3632 (9.283 sec)
INFO:tensorflow:loss_intent = 1.1289306, loss_slots = 0.01928258, lr = 0.00047854552 (9.282 sec)
INFO:tensorflow:global_step/sec: 10.8
INFO:tensorflow:loss = 2.19045, step = 3732 (9.259 sec)
INFO:tensorflow:loss_intent = 1.0937341, loss_slots = 0.0029816628, lr = 0.00045042182 (9.259 sec)
INFO:tensorflow:Saving checkpoints for 3744 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1754014.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-3744
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.970
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.954
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-3744
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 3744 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1782222, step = 3744
INFO:tensorflow:loss_intent = 1.084506, loss_slots = 0.00921011, lr = 0.000447047
INFO:tensorflow:global_step/sec: 9.92916
INFO:tensorflow:loss = 2.1727796, step = 3844 (10.075 sec)
INFO:tensorflow:loss_intent = 1.0848248, loss_slots = 0.003129959, lr = 0.00041892327 (10.073 sec)
INFO:tensorflow:global_step/sec: 11.0366
INFO:tensorflow:loss = 2.1872876, step = 3944 (9.060 sec)
INFO:tensorflow:loss_intent = 1.0863062, loss_slots = 0.01467514, lr = 0.00039079945 (9.061 sec)
INFO:tensorflow:global_step/sec: 11.0853
INFO:tensorflow:loss = 2.190952, step = 4044 (9.019 sec)
INFO:tensorflow:loss_intent = 1.0925153, loss_slots = 0.005921364, lr = 0.00036267572 (9.019 sec)
INFO:tensorflow:Saving checkpoints for 4056 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.2027369.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-4056
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-4056
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 4056 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1660006, step = 4056
INFO:tensorflow:loss_intent = 1.0813574, loss_slots = 0.0032858849, lr = 0.0003593009
INFO:tensorflow:global_step/sec: 9.85432
INFO:tensorflow:loss = 2.1767375, step = 4156 (10.152 sec)
INFO:tensorflow:loss_intent = 1.0874006, loss_slots = 0.0019364357, lr = 0.00033117717 (10.150 sec)
INFO:tensorflow:global_step/sec: 11.1175
INFO:tensorflow:loss = 2.1832716, step = 4256 (8.994 sec)
INFO:tensorflow:loss_intent = 1.0895478, loss_slots = 0.00417614, lr = 0.00030305336 (8.996 sec)
INFO:tensorflow:global_step/sec: 9.5131
INFO:tensorflow:loss = 2.2576437, step = 4356 (10.788 sec)
INFO:tensorflow:loss_intent = 1.1265287, loss_slots = 0.00458622, lr = 0.00027492965 (10.787 sec)
INFO:tensorflow:Saving checkpoints for 4368 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.172871.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-4368
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.959, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-4368
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 4368 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.167914, step = 4368
INFO:tensorflow:loss_intent = 1.0826817, loss_slots = 0.002550602, lr = 0.00027155483
INFO:tensorflow:global_step/sec: 9.49073
INFO:tensorflow:loss = 2.1666546, step = 4468 (10.540 sec)
INFO:tensorflow:loss_intent = 1.0807037, loss_slots = 0.005247116, lr = 0.0002434311 (10.544 sec)
INFO:tensorflow:global_step/sec: 10.6036
INFO:tensorflow:loss = 2.1921654, step = 4568 (9.431 sec)
INFO:tensorflow:loss_intent = 1.0943382, loss_slots = 0.0034890175, lr = 0.00021530729 (9.428 sec)
INFO:tensorflow:global_step/sec: 10.9257
INFO:tensorflow:loss = 2.1726325, step = 4668 (9.154 sec)
INFO:tensorflow:loss_intent = 1.0857731, loss_slots = 0.001086235, lr = 0.00018718356 (9.154 sec)
INFO:tensorflow:Saving checkpoints for 4680 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.189911.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-4680
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-4680
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 4680 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.17561, step = 4680
INFO:tensorflow:loss_intent = 1.0858896, loss_slots = 0.0038309097, lr = 0.00018380873
INFO:tensorflow:global_step/sec: 10.0027
INFO:tensorflow:loss = 2.1685274, step = 4780 (10.000 sec)
INFO:tensorflow:loss_intent = 1.0821049, loss_slots = 0.004317522, lr = 0.00015568502 (10.000 sec)
INFO:tensorflow:global_step/sec: 10.9639
INFO:tensorflow:loss = 2.1662686, step = 4880 (9.123 sec)
INFO:tensorflow:loss_intent = 1.0824023, loss_slots = 0.0014638901, lr = 0.00012756122 (9.123 sec)
INFO:tensorflow:global_step/sec: 11.2187
INFO:tensorflow:loss = 2.1673095, step = 4980 (8.912 sec)
INFO:tensorflow:loss_intent = 1.0813756, loss_slots = 0.004558325, lr = 0.00010028121 (8.912 sec)
INFO:tensorflow:Saving checkpoints for 4992 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.2003324.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-4992
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.991     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.678     0.662     0.656       888
                            weighted avg      0.973     0.977     0.973       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.985     1.000     0.992     704.0
           I-toloc.city_name      0.960     0.996     0.978     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.939     0.544     0.689      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.933     0.467     0.622      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.909     0.952      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.500     1.000     0.667       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.500     0.111     0.182       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.960     0.959    3657.0
                   macro avg      0.697     0.667     0.668    3657.0
                weighted avg      0.964     0.960     0.956    3657.0

INFO:tensorflow:Best Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-4992
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 4992 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1724937, step = 4992
INFO:tensorflow:loss_intent = 1.0846373, loss_slots = 0.0032191277, lr = 0.00010196867
INFO:tensorflow:global_step/sec: 9.54064
INFO:tensorflow:loss = 2.1693795, step = 5092 (10.484 sec)
INFO:tensorflow:loss_intent = 1.0836073, loss_slots = 0.0021648407, lr = 0.00011603057 (10.480 sec)
INFO:tensorflow:global_step/sec: 10.7854
INFO:tensorflow:loss = 2.1923807, step = 5192 (9.272 sec)
INFO:tensorflow:loss_intent = 1.0820524, loss_slots = 0.028275967, lr = 0.00013009239 (9.273 sec)
INFO:tensorflow:global_step/sec: 9.5687
INFO:tensorflow:loss = 2.1772637, step = 5292 (10.452 sec)
INFO:tensorflow:loss_intent = 1.0871272, loss_slots = 0.0030093193, lr = 0.0001441543 (10.452 sec)
INFO:tensorflow:Saving checkpoints for 5304 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1645207.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-5304
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.957, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-5304
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 5304 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.165185, step = 5304
INFO:tensorflow:loss_intent = 1.0823884, loss_slots = 0.0004081726, lr = 0.00014584175
INFO:tensorflow:global_step/sec: 9.64849
INFO:tensorflow:loss = 2.1745162, step = 5404 (10.368 sec)
INFO:tensorflow:loss_intent = 1.0830464, loss_slots = 0.008423328, lr = 0.00015990357 (10.368 sec)
INFO:tensorflow:global_step/sec: 11.0103
INFO:tensorflow:loss = 2.1749046, step = 5504 (9.082 sec)
INFO:tensorflow:loss_intent = 1.0841264, loss_slots = 0.0066518784, lr = 0.00017396547 (9.082 sec)
INFO:tensorflow:global_step/sec: 10.9234
INFO:tensorflow:loss = 2.1938572, step = 5604 (9.153 sec)
INFO:tensorflow:loss_intent = 1.0962489, loss_slots = 0.0013594627, lr = 0.00018802729 (9.161 sec)
INFO:tensorflow:Saving checkpoints for 5616 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.161572.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-5616
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-5616
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 5616 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1717966, step = 5616
INFO:tensorflow:loss_intent = 1.08351, loss_slots = 0.004776478, lr = 0.00018971474
INFO:tensorflow:global_step/sec: 9.68855
INFO:tensorflow:loss = 2.1701376, step = 5716 (10.325 sec)
INFO:tensorflow:loss_intent = 1.0833324, loss_slots = 0.003472805, lr = 0.00020377665 (10.323 sec)
INFO:tensorflow:global_step/sec: 9.807
INFO:tensorflow:loss = 2.1783338, step = 5816 (10.196 sec)
INFO:tensorflow:loss_intent = 1.087311, loss_slots = 0.0037117004, lr = 0.00021783847 (10.196 sec)
INFO:tensorflow:global_step/sec: 8.71905
INFO:tensorflow:loss = 2.1650558, step = 5916 (11.471 sec)
INFO:tensorflow:loss_intent = 1.0814867, loss_slots = 0.0020823479, lr = 0.00023190037 (11.471 sec)
INFO:tensorflow:Saving checkpoints for 5928 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1643996.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-5928
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.957, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-5928
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 5928 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1783955, step = 5928
INFO:tensorflow:loss_intent = 1.0818554, loss_slots = 0.014684677, lr = 0.00023358784
INFO:tensorflow:global_step/sec: 9.61932
INFO:tensorflow:loss = 2.1721523, step = 6028 (10.398 sec)
INFO:tensorflow:loss_intent = 1.0857352, loss_slots = 0.00068187714, lr = 0.00024764964 (10.400 sec)
INFO:tensorflow:global_step/sec: 10.9337
INFO:tensorflow:loss = 2.1669927, step = 6128 (9.148 sec)
INFO:tensorflow:loss_intent = 1.0827076, loss_slots = 0.0015773773, lr = 0.00026171157 (9.146 sec)
INFO:tensorflow:global_step/sec: 10.5957
INFO:tensorflow:loss = 2.1988664, step = 6228 (9.437 sec)
INFO:tensorflow:loss_intent = 1.0984714, loss_slots = 0.0019235611, lr = 0.00027577337 (9.437 sec)
INFO:tensorflow:Saving checkpoints for 6240 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1805496.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-6240
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-6240
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 6240 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.2132034, step = 6240
INFO:tensorflow:loss_intent = 1.0866857, loss_slots = 0.039832115, lr = 0.00027746081
INFO:tensorflow:global_step/sec: 7.45793
INFO:tensorflow:loss = 2.166763, step = 6340 (13.413 sec)
INFO:tensorflow:loss_intent = 1.0827864, loss_slots = 0.0011901855, lr = 0.00029152274 (13.412 sec)
INFO:tensorflow:global_step/sec: 11.4243
INFO:tensorflow:loss = 2.2056525, step = 6440 (8.751 sec)
INFO:tensorflow:loss_intent = 1.100569, loss_slots = 0.004514456, lr = 0.00030558454 (8.749 sec)
INFO:tensorflow:global_step/sec: 11.2123
INFO:tensorflow:loss = 2.1812677, step = 6540 (8.918 sec)
INFO:tensorflow:loss_intent = 1.090049, loss_slots = 0.0011696815, lr = 0.00031964647 (8.918 sec)
INFO:tensorflow:Saving checkpoints for 6552 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.162105.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-6552
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-6552
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 6552 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1877472, step = 6552
INFO:tensorflow:loss_intent = 1.0861663, loss_slots = 0.015414715, lr = 0.0003213339
INFO:tensorflow:global_step/sec: 9.7981
INFO:tensorflow:loss = 2.16578, step = 6652 (10.208 sec)
INFO:tensorflow:loss_intent = 1.0819345, loss_slots = 0.0019111633, lr = 0.00033539574 (10.207 sec)
INFO:tensorflow:global_step/sec: 10.8761
INFO:tensorflow:loss = 2.18299, step = 6752 (9.197 sec)
INFO:tensorflow:loss_intent = 1.0846429, loss_slots = 0.0137043, lr = 0.00034945764 (9.196 sec)
INFO:tensorflow:global_step/sec: 11.4155
INFO:tensorflow:loss = 2.1686225, step = 6852 (8.759 sec)
INFO:tensorflow:loss_intent = 1.0836303, loss_slots = 0.0013618469, lr = 0.00036351947 (8.760 sec)
INFO:tensorflow:Saving checkpoints for 6864 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1682174.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-6864
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.971
INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-6864
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 6864 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.2458565, step = 6864
INFO:tensorflow:loss_intent = 1.088629, loss_slots = 0.06859851, lr = 0.0003652069
INFO:tensorflow:global_step/sec: 9.1447
INFO:tensorflow:loss = 2.2088943, step = 6964 (11.882 sec)
INFO:tensorflow:loss_intent = 1.1020873, loss_slots = 0.004719734, lr = 0.0003792688 (11.890 sec)
INFO:tensorflow:global_step/sec: 9.75856
INFO:tensorflow:loss = 2.206244, step = 7064 (9.304 sec)
INFO:tensorflow:loss_intent = 1.0926347, loss_slots = 0.020974636, lr = 0.00039333064 (9.295 sec)
INFO:tensorflow:global_step/sec: 10.3084
INFO:tensorflow:loss = 2.177629, step = 7164 (9.703 sec)
INFO:tensorflow:loss_intent = 1.0853057, loss_slots = 0.0070176125, lr = 0.00040739254 (9.707 sec)
INFO:tensorflow:Saving checkpoints for 7176 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1641877.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-7176
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.957, Intent Acc: 0.970
INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-7176
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 7176 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.179741, step = 7176
INFO:tensorflow:loss_intent = 1.0885744, loss_slots = 0.0025920868, lr = 0.00040908
INFO:tensorflow:global_step/sec: 10.2835
INFO:tensorflow:loss = 2.196632, step = 7276 (9.729 sec)
INFO:tensorflow:loss_intent = 1.0956156, loss_slots = 0.0054006577, lr = 0.0004231418 (9.729 sec)
INFO:tensorflow:global_step/sec: 11.3337
INFO:tensorflow:loss = 2.210007, step = 7376 (8.823 sec)
INFO:tensorflow:loss_intent = 1.1006418, loss_slots = 0.008723259, lr = 0.0004372037 (8.823 sec)
INFO:tensorflow:global_step/sec: 10.5368
INFO:tensorflow:loss = 2.209666, step = 7476 (9.490 sec)
INFO:tensorflow:loss_intent = 1.084262, loss_slots = 0.041141987, lr = 0.00044873444 (9.491 sec)
INFO:tensorflow:Saving checkpoints for 7488 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1666112.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-7488
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.957, Intent Acc: 0.971
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.989     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.674     0.659     0.653       888
                            weighted avg      0.973     0.976     0.972       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.993     0.985     716.0
         B-fromloc.city_name      0.982     1.000     0.991     704.0
           I-toloc.city_name      0.963     0.981     0.972     265.0
      B-depart_date.day_name      0.981     0.991     0.986     212.0
              B-airline_name      0.980     0.980     0.980     101.0
         I-fromloc.city_name      0.952     1.000     0.975     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      0.970     1.000     0.985      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      1.000     0.544     0.705      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.969     0.912     0.939      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.833     0.909     0.870      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.848     1.000     0.918      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     1.000     1.000      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.933     0.467     0.622      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.909     0.952      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.727     1.000     0.842       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.500     1.000     0.667       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.500     0.111     0.182       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     1.000     1.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.958     0.957    3657.0
                   macro avg      0.700     0.666     0.668    3657.0
                weighted avg      0.965     0.958     0.956    3657.0

INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-7488
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 7488 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1935637, step = 7488
INFO:tensorflow:loss_intent = 1.0963936, loss_slots = 0.0007765293, lr = 0.000447047
INFO:tensorflow:global_step/sec: 10.0639
INFO:tensorflow:loss = 2.1675441, step = 7588 (9.941 sec)
INFO:tensorflow:loss_intent = 1.0834318, loss_slots = 0.0006804466, lr = 0.00043298508 (9.941 sec)
INFO:tensorflow:global_step/sec: 11.3194
INFO:tensorflow:loss = 2.3672228, step = 7688 (8.838 sec)
INFO:tensorflow:loss_intent = 1.0960181, loss_slots = 0.17518663, lr = 0.00041892327 (8.836 sec)
INFO:tensorflow:global_step/sec: 11.1083
INFO:tensorflow:loss = 2.3562129, step = 7788 (8.997 sec)
INFO:tensorflow:loss_intent = 1.0996319, loss_slots = 0.15694904, lr = 0.00040486135 (8.998 sec)
INFO:tensorflow:Saving checkpoints for 7800 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1602454.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-7800
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-7800
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 7800 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.2151399, step = 7800
INFO:tensorflow:loss_intent = 1.1013811, loss_slots = 0.012377739, lr = 0.0004031739
INFO:tensorflow:global_step/sec: 10.0982
INFO:tensorflow:loss = 2.1792953, step = 7900 (9.907 sec)
INFO:tensorflow:loss_intent = 1.0830764, loss_slots = 0.013142586, lr = 0.00038911207 (9.908 sec)
INFO:tensorflow:global_step/sec: 10.939
INFO:tensorflow:loss = 2.205876, step = 8000 (9.141 sec)
INFO:tensorflow:loss_intent = 1.0895334, loss_slots = 0.026809216, lr = 0.00037505018 (9.142 sec)
INFO:tensorflow:global_step/sec: 11.0885
INFO:tensorflow:loss = 2.7362118, step = 8100 (9.020 sec)
INFO:tensorflow:loss_intent = 1.0859573, loss_slots = 0.5642972, lr = 0.00036098837 (9.017 sec)
INFO:tensorflow:Saving checkpoints for 8112 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.2809715.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-8112
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-8112
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 8112 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1709714, step = 8112
INFO:tensorflow:loss_intent = 1.0850894, loss_slots = 0.00079250336, lr = 0.0003593009
INFO:tensorflow:global_step/sec: 9.98134
INFO:tensorflow:loss = 2.1651063, step = 8212 (10.022 sec)
INFO:tensorflow:loss_intent = 1.0821941, loss_slots = 0.00071811676, lr = 0.000345239 (10.024 sec)
INFO:tensorflow:global_step/sec: 10.7668
INFO:tensorflow:loss = 2.1866138, step = 8312 (9.289 sec)
INFO:tensorflow:loss_intent = 1.0926054, loss_slots = 0.0014030933, lr = 0.00033117717 (9.288 sec)
INFO:tensorflow:global_step/sec: 11.6234
INFO:tensorflow:loss = 2.235914, step = 8412 (8.602 sec)
INFO:tensorflow:loss_intent = 1.0943478, loss_slots = 0.047218323, lr = 0.00031711528 (8.602 sec)
INFO:tensorflow:Saving checkpoints for 8424 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1721077.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-8424
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.959, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.960, Intent Acc: 0.972
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-8424
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 8424 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.16558, step = 8424
INFO:tensorflow:loss_intent = 1.0826212, loss_slots = 0.0003376007, lr = 0.0003154278
INFO:tensorflow:global_step/sec: 10.1697
INFO:tensorflow:loss = 2.1641648, step = 8524 (9.837 sec)
INFO:tensorflow:loss_intent = 1.0811856, loss_slots = 0.001793623, lr = 0.000301366 (9.836 sec)
INFO:tensorflow:global_step/sec: 11.1094
INFO:tensorflow:loss = 2.16388, step = 8624 (9.001 sec)
INFO:tensorflow:loss_intent = 1.0816468, loss_slots = 0.0005865097, lr = 0.0002873041 (9.003 sec)
INFO:tensorflow:global_step/sec: 11.1872
INFO:tensorflow:loss = 2.1636753, step = 8724 (8.939 sec)
INFO:tensorflow:loss_intent = 1.0810571, loss_slots = 0.0015611649, lr = 0.00027324227 (8.938 sec)
INFO:tensorflow:Saving checkpoints for 8736 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.160895.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-8736
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-8736
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 8736 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1654763, step = 8736
INFO:tensorflow:loss_intent = 1.0823507, loss_slots = 0.0007748604, lr = 0.00027155483
INFO:tensorflow:global_step/sec: 10.0655
INFO:tensorflow:loss = 2.2243233, step = 8836 (9.937 sec)
INFO:tensorflow:loss_intent = 1.0842183, loss_slots = 0.055886745, lr = 0.0002574929 (9.935 sec)
INFO:tensorflow:global_step/sec: 11.1591
INFO:tensorflow:loss = 2.1639872, step = 8936 (8.963 sec)
INFO:tensorflow:loss_intent = 1.0815549, loss_slots = 0.0008773804, lr = 0.0002434311 (8.964 sec)
INFO:tensorflow:global_step/sec: 10.8742
INFO:tensorflow:loss = 2.1703653, step = 9036 (9.195 sec)
INFO:tensorflow:loss_intent = 1.0840113, loss_slots = 0.002342701, lr = 0.00022936918 (9.195 sec)
INFO:tensorflow:Saving checkpoints for 9048 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1665385.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-9048
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-9048
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 9048 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1818783, step = 9048
INFO:tensorflow:loss_intent = 1.0902371, loss_slots = 0.001404047, lr = 0.00022768174
INFO:tensorflow:global_step/sec: 9.97917
INFO:tensorflow:loss = 2.261107, step = 9148 (10.024 sec)
INFO:tensorflow:loss_intent = 1.0814416, loss_slots = 0.098223686, lr = 0.00021361992 (10.025 sec)
INFO:tensorflow:global_step/sec: 10.9371
INFO:tensorflow:loss = 2.168643, step = 9248 (9.143 sec)
INFO:tensorflow:loss_intent = 1.0835712, loss_slots = 0.0015006065, lr = 0.00019955801 (9.144 sec)
INFO:tensorflow:global_step/sec: 11.2589
INFO:tensorflow:loss = 2.1607633, step = 9348 (8.880 sec)
INFO:tensorflow:loss_intent = 1.0802805, loss_slots = 0.00020217896, lr = 0.0001854962 (8.878 sec)
INFO:tensorflow:Saving checkpoints for 9360 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1595693.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-9360
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-9360
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 9360 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1666868, step = 9360
INFO:tensorflow:loss_intent = 1.082961, loss_slots = 0.0007648468, lr = 0.00018380873
INFO:tensorflow:global_step/sec: 9.83838
INFO:tensorflow:loss = 2.1629553, step = 9460 (10.168 sec)
INFO:tensorflow:loss_intent = 1.0813465, loss_slots = 0.00026226044, lr = 0.00016974684 (10.169 sec)
INFO:tensorflow:global_step/sec: 11.1965
INFO:tensorflow:loss = 2.1760721, step = 9560 (8.932 sec)
INFO:tensorflow:loss_intent = 1.0875635, loss_slots = 0.00094509125, lr = 0.00015568502 (8.930 sec)
INFO:tensorflow:global_step/sec: 10.9763
INFO:tensorflow:loss = 2.1654782, step = 9660 (9.110 sec)
INFO:tensorflow:loss_intent = 1.0824339, loss_slots = 0.00061035156, lr = 0.00014162311 (9.110 sec)
INFO:tensorflow:Saving checkpoints for 9672 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1591868.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-9672
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-9672
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 9672 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1645432, step = 9672
INFO:tensorflow:loss_intent = 1.0813813, loss_slots = 0.00178051, lr = 0.00013993566
INFO:tensorflow:global_step/sec: 10.2443
INFO:tensorflow:loss = 2.160378, step = 9772 (9.766 sec)
INFO:tensorflow:loss_intent = 1.0800856, loss_slots = 0.00020670891, lr = 0.00012587383 (9.766 sec)
INFO:tensorflow:global_step/sec: 10.1189
INFO:tensorflow:loss = 2.1665232, step = 9872 (9.884 sec)
INFO:tensorflow:loss_intent = 1.083137, loss_slots = 0.00024914742, lr = 0.00011181193 (9.884 sec)
INFO:tensorflow:global_step/sec: 11.0152
INFO:tensorflow:loss = 2.1611028, step = 9972 (9.077 sec)
INFO:tensorflow:loss_intent = 1.0801318, loss_slots = 0.0008392334, lr = 0.00010112494 (9.075 sec)
INFO:tensorflow:Saving checkpoints for 9984 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1626806.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-9984
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.974
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.991     0.986       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.679     0.671     0.664       888
                            weighted avg      0.976     0.980     0.976       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.985     1.000     0.992     704.0
           I-toloc.city_name      0.956     0.989     0.972     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      1.000     0.931     0.964     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.923     1.000     0.960      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     1.000     1.000      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.500     1.000     0.667      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.963     0.929     0.945      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     1.000     1.000      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.933     0.467     0.622      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.909     0.952      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     1.000     1.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     1.000     1.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.962     0.961    3657.0
                   macro avg      0.708     0.681     0.683    3657.0
                weighted avg      0.966     0.962     0.959    3657.0

INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-9984
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 9984 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1628938, step = 9984
INFO:tensorflow:loss_intent = 1.0804751, loss_slots = 0.0019435883, lr = 0.00010196867
INFO:tensorflow:global_step/sec: 9.90337
INFO:tensorflow:loss = 2.16682, step = 10084 (10.101 sec)
INFO:tensorflow:loss_intent = 1.0832541, loss_slots = 0.0003118515, lr = 0.00010899962 (10.101 sec)
INFO:tensorflow:global_step/sec: 11.0731
INFO:tensorflow:loss = 2.164033, step = 10184 (9.031 sec)
INFO:tensorflow:loss_intent = 1.0817862, loss_slots = 0.0004606247, lr = 0.00011603057 (9.032 sec)
INFO:tensorflow:global_step/sec: 11.2235
INFO:tensorflow:loss = 2.1707416, step = 10284 (8.910 sec)
INFO:tensorflow:loss_intent = 1.0843523, loss_slots = 0.0020370483, lr = 0.00012306144 (8.911 sec)
INFO:tensorflow:Saving checkpoints for 10296 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1596813.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-10296
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-10296
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 10296 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1668806, step = 10296
INFO:tensorflow:loss_intent = 1.0832682, loss_slots = 0.00034427643, lr = 0.00012390516
INFO:tensorflow:global_step/sec: 9.86559
INFO:tensorflow:loss = 2.1643202, step = 10396 (10.138 sec)
INFO:tensorflow:loss_intent = 1.0817368, loss_slots = 0.0008466244, lr = 0.00013093613 (10.136 sec)
INFO:tensorflow:global_step/sec: 11.2171
INFO:tensorflow:loss = 2.1756868, step = 10496 (8.916 sec)
INFO:tensorflow:loss_intent = 1.0874844, loss_slots = 0.00071811676, lr = 0.00013796707 (8.915 sec)
INFO:tensorflow:global_step/sec: 11.1335
INFO:tensorflow:loss = 2.1633, step = 10596 (8.984 sec)
INFO:tensorflow:loss_intent = 1.0815866, loss_slots = 0.00012683868, lr = 0.00014499802 (8.984 sec)
INFO:tensorflow:Saving checkpoints for 10608 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1710305.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-10608
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-10608
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 10608 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1725752, step = 10608
INFO:tensorflow:loss_intent = 1.0831441, loss_slots = 0.006287098, lr = 0.00014584175
INFO:tensorflow:global_step/sec: 10.1702
INFO:tensorflow:loss = 2.1606848, step = 10708 (9.836 sec)
INFO:tensorflow:loss_intent = 1.0801998, loss_slots = 0.00028514862, lr = 0.00015287261 (9.836 sec)
INFO:tensorflow:global_step/sec: 10.931
INFO:tensorflow:loss = 2.1660733, step = 10808 (9.149 sec)
INFO:tensorflow:loss_intent = 1.082062, loss_slots = 0.0019493103, lr = 0.00015990357 (9.149 sec)
INFO:tensorflow:global_step/sec: 10.6519
INFO:tensorflow:loss = 2.1620412, step = 10908 (9.387 sec)
INFO:tensorflow:loss_intent = 1.0809543, loss_slots = 0.00013256073, lr = 0.00016693452 (9.389 sec)
INFO:tensorflow:Saving checkpoints for 10920 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.163649.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-10920
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-10920
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 10920 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1613832, step = 10920
INFO:tensorflow:loss_intent = 1.0805213, loss_slots = 0.00034046173, lr = 0.00016777826
INFO:tensorflow:global_step/sec: 10.1706
INFO:tensorflow:loss = 2.1805472, step = 11020 (9.835 sec)
INFO:tensorflow:loss_intent = 1.089859, loss_slots = 0.0008292198, lr = 0.0001748092 (9.835 sec)
INFO:tensorflow:global_step/sec: 11.0869
INFO:tensorflow:loss = 2.172937, step = 11120 (9.021 sec)
INFO:tensorflow:loss_intent = 1.0853627, loss_slots = 0.0022115707, lr = 0.00018184006 (9.022 sec)
INFO:tensorflow:global_step/sec: 10.7452
INFO:tensorflow:loss = 2.1762452, step = 11220 (9.306 sec)
INFO:tensorflow:loss_intent = 1.0877368, loss_slots = 0.0007715225, lr = 0.00018887102 (9.305 sec)
INFO:tensorflow:Saving checkpoints for 11232 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1630907.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-11232
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.961, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-11232
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 11232 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1730075, step = 11232
INFO:tensorflow:loss_intent = 1.0864366, loss_slots = 0.00013422966, lr = 0.00018971474
INFO:tensorflow:global_step/sec: 9.8883
INFO:tensorflow:loss = 2.1832838, step = 11332 (10.117 sec)
INFO:tensorflow:loss_intent = 1.0911505, loss_slots = 0.0009827614, lr = 0.0001967457 (10.117 sec)
INFO:tensorflow:global_step/sec: 11.2256
INFO:tensorflow:loss = 2.1666446, step = 11432 (8.905 sec)
INFO:tensorflow:loss_intent = 1.0831602, loss_slots = 0.00032424927, lr = 0.00020377665 (8.905 sec)
INFO:tensorflow:global_step/sec: 11.575
INFO:tensorflow:loss = 2.1631622, step = 11532 (8.642 sec)
INFO:tensorflow:loss_intent = 1.0814576, loss_slots = 0.00024700165, lr = 0.0002108076 (8.642 sec)
INFO:tensorflow:Saving checkpoints for 11544 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1695623.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-11544
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.960, Intent Acc: 0.970
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-11544
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 11544 into ../model/elmo_bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:loss = 2.1696124, step = 11544
INFO:tensorflow:loss_intent = 1.0845778, loss_slots = 0.00045681, lr = 0.00021165125
INFO:tensorflow:global_step/sec: 10.0299
INFO:tensorflow:loss = 2.1633878, step = 11644 (9.973 sec)
INFO:tensorflow:loss_intent = 1.0814095, loss_slots = 0.00056886673, lr = 0.0002186822 (9.974 sec)
INFO:tensorflow:global_step/sec: 10.7883
INFO:tensorflow:loss = 2.165302, step = 11744 (9.271 sec)
INFO:tensorflow:loss_intent = 1.0824283, loss_slots = 0.0004453659, lr = 0.00022571316 (9.271 sec)
INFO:tensorflow:global_step/sec: 11.4493
INFO:tensorflow:loss = 2.167643, step = 11844 (8.732 sec)
INFO:tensorflow:loss_intent = 1.0833323, loss_slots = 0.0009784698, lr = 0.00023274409 (8.735 sec)
INFO:tensorflow:Saving checkpoints for 11856 into ../model/elmo_bigru_crf/model.ckpt.
INFO:tensorflow:Loss for final step: 2.167039.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/elmo_bigru_crf/model.ckpt-11856
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.959, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.963, Intent Acc: 0.973
10 times not improve the best result, therefore stop training