In [0]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/knowledge_graph_completion/wn18/main')
In [2]:
%tensorflow_version 2.x
!pip install tensorflow-addons
Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (0.8.3)
Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons) (2.7.1)
In [3]:
from tensorflow_addons.optimizers.cyclical_learning_rate import Triangular2CyclicalLearningRate

import tensorflow as tf
import pprint
import logging
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.2.0
WARNING:tensorflow:From <ipython-input-3-177c62f47bc4>:9: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU Enabled: True
In [0]:
def get_vocab(f_path):
  word2idx = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
"""
we use 1vN fast evaluation as purposed in ConvE paper:
"https://arxiv.org/abs/1707.01476"
sp2o is a dictionary that maps a pair of <subject, predicate>
to multiple possible corresponding <objects> in graph
"""
def make_sp2o(f_paths, e2idx, r2idx):
    sp2o = {}
    for f_path in f_paths:
      with open(f_path) as f:
        for line in f:
            line = line.rstrip()
            s, p, o = line.split()
            s, p, o = e2idx[s], r2idx[p], e2idx[o]
            if (s,p) not in sp2o:
                sp2o[(s,p)] = [o]
            else:
                if o not in sp2o[(s,p)]:
                    sp2o[(s,p)].append(o)
    return sp2o
In [0]:
def map_fn(x, y):
  i, v, s = y[0]
  one_hot = tf.SparseTensor(i, v, s)
  return x, (one_hot, y[1], y[2])


# stream data from text files
def data_generator(f_path, params, sp2o):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = line.rstrip()
      s, p, o = line.split()
      s, p, o = params['e2idx'][s], params['r2idx'][p], params['e2idx'][o]
      sparse_i = [[x] for x in sp2o[(s, p)]]
      sparse_v = [1.] * len(sparse_i)
      sparse_s = [len(params['e2idx'])]
      yield ((s, p), ((sparse_i, sparse_v, sparse_s), o, len(sparse_i)))


def dataset(is_training, params, sp2o):
  _shapes = (([], []), (([None, 1], [None], [1]), [], []))
  _types = ((tf.int32, tf.int32),
            ((tf.int64, tf.float32, tf.int64), tf.int32, tf.int32))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params, sp2o),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.map(map_fn)
    ds = ds.batch(params['batch_size'])
  
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params, sp2o),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.map(map_fn)
    ds = ds.batch(params['batch_size'])
  
  return ds
In [0]:
def update_metrics(scores, query, metrics):
  to_float = lambda x: tf.cast(x, tf.float32)
  
  _, i = tf.math.top_k(scores, sorted=True, k=scores.shape[1])
  query = tf.expand_dims(query, 1)
  is_query = to_float(tf.equal(i, query))
  r = tf.argmax(is_query, -1) + 1
  
  mrr = 1. / to_float(r)
  hits_10 = to_float(tf.less_equal(r, 10))
  hits_3 = to_float(tf.less_equal(r, 3))
  hits_1 = to_float(tf.less_equal(r, 1))
  
  metrics['mrr'].update_state(mrr)
  metrics['hits_10'].update_state(hits_10)
  metrics['hits_3'].update_state(hits_3)
  metrics['hits_1'].update_state(hits_1)
In [0]:
class DistMult(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.embed_ent = tf.keras.layers.Embedding(input_dim=len(params['e2idx']),
                                               output_dim=params['embed_dim'],
                                               embeddings_initializer=tf.initializers.RandomUniform(),
                                               name='Entity')
    self.embed_rel = tf.keras.layers.Embedding(input_dim=len(params['r2idx']),
                                               output_dim=params['embed_dim'],
                                               embeddings_initializer=tf.initializers.RandomUniform(),
                                               name='Relation')
    self.out_bias = self.add_weight(name='out_bias', shape=[len(params['e2idx'])])
  
  
  def call(self, inputs):
    s, p = inputs
    s = self.embed_ent(s)
    p = self.embed_rel(p)
    x = tf.matmul(s * p, self.embed_ent.embeddings, transpose_b=True)
    x = tf.nn.bias_add(x, self.out_bias)
    return x
In [0]:
def label_smoothing(inputs, epsilon):
  V = inputs.get_shape().as_list()[-1]
  return ((1-epsilon) * inputs) + (epsilon / V)
In [0]:
params = {
    'train_path': '../data/wn18/train.txt',
    'valid_path': '../data/wn18/valid.txt',
    'test_path': '../data/wn18/test.txt',
    'entity_path': '../vocab/entity.txt',
    'relation_path': '../vocab/relation.txt',
    'batch_size': 128,
    'embed_dim': 200,
    'num_samples': 141442,
    'init_lr': 1e-4,
    'max_lr': 3e-3,
    'num_patience': 10,
    'epsilon': .1,
}
In [0]:
params['e2idx'] = get_vocab(params['entity_path'])
params['r2idx'] = get_vocab(params['relation_path'])
sp2o_tr = make_sp2o([params['train_path']], params['e2idx'], params['r2idx'])
sp2o_all = make_sp2o([params['train_path'],
                      params['test_path'],
                      params['valid_path']], params['e2idx'], params['r2idx'])
In [12]:
model = DistMult(params)
model.build(input_shape=[[None], [None]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = Triangular2CyclicalLearningRate(
  initial_learning_rate = params['init_lr'],
  maximal_learning_rate = params['max_lr'],
  step_size = 8 * params['num_samples'] // params['batch_size'],)
optim = tf.optimizers.Adam(params['init_lr'])
global_step = 0

best_mrr = 0.
count = 0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)


while True:
  # TRAINING
  for ((s, p), (multi_o, o, num_pos)) in dataset(is_training=True, params=params, sp2o=sp2o_tr):
    with tf.GradientTape() as tape:
      logits = model((s, p))
      multi_o = tf.sparse.to_dense(multi_o, validate_indices=False)
      num_neg = len(params['e2idx']) - num_pos
      pos_weight = tf.expand_dims(tf.cast(num_neg/num_pos, tf.float32), 1)
      labels = label_smoothing(multi_o, params['epsilon'])
      loss = tf.nn.weighted_cross_entropy_with_logits(labels=labels, logits=logits, pos_weight=pos_weight)
      loss = tf.reduce_mean(loss)
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  metrics = {
    'mrr': tf.metrics.Mean(),
    'hits_10': tf.metrics.Mean(),
    'hits_3': tf.metrics.Mean(),
    'hits_1': tf.metrics.Mean(),
  }
  for ((s, p), (multi_o, o, num_pos)) in dataset(is_training=False, params=params, sp2o=sp2o_all):
    logits = model((s, p))
    multi_o = tf.sparse.to_dense(multi_o, validate_indices=False)
    # create masks for Filtered MRR
    o_one_hot = tf.one_hot(o, len(params['e2idx']))
    unwanted = multi_o - o_one_hot
    masks = tf.cast(tf.equal(unwanted, 0.), tf.float32)
    scores = tf.sigmoid(logits) * masks
    
    update_metrics(scores=scores, query=o, metrics=metrics)
  
  logger.info("MRR: {:.3f}| [email protected]: {:.3f} | [email protected]: {:.3f} | [email protected]: {:.3f}".format(
    metrics['mrr'].result().numpy(),
    metrics['hits_10'].result().numpy(),
    metrics['hits_3'].result().numpy(),
    metrics['hits_1'].result().numpy()))
  
  mrr = metrics['mrr'].result().numpy()
  
  if mrr > best_mrr:
    best_mrr = mrr
    # you can save model here
    count = 0
  else:
    count += 1
  logger.info("Best MRR: {:.3f}".format(best_mrr))
  
  if count == params['num_patience']:
    print(params['num_patience'], "times not improve the best result, therefore stop training")
    break
[('Entity/embeddings:0', TensorShape([40943, 200])),
 ('Relation/embeddings:0', TensorShape([18, 200])),
 ('out_bias:0', TensorShape([40943]))]
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 0 | Loss: 1.3553 | Spent: 32.1 secs | LR: 0.000100
INFO:tensorflow:Step 50 | Loss: 1.3627 | Spent: 4.0 secs | LR: 0.000116
INFO:tensorflow:Step 100 | Loss: 1.3609 | Spent: 4.0 secs | LR: 0.000133
INFO:tensorflow:Step 150 | Loss: 1.3597 | Spent: 4.0 secs | LR: 0.000149
INFO:tensorflow:Step 200 | Loss: 1.3536 | Spent: 4.0 secs | LR: 0.000166
INFO:tensorflow:Step 250 | Loss: 1.3564 | Spent: 4.0 secs | LR: 0.000182
INFO:tensorflow:Step 300 | Loss: 1.3542 | Spent: 4.0 secs | LR: 0.000198
INFO:tensorflow:Step 350 | Loss: 1.3544 | Spent: 4.0 secs | LR: 0.000215
INFO:tensorflow:Step 400 | Loss: 1.3534 | Spent: 4.0 secs | LR: 0.000231
INFO:tensorflow:Step 450 | Loss: 1.3496 | Spent: 4.0 secs | LR: 0.000248
INFO:tensorflow:Step 500 | Loss: 1.3546 | Spent: 4.0 secs | LR: 0.000264
INFO:tensorflow:Step 550 | Loss: 1.3560 | Spent: 4.0 secs | LR: 0.000280
INFO:tensorflow:Step 600 | Loss: 1.3520 | Spent: 4.0 secs | LR: 0.000297
INFO:tensorflow:Step 650 | Loss: 1.3487 | Spent: 4.0 secs | LR: 0.000313
INFO:tensorflow:Step 700 | Loss: 1.3521 | Spent: 4.0 secs | LR: 0.000330
INFO:tensorflow:Step 750 | Loss: 1.3482 | Spent: 4.0 secs | LR: 0.000346
INFO:tensorflow:Step 800 | Loss: 1.3559 | Spent: 4.0 secs | LR: 0.000362
INFO:tensorflow:Step 850 | Loss: 1.3450 | Spent: 4.0 secs | LR: 0.000379
INFO:tensorflow:Step 900 | Loss: 1.3468 | Spent: 4.0 secs | LR: 0.000395
INFO:tensorflow:Step 950 | Loss: 1.3480 | Spent: 4.0 secs | LR: 0.000412
INFO:tensorflow:Step 1000 | Loss: 1.3463 | Spent: 4.0 secs | LR: 0.000428
INFO:tensorflow:Step 1050 | Loss: 1.3427 | Spent: 4.0 secs | LR: 0.000444
INFO:tensorflow:Step 1100 | Loss: 1.3287 | Spent: 4.0 secs | LR: 0.000461
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.049| [email protected]: 0.056 | [email protected]: 0.049 | [email protected]: 0.041
INFO:tensorflow:Best MRR: 0.049
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 1150 | Loss: 1.3148 | Spent: 36.4 secs | LR: 0.000477
INFO:tensorflow:Step 1200 | Loss: 1.2857 | Spent: 4.0 secs | LR: 0.000494
INFO:tensorflow:Step 1250 | Loss: 1.2948 | Spent: 4.0 secs | LR: 0.000510
INFO:tensorflow:Step 1300 | Loss: 1.2604 | Spent: 4.0 secs | LR: 0.000526
INFO:tensorflow:Step 1350 | Loss: 1.2490 | Spent: 4.0 secs | LR: 0.000543
INFO:tensorflow:Step 1400 | Loss: 1.2085 | Spent: 4.0 secs | LR: 0.000559
INFO:tensorflow:Step 1450 | Loss: 1.2039 | Spent: 4.0 secs | LR: 0.000576
INFO:tensorflow:Step 1500 | Loss: 1.1682 | Spent: 4.0 secs | LR: 0.000592
INFO:tensorflow:Step 1550 | Loss: 1.1411 | Spent: 4.0 secs | LR: 0.000608
INFO:tensorflow:Step 1600 | Loss: 1.1287 | Spent: 4.0 secs | LR: 0.000625
INFO:tensorflow:Step 1650 | Loss: 1.1098 | Spent: 4.0 secs | LR: 0.000641
INFO:tensorflow:Step 1700 | Loss: 1.0511 | Spent: 4.0 secs | LR: 0.000658
INFO:tensorflow:Step 1750 | Loss: 1.0533 | Spent: 4.0 secs | LR: 0.000674
INFO:tensorflow:Step 1800 | Loss: 1.0379 | Spent: 4.0 secs | LR: 0.000690
INFO:tensorflow:Step 1850 | Loss: 1.0044 | Spent: 4.0 secs | LR: 0.000707
INFO:tensorflow:Step 1900 | Loss: 0.9724 | Spent: 4.0 secs | LR: 0.000723
INFO:tensorflow:Step 1950 | Loss: 0.9719 | Spent: 4.0 secs | LR: 0.000740
INFO:tensorflow:Step 2000 | Loss: 0.9627 | Spent: 4.0 secs | LR: 0.000756
INFO:tensorflow:Step 2050 | Loss: 0.8906 | Spent: 4.0 secs | LR: 0.000773
INFO:tensorflow:Step 2100 | Loss: 0.8760 | Spent: 4.0 secs | LR: 0.000789
INFO:tensorflow:Step 2150 | Loss: 0.8780 | Spent: 4.0 secs | LR: 0.000805
INFO:tensorflow:Step 2200 | Loss: 0.8175 | Spent: 4.0 secs | LR: 0.000822
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.343| [email protected]: 0.474 | [email protected]: 0.370 | [email protected]: 0.276
INFO:tensorflow:Best MRR: 0.343
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 2250 | Loss: 0.7731 | Spent: 36.1 secs | LR: 0.000838
INFO:tensorflow:Step 2300 | Loss: 0.7420 | Spent: 4.0 secs | LR: 0.000855
INFO:tensorflow:Step 2350 | Loss: 0.7524 | Spent: 4.0 secs | LR: 0.000871
INFO:tensorflow:Step 2400 | Loss: 0.7084 | Spent: 4.0 secs | LR: 0.000887
INFO:tensorflow:Step 2450 | Loss: 0.6626 | Spent: 4.0 secs | LR: 0.000904
INFO:tensorflow:Step 2500 | Loss: 0.6169 | Spent: 4.0 secs | LR: 0.000920
INFO:tensorflow:Step 2550 | Loss: 0.6108 | Spent: 4.0 secs | LR: 0.000937
INFO:tensorflow:Step 2600 | Loss: 0.5910 | Spent: 4.0 secs | LR: 0.000953
INFO:tensorflow:Step 2650 | Loss: 0.5527 | Spent: 4.0 secs | LR: 0.000969
INFO:tensorflow:Step 2700 | Loss: 0.5178 | Spent: 4.0 secs | LR: 0.000986
INFO:tensorflow:Step 2750 | Loss: 0.5138 | Spent: 4.0 secs | LR: 0.001002
INFO:tensorflow:Step 2800 | Loss: 0.5030 | Spent: 4.0 secs | LR: 0.001019
INFO:tensorflow:Step 2850 | Loss: 0.4962 | Spent: 4.1 secs | LR: 0.001035
INFO:tensorflow:Step 2900 | Loss: 0.4725 | Spent: 4.0 secs | LR: 0.001051
INFO:tensorflow:Step 2950 | Loss: 0.4723 | Spent: 4.0 secs | LR: 0.001068
INFO:tensorflow:Step 3000 | Loss: 0.4273 | Spent: 4.0 secs | LR: 0.001084
INFO:tensorflow:Step 3050 | Loss: 0.4137 | Spent: 4.0 secs | LR: 0.001101
INFO:tensorflow:Step 3100 | Loss: 0.3713 | Spent: 4.0 secs | LR: 0.001117
INFO:tensorflow:Step 3150 | Loss: 0.3965 | Spent: 4.0 secs | LR: 0.001133
INFO:tensorflow:Step 3200 | Loss: 0.3403 | Spent: 4.0 secs | LR: 0.001150
INFO:tensorflow:Step 3250 | Loss: 0.3343 | Spent: 4.0 secs | LR: 0.001166
INFO:tensorflow:Step 3300 | Loss: 0.3481 | Spent: 4.0 secs | LR: 0.001183
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.482| [email protected]: 0.803 | [email protected]: 0.585 | [email protected]: 0.320
INFO:tensorflow:Best MRR: 0.482
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 3350 | Loss: 0.2873 | Spent: 35.9 secs | LR: 0.001199
INFO:tensorflow:Step 3400 | Loss: 0.2908 | Spent: 4.0 secs | LR: 0.001215
INFO:tensorflow:Step 3450 | Loss: 0.2916 | Spent: 4.0 secs | LR: 0.001232
INFO:tensorflow:Step 3500 | Loss: 0.2801 | Spent: 4.0 secs | LR: 0.001248
INFO:tensorflow:Step 3550 | Loss: 0.2878 | Spent: 4.0 secs | LR: 0.001265
INFO:tensorflow:Step 3600 | Loss: 0.2671 | Spent: 4.0 secs | LR: 0.001281
INFO:tensorflow:Step 3650 | Loss: 0.2844 | Spent: 4.0 secs | LR: 0.001297
INFO:tensorflow:Step 3700 | Loss: 0.2860 | Spent: 4.0 secs | LR: 0.001314
INFO:tensorflow:Step 3750 | Loss: 0.2771 | Spent: 4.0 secs | LR: 0.001330
INFO:tensorflow:Step 3800 | Loss: 0.2388 | Spent: 3.9 secs | LR: 0.001347
INFO:tensorflow:Step 3850 | Loss: 0.2420 | Spent: 3.9 secs | LR: 0.001363
INFO:tensorflow:Step 3900 | Loss: 0.2350 | Spent: 4.0 secs | LR: 0.001379
INFO:tensorflow:Step 3950 | Loss: 0.2514 | Spent: 3.9 secs | LR: 0.001396
INFO:tensorflow:Step 4000 | Loss: 0.2493 | Spent: 4.0 secs | LR: 0.001412
INFO:tensorflow:Step 4050 | Loss: 0.2436 | Spent: 4.0 secs | LR: 0.001429
INFO:tensorflow:Step 4100 | Loss: 0.2627 | Spent: 4.0 secs | LR: 0.001445
INFO:tensorflow:Step 4150 | Loss: 0.2586 | Spent: 4.0 secs | LR: 0.001461
INFO:tensorflow:Step 4200 | Loss: 0.2694 | Spent: 4.0 secs | LR: 0.001478
INFO:tensorflow:Step 4250 | Loss: 0.2796 | Spent: 4.0 secs | LR: 0.001494
INFO:tensorflow:Step 4300 | Loss: 0.2518 | Spent: 4.0 secs | LR: 0.001511
INFO:tensorflow:Step 4350 | Loss: 0.2651 | Spent: 4.0 secs | LR: 0.001527
INFO:tensorflow:Step 4400 | Loss: 0.2332 | Spent: 4.1 secs | LR: 0.001543
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.704| [email protected]: 0.924 | [email protected]: 0.848 | [email protected]: 0.548
INFO:tensorflow:Best MRR: 0.704
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 4450 | Loss: 0.2396 | Spent: 36.7 secs | LR: 0.001560
INFO:tensorflow:Step 4500 | Loss: 0.2260 | Spent: 4.0 secs | LR: 0.001576
INFO:tensorflow:Step 4550 | Loss: 0.2359 | Spent: 4.0 secs | LR: 0.001593
INFO:tensorflow:Step 4600 | Loss: 0.2678 | Spent: 4.0 secs | LR: 0.001609
INFO:tensorflow:Step 4650 | Loss: 0.2301 | Spent: 4.0 secs | LR: 0.001625
INFO:tensorflow:Step 4700 | Loss: 0.2305 | Spent: 4.0 secs | LR: 0.001642
INFO:tensorflow:Step 4750 | Loss: 0.2295 | Spent: 4.0 secs | LR: 0.001658
INFO:tensorflow:Step 4800 | Loss: 0.2416 | Spent: 4.0 secs | LR: 0.001675
INFO:tensorflow:Step 4850 | Loss: 0.2256 | Spent: 4.0 secs | LR: 0.001691
INFO:tensorflow:Step 4900 | Loss: 0.2484 | Spent: 4.0 secs | LR: 0.001707
INFO:tensorflow:Step 4950 | Loss: 0.2199 | Spent: 4.0 secs | LR: 0.001724
INFO:tensorflow:Step 5000 | Loss: 0.2642 | Spent: 4.0 secs | LR: 0.001740
INFO:tensorflow:Step 5050 | Loss: 0.2321 | Spent: 4.0 secs | LR: 0.001757
INFO:tensorflow:Step 5100 | Loss: 0.2492 | Spent: 4.0 secs | LR: 0.001773
INFO:tensorflow:Step 5150 | Loss: 0.2454 | Spent: 4.0 secs | LR: 0.001789
INFO:tensorflow:Step 5200 | Loss: 0.2404 | Spent: 4.0 secs | LR: 0.001806
INFO:tensorflow:Step 5250 | Loss: 0.2371 | Spent: 4.0 secs | LR: 0.001822
INFO:tensorflow:Step 5300 | Loss: 0.2447 | Spent: 4.0 secs | LR: 0.001839
INFO:tensorflow:Step 5350 | Loss: 0.2580 | Spent: 4.0 secs | LR: 0.001855
INFO:tensorflow:Step 5400 | Loss: 0.2449 | Spent: 4.0 secs | LR: 0.001871
INFO:tensorflow:Step 5450 | Loss: 0.2307 | Spent: 4.0 secs | LR: 0.001888
INFO:tensorflow:Step 5500 | Loss: 0.2600 | Spent: 4.0 secs | LR: 0.001904
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.757| [email protected]: 0.939 | [email protected]: 0.896 | [email protected]: 0.616
INFO:tensorflow:Best MRR: 0.757
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 5550 | Loss: 0.2526 | Spent: 36.7 secs | LR: 0.001921
INFO:tensorflow:Step 5600 | Loss: 0.2216 | Spent: 4.1 secs | LR: 0.001937
INFO:tensorflow:Step 5650 | Loss: 0.2620 | Spent: 4.0 secs | LR: 0.001954
INFO:tensorflow:Step 5700 | Loss: 0.2688 | Spent: 4.0 secs | LR: 0.001970
INFO:tensorflow:Step 5750 | Loss: 0.2488 | Spent: 4.0 secs | LR: 0.001986
INFO:tensorflow:Step 5800 | Loss: 0.2376 | Spent: 4.0 secs | LR: 0.002003
INFO:tensorflow:Step 5850 | Loss: 0.2249 | Spent: 4.0 secs | LR: 0.002019
INFO:tensorflow:Step 5900 | Loss: 0.2596 | Spent: 4.0 secs | LR: 0.002036
INFO:tensorflow:Step 5950 | Loss: 0.2371 | Spent: 4.0 secs | LR: 0.002052
INFO:tensorflow:Step 6000 | Loss: 0.2265 | Spent: 4.0 secs | LR: 0.002068
INFO:tensorflow:Step 6050 | Loss: 0.2421 | Spent: 4.0 secs | LR: 0.002085
INFO:tensorflow:Step 6100 | Loss: 0.2266 | Spent: 4.0 secs | LR: 0.002101
INFO:tensorflow:Step 6150 | Loss: 0.2317 | Spent: 4.0 secs | LR: 0.002118
INFO:tensorflow:Step 6200 | Loss: 0.2363 | Spent: 4.0 secs | LR: 0.002134
INFO:tensorflow:Step 6250 | Loss: 0.2471 | Spent: 4.1 secs | LR: 0.002150
INFO:tensorflow:Step 6300 | Loss: 0.2447 | Spent: 4.0 secs | LR: 0.002167
INFO:tensorflow:Step 6350 | Loss: 0.2421 | Spent: 4.0 secs | LR: 0.002183
INFO:tensorflow:Step 6400 | Loss: 0.2482 | Spent: 4.0 secs | LR: 0.002200
INFO:tensorflow:Step 6450 | Loss: 0.2383 | Spent: 4.0 secs | LR: 0.002216
INFO:tensorflow:Step 6500 | Loss: 0.2379 | Spent: 4.0 secs | LR: 0.002232
INFO:tensorflow:Step 6550 | Loss: 0.2281 | Spent: 4.0 secs | LR: 0.002249
INFO:tensorflow:Step 6600 | Loss: 0.2357 | Spent: 4.0 secs | LR: 0.002265
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.776| [email protected]: 0.945 | [email protected]: 0.910 | [email protected]: 0.643
INFO:tensorflow:Best MRR: 0.776
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 6650 | Loss: 0.2151 | Spent: 35.9 secs | LR: 0.002282
INFO:tensorflow:Step 6700 | Loss: 0.2476 | Spent: 4.1 secs | LR: 0.002298
INFO:tensorflow:Step 6750 | Loss: 0.2378 | Spent: 4.0 secs | LR: 0.002314
INFO:tensorflow:Step 6800 | Loss: 0.2367 | Spent: 4.0 secs | LR: 0.002331
INFO:tensorflow:Step 6850 | Loss: 0.2473 | Spent: 4.0 secs | LR: 0.002347
INFO:tensorflow:Step 6900 | Loss: 0.2432 | Spent: 4.0 secs | LR: 0.002364
INFO:tensorflow:Step 6950 | Loss: 0.2304 | Spent: 4.0 secs | LR: 0.002380
INFO:tensorflow:Step 7000 | Loss: 0.2337 | Spent: 4.0 secs | LR: 0.002396
INFO:tensorflow:Step 7050 | Loss: 0.2351 | Spent: 4.0 secs | LR: 0.002413
INFO:tensorflow:Step 7100 | Loss: 0.2460 | Spent: 4.0 secs | LR: 0.002429
INFO:tensorflow:Step 7150 | Loss: 0.2551 | Spent: 4.0 secs | LR: 0.002446
INFO:tensorflow:Step 7200 | Loss: 0.2374 | Spent: 3.9 secs | LR: 0.002462
INFO:tensorflow:Step 7250 | Loss: 0.2396 | Spent: 4.0 secs | LR: 0.002478
INFO:tensorflow:Step 7300 | Loss: 0.2541 | Spent: 4.0 secs | LR: 0.002495
INFO:tensorflow:Step 7350 | Loss: 0.2249 | Spent: 4.0 secs | LR: 0.002511
INFO:tensorflow:Step 7400 | Loss: 0.2409 | Spent: 4.0 secs | LR: 0.002528
INFO:tensorflow:Step 7450 | Loss: 0.2399 | Spent: 4.0 secs | LR: 0.002544
INFO:tensorflow:Step 7500 | Loss: 0.2391 | Spent: 4.0 secs | LR: 0.002560
INFO:tensorflow:Step 7550 | Loss: 0.2262 | Spent: 4.0 secs | LR: 0.002577
INFO:tensorflow:Step 7600 | Loss: 0.2340 | Spent: 4.0 secs | LR: 0.002593
INFO:tensorflow:Step 7650 | Loss: 0.2234 | Spent: 4.0 secs | LR: 0.002610
INFO:tensorflow:Step 7700 | Loss: 0.2540 | Spent: 4.0 secs | LR: 0.002626
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.786| [email protected]: 0.943 | [email protected]: 0.903 | [email protected]: 0.664
INFO:tensorflow:Best MRR: 0.786
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 7750 | Loss: 0.2372 | Spent: 36.1 secs | LR: 0.002642
INFO:tensorflow:Step 7800 | Loss: 0.2406 | Spent: 4.0 secs | LR: 0.002659
INFO:tensorflow:Step 7850 | Loss: 0.2213 | Spent: 4.0 secs | LR: 0.002675
INFO:tensorflow:Step 7900 | Loss: 0.2418 | Spent: 4.0 secs | LR: 0.002692
INFO:tensorflow:Step 7950 | Loss: 0.2363 | Spent: 4.0 secs | LR: 0.002708
INFO:tensorflow:Step 8000 | Loss: 0.2403 | Spent: 4.0 secs | LR: 0.002724
INFO:tensorflow:Step 8050 | Loss: 0.2382 | Spent: 4.0 secs | LR: 0.002741
INFO:tensorflow:Step 8100 | Loss: 0.2274 | Spent: 4.0 secs | LR: 0.002757
INFO:tensorflow:Step 8150 | Loss: 0.2542 | Spent: 4.0 secs | LR: 0.002774
INFO:tensorflow:Step 8200 | Loss: 0.2428 | Spent: 4.0 secs | LR: 0.002790
INFO:tensorflow:Step 8250 | Loss: 0.2449 | Spent: 4.0 secs | LR: 0.002806
INFO:tensorflow:Step 8300 | Loss: 0.2486 | Spent: 4.0 secs | LR: 0.002823
INFO:tensorflow:Step 8350 | Loss: 0.2435 | Spent: 4.0 secs | LR: 0.002839
INFO:tensorflow:Step 8400 | Loss: 0.2206 | Spent: 4.0 secs | LR: 0.002856
INFO:tensorflow:Step 8450 | Loss: 0.2406 | Spent: 4.0 secs | LR: 0.002872
INFO:tensorflow:Step 8500 | Loss: 0.2405 | Spent: 4.0 secs | LR: 0.002888
INFO:tensorflow:Step 8550 | Loss: 0.2451 | Spent: 4.0 secs | LR: 0.002905
INFO:tensorflow:Step 8600 | Loss: 0.2287 | Spent: 4.0 secs | LR: 0.002921
INFO:tensorflow:Step 8650 | Loss: 0.2521 | Spent: 4.0 secs | LR: 0.002938
INFO:tensorflow:Step 8700 | Loss: 0.2294 | Spent: 4.0 secs | LR: 0.002954
INFO:tensorflow:Step 8750 | Loss: 0.2546 | Spent: 4.0 secs | LR: 0.002970
INFO:tensorflow:Step 8800 | Loss: 0.2292 | Spent: 4.0 secs | LR: 0.002987
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.786| [email protected]: 0.937 | [email protected]: 0.895 | [email protected]: 0.671
INFO:tensorflow:Best MRR: 0.786
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 8850 | Loss: 0.2286 | Spent: 35.8 secs | LR: 0.002997
INFO:tensorflow:Step 8900 | Loss: 0.2333 | Spent: 4.0 secs | LR: 0.002980
INFO:tensorflow:Step 8950 | Loss: 0.2372 | Spent: 4.0 secs | LR: 0.002964
INFO:tensorflow:Step 9000 | Loss: 0.2453 | Spent: 4.0 secs | LR: 0.002948
INFO:tensorflow:Step 9050 | Loss: 0.2387 | Spent: 4.0 secs | LR: 0.002931
INFO:tensorflow:Step 9100 | Loss: 0.2696 | Spent: 4.0 secs | LR: 0.002915
INFO:tensorflow:Step 9150 | Loss: 0.2392 | Spent: 4.0 secs | LR: 0.002898
INFO:tensorflow:Step 9200 | Loss: 0.2451 | Spent: 4.0 secs | LR: 0.002882
INFO:tensorflow:Step 9250 | Loss: 0.2506 | Spent: 4.0 secs | LR: 0.002865
INFO:tensorflow:Step 9300 | Loss: 0.2390 | Spent: 3.9 secs | LR: 0.002849
INFO:tensorflow:Step 9350 | Loss: 0.2394 | Spent: 3.9 secs | LR: 0.002833
INFO:tensorflow:Step 9400 | Loss: 0.2478 | Spent: 4.0 secs | LR: 0.002816
INFO:tensorflow:Step 9450 | Loss: 0.2371 | Spent: 4.0 secs | LR: 0.002800
INFO:tensorflow:Step 9500 | Loss: 0.2389 | Spent: 4.0 secs | LR: 0.002783
INFO:tensorflow:Step 9550 | Loss: 0.2225 | Spent: 4.0 secs | LR: 0.002767
INFO:tensorflow:Step 9600 | Loss: 0.2307 | Spent: 4.0 secs | LR: 0.002751
INFO:tensorflow:Step 9650 | Loss: 0.2427 | Spent: 3.9 secs | LR: 0.002734
INFO:tensorflow:Step 9700 | Loss: 0.2404 | Spent: 4.0 secs | LR: 0.002718
INFO:tensorflow:Step 9750 | Loss: 0.2559 | Spent: 4.0 secs | LR: 0.002701
INFO:tensorflow:Step 9800 | Loss: 0.2292 | Spent: 4.0 secs | LR: 0.002685
INFO:tensorflow:Step 9850 | Loss: 0.2510 | Spent: 4.0 secs | LR: 0.002669
INFO:tensorflow:Step 9900 | Loss: 0.2259 | Spent: 4.0 secs | LR: 0.002652
INFO:tensorflow:Step 9950 | Loss: 0.2185 | Spent: 4.0 secs | LR: 0.002636
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.793| [email protected]: 0.936 | [email protected]: 0.897 | [email protected]: 0.683
INFO:tensorflow:Best MRR: 0.793
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 10000 | Loss: 0.2455 | Spent: 35.9 secs | LR: 0.002619
INFO:tensorflow:Step 10050 | Loss: 0.2523 | Spent: 4.0 secs | LR: 0.002603
INFO:tensorflow:Step 10100 | Loss: 0.2268 | Spent: 4.0 secs | LR: 0.002587
INFO:tensorflow:Step 10150 | Loss: 0.2472 | Spent: 4.0 secs | LR: 0.002570
INFO:tensorflow:Step 10200 | Loss: 0.2197 | Spent: 4.0 secs | LR: 0.002554
INFO:tensorflow:Step 10250 | Loss: 0.2512 | Spent: 4.0 secs | LR: 0.002537
INFO:tensorflow:Step 10300 | Loss: 0.2422 | Spent: 4.0 secs | LR: 0.002521
INFO:tensorflow:Step 10350 | Loss: 0.2328 | Spent: 3.9 secs | LR: 0.002505
INFO:tensorflow:Step 10400 | Loss: 0.2427 | Spent: 3.9 secs | LR: 0.002488
INFO:tensorflow:Step 10450 | Loss: 0.2519 | Spent: 4.0 secs | LR: 0.002472
INFO:tensorflow:Step 10500 | Loss: 0.2424 | Spent: 3.9 secs | LR: 0.002455
INFO:tensorflow:Step 10550 | Loss: 0.2400 | Spent: 4.0 secs | LR: 0.002439
INFO:tensorflow:Step 10600 | Loss: 0.2287 | Spent: 4.0 secs | LR: 0.002423
INFO:tensorflow:Step 10650 | Loss: 0.2342 | Spent: 4.0 secs | LR: 0.002406
INFO:tensorflow:Step 10700 | Loss: 0.2408 | Spent: 4.0 secs | LR: 0.002390
INFO:tensorflow:Step 10750 | Loss: 0.2330 | Spent: 4.0 secs | LR: 0.002373
INFO:tensorflow:Step 10800 | Loss: 0.2282 | Spent: 4.0 secs | LR: 0.002357
INFO:tensorflow:Step 10850 | Loss: 0.2398 | Spent: 4.0 secs | LR: 0.002341
INFO:tensorflow:Step 10900 | Loss: 0.2580 | Spent: 4.0 secs | LR: 0.002324
INFO:tensorflow:Step 10950 | Loss: 0.2452 | Spent: 4.0 secs | LR: 0.002308
INFO:tensorflow:Step 11000 | Loss: 0.2407 | Spent: 4.0 secs | LR: 0.002291
INFO:tensorflow:Step 11050 | Loss: 0.2528 | Spent: 4.0 secs | LR: 0.002275
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.797| [email protected]: 0.938 | [email protected]: 0.902 | [email protected]: 0.688
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 11100 | Loss: 0.2446 | Spent: 36.0 secs | LR: 0.002259
INFO:tensorflow:Step 11150 | Loss: 0.2163 | Spent: 4.0 secs | LR: 0.002242
INFO:tensorflow:Step 11200 | Loss: 0.2644 | Spent: 3.9 secs | LR: 0.002226
INFO:tensorflow:Step 11250 | Loss: 0.2572 | Spent: 4.0 secs | LR: 0.002209
INFO:tensorflow:Step 11300 | Loss: 0.2416 | Spent: 4.0 secs | LR: 0.002193
INFO:tensorflow:Step 11350 | Loss: 0.2394 | Spent: 4.0 secs | LR: 0.002177
INFO:tensorflow:Step 11400 | Loss: 0.2501 | Spent: 4.0 secs | LR: 0.002160
INFO:tensorflow:Step 11450 | Loss: 0.2384 | Spent: 4.0 secs | LR: 0.002144
INFO:tensorflow:Step 11500 | Loss: 0.2354 | Spent: 4.0 secs | LR: 0.002127
INFO:tensorflow:Step 11550 | Loss: 0.2235 | Spent: 3.9 secs | LR: 0.002111
INFO:tensorflow:Step 11600 | Loss: 0.2320 | Spent: 3.9 secs | LR: 0.002095
INFO:tensorflow:Step 11650 | Loss: 0.2522 | Spent: 3.9 secs | LR: 0.002078
INFO:tensorflow:Step 11700 | Loss: 0.2508 | Spent: 3.9 secs | LR: 0.002062
INFO:tensorflow:Step 11750 | Loss: 0.2407 | Spent: 4.0 secs | LR: 0.002045
INFO:tensorflow:Step 11800 | Loss: 0.2291 | Spent: 4.0 secs | LR: 0.002029
INFO:tensorflow:Step 11850 | Loss: 0.2416 | Spent: 4.0 secs | LR: 0.002013
INFO:tensorflow:Step 11900 | Loss: 0.2315 | Spent: 4.0 secs | LR: 0.001996
INFO:tensorflow:Step 11950 | Loss: 0.2374 | Spent: 3.9 secs | LR: 0.001980
INFO:tensorflow:Step 12000 | Loss: 0.2481 | Spent: 4.0 secs | LR: 0.001963
INFO:tensorflow:Step 12050 | Loss: 0.2326 | Spent: 4.0 secs | LR: 0.001947
INFO:tensorflow:Step 12100 | Loss: 0.2323 | Spent: 3.9 secs | LR: 0.001931
INFO:tensorflow:Step 12150 | Loss: 0.2150 | Spent: 4.0 secs | LR: 0.001914
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.794| [email protected]: 0.942 | [email protected]: 0.904 | [email protected]: 0.679
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 12200 | Loss: 0.2076 | Spent: 35.7 secs | LR: 0.001898
INFO:tensorflow:Step 12250 | Loss: 0.2196 | Spent: 4.0 secs | LR: 0.001881
INFO:tensorflow:Step 12300 | Loss: 0.2376 | Spent: 4.0 secs | LR: 0.001865
INFO:tensorflow:Step 12350 | Loss: 0.2326 | Spent: 3.9 secs | LR: 0.001849
INFO:tensorflow:Step 12400 | Loss: 0.2380 | Spent: 3.9 secs | LR: 0.001832
INFO:tensorflow:Step 12450 | Loss: 0.2417 | Spent: 4.0 secs | LR: 0.001816
INFO:tensorflow:Step 12500 | Loss: 0.2465 | Spent: 4.0 secs | LR: 0.001799
INFO:tensorflow:Step 12550 | Loss: 0.2401 | Spent: 4.0 secs | LR: 0.001783
INFO:tensorflow:Step 12600 | Loss: 0.2209 | Spent: 3.9 secs | LR: 0.001767
INFO:tensorflow:Step 12650 | Loss: 0.2305 | Spent: 3.9 secs | LR: 0.001750
INFO:tensorflow:Step 12700 | Loss: 0.2219 | Spent: 4.0 secs | LR: 0.001734
INFO:tensorflow:Step 12750 | Loss: 0.2336 | Spent: 3.9 secs | LR: 0.001717
INFO:tensorflow:Step 12800 | Loss: 0.2364 | Spent: 3.9 secs | LR: 0.001701
INFO:tensorflow:Step 12850 | Loss: 0.2335 | Spent: 4.0 secs | LR: 0.001685
INFO:tensorflow:Step 12900 | Loss: 0.2101 | Spent: 4.0 secs | LR: 0.001668
INFO:tensorflow:Step 12950 | Loss: 0.2239 | Spent: 4.0 secs | LR: 0.001652
INFO:tensorflow:Step 13000 | Loss: 0.2301 | Spent: 4.0 secs | LR: 0.001635
INFO:tensorflow:Step 13050 | Loss: 0.2455 | Spent: 3.9 secs | LR: 0.001619
INFO:tensorflow:Step 13100 | Loss: 0.2448 | Spent: 4.0 secs | LR: 0.001602
INFO:tensorflow:Step 13150 | Loss: 0.2552 | Spent: 3.9 secs | LR: 0.001586
INFO:tensorflow:Step 13200 | Loss: 0.2299 | Spent: 3.9 secs | LR: 0.001570
INFO:tensorflow:Step 13250 | Loss: 0.2243 | Spent: 4.0 secs | LR: 0.001553
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.791| [email protected]: 0.945 | [email protected]: 0.912 | [email protected]: 0.667
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 13300 | Loss: 0.2198 | Spent: 35.5 secs | LR: 0.001537
INFO:tensorflow:Step 13350 | Loss: 0.2415 | Spent: 4.0 secs | LR: 0.001520
INFO:tensorflow:Step 13400 | Loss: 0.2158 | Spent: 4.0 secs | LR: 0.001504
INFO:tensorflow:Step 13450 | Loss: 0.2434 | Spent: 3.9 secs | LR: 0.001488
INFO:tensorflow:Step 13500 | Loss: 0.2432 | Spent: 4.0 secs | LR: 0.001471
INFO:tensorflow:Step 13550 | Loss: 0.2355 | Spent: 4.0 secs | LR: 0.001455
INFO:tensorflow:Step 13600 | Loss: 0.2258 | Spent: 4.0 secs | LR: 0.001438
INFO:tensorflow:Step 13650 | Loss: 0.2029 | Spent: 3.9 secs | LR: 0.001422
INFO:tensorflow:Step 13700 | Loss: 0.2275 | Spent: 4.0 secs | LR: 0.001406
INFO:tensorflow:Step 13750 | Loss: 0.2380 | Spent: 3.9 secs | LR: 0.001389
INFO:tensorflow:Step 13800 | Loss: 0.2231 | Spent: 4.0 secs | LR: 0.001373
INFO:tensorflow:Step 13850 | Loss: 0.2358 | Spent: 4.0 secs | LR: 0.001356
INFO:tensorflow:Step 13900 | Loss: 0.2420 | Spent: 4.0 secs | LR: 0.001340
INFO:tensorflow:Step 13950 | Loss: 0.2256 | Spent: 3.9 secs | LR: 0.001324
INFO:tensorflow:Step 14000 | Loss: 0.2389 | Spent: 3.9 secs | LR: 0.001307
INFO:tensorflow:Step 14050 | Loss: 0.2447 | Spent: 3.9 secs | LR: 0.001291
INFO:tensorflow:Step 14100 | Loss: 0.2416 | Spent: 3.9 secs | LR: 0.001274
INFO:tensorflow:Step 14150 | Loss: 0.2352 | Spent: 4.0 secs | LR: 0.001258
INFO:tensorflow:Step 14200 | Loss: 0.2441 | Spent: 4.0 secs | LR: 0.001242
INFO:tensorflow:Step 14250 | Loss: 0.2220 | Spent: 4.0 secs | LR: 0.001225
INFO:tensorflow:Step 14300 | Loss: 0.2330 | Spent: 4.0 secs | LR: 0.001209
INFO:tensorflow:Step 14350 | Loss: 0.2271 | Spent: 3.9 secs | LR: 0.001192
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.781| [email protected]: 0.944 | [email protected]: 0.912 | [email protected]: 0.649
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 14400 | Loss: 0.2401 | Spent: 36.4 secs | LR: 0.001176
INFO:tensorflow:Step 14450 | Loss: 0.2464 | Spent: 4.0 secs | LR: 0.001160
INFO:tensorflow:Step 14500 | Loss: 0.2417 | Spent: 3.9 secs | LR: 0.001143
INFO:tensorflow:Step 14550 | Loss: 0.2293 | Spent: 3.9 secs | LR: 0.001127
INFO:tensorflow:Step 14600 | Loss: 0.2376 | Spent: 4.0 secs | LR: 0.001110
INFO:tensorflow:Step 14650 | Loss: 0.2372 | Spent: 3.9 secs | LR: 0.001094
INFO:tensorflow:Step 14700 | Loss: 0.2310 | Spent: 3.9 secs | LR: 0.001078
INFO:tensorflow:Step 14750 | Loss: 0.2602 | Spent: 3.9 secs | LR: 0.001061
INFO:tensorflow:Step 14800 | Loss: 0.2399 | Spent: 4.0 secs | LR: 0.001045
INFO:tensorflow:Step 14850 | Loss: 0.2086 | Spent: 4.0 secs | LR: 0.001028
INFO:tensorflow:Step 14900 | Loss: 0.2308 | Spent: 4.0 secs | LR: 0.001012
INFO:tensorflow:Step 14950 | Loss: 0.2257 | Spent: 3.9 secs | LR: 0.000996
INFO:tensorflow:Step 15000 | Loss: 0.2464 | Spent: 3.9 secs | LR: 0.000979
INFO:tensorflow:Step 15050 | Loss: 0.2477 | Spent: 3.9 secs | LR: 0.000963
INFO:tensorflow:Step 15100 | Loss: 0.2164 | Spent: 4.0 secs | LR: 0.000946
INFO:tensorflow:Step 15150 | Loss: 0.2276 | Spent: 4.0 secs | LR: 0.000930
INFO:tensorflow:Step 15200 | Loss: 0.2356 | Spent: 4.0 secs | LR: 0.000914
INFO:tensorflow:Step 15250 | Loss: 0.2461 | Spent: 4.0 secs | LR: 0.000897
INFO:tensorflow:Step 15300 | Loss: 0.2477 | Spent: 4.0 secs | LR: 0.000881
INFO:tensorflow:Step 15350 | Loss: 0.2373 | Spent: 4.0 secs | LR: 0.000864
INFO:tensorflow:Step 15400 | Loss: 0.2413 | Spent: 4.0 secs | LR: 0.000848
INFO:tensorflow:Step 15450 | Loss: 0.2301 | Spent: 4.0 secs | LR: 0.000832
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.778| [email protected]: 0.946 | [email protected]: 0.914 | [email protected]: 0.640
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 15500 | Loss: 0.2423 | Spent: 35.9 secs | LR: 0.000815
INFO:tensorflow:Step 15550 | Loss: 0.2322 | Spent: 4.0 secs | LR: 0.000799
INFO:tensorflow:Step 15600 | Loss: 0.2305 | Spent: 4.0 secs | LR: 0.000782
INFO:tensorflow:Step 15650 | Loss: 0.2337 | Spent: 4.0 secs | LR: 0.000766
INFO:tensorflow:Step 15700 | Loss: 0.2263 | Spent: 4.0 secs | LR: 0.000750
INFO:tensorflow:Step 15750 | Loss: 0.2461 | Spent: 4.0 secs | LR: 0.000733
INFO:tensorflow:Step 15800 | Loss: 0.2250 | Spent: 4.0 secs | LR: 0.000717
INFO:tensorflow:Step 15850 | Loss: 0.2397 | Spent: 4.0 secs | LR: 0.000700
INFO:tensorflow:Step 15900 | Loss: 0.2317 | Spent: 4.0 secs | LR: 0.000684
INFO:tensorflow:Step 15950 | Loss: 0.2327 | Spent: 4.0 secs | LR: 0.000668
INFO:tensorflow:Step 16000 | Loss: 0.2495 | Spent: 3.9 secs | LR: 0.000651
INFO:tensorflow:Step 16050 | Loss: 0.2160 | Spent: 4.0 secs | LR: 0.000635
INFO:tensorflow:Step 16100 | Loss: 0.2352 | Spent: 4.0 secs | LR: 0.000618
INFO:tensorflow:Step 16150 | Loss: 0.2268 | Spent: 4.0 secs | LR: 0.000602
INFO:tensorflow:Step 16200 | Loss: 0.2452 | Spent: 4.0 secs | LR: 0.000586
INFO:tensorflow:Step 16250 | Loss: 0.2300 | Spent: 4.0 secs | LR: 0.000569
INFO:tensorflow:Step 16300 | Loss: 0.2395 | Spent: 3.9 secs | LR: 0.000553
INFO:tensorflow:Step 16350 | Loss: 0.2288 | Spent: 3.9 secs | LR: 0.000536
INFO:tensorflow:Step 16400 | Loss: 0.2384 | Spent: 4.0 secs | LR: 0.000520
INFO:tensorflow:Step 16450 | Loss: 0.2217 | Spent: 4.0 secs | LR: 0.000504
INFO:tensorflow:Step 16500 | Loss: 0.2321 | Spent: 4.0 secs | LR: 0.000487
INFO:tensorflow:Step 16550 | Loss: 0.2184 | Spent: 4.0 secs | LR: 0.000471
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.777| [email protected]: 0.950 | [email protected]: 0.916 | [email protected]: 0.637
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 16600 | Loss: 0.2286 | Spent: 36.2 secs | LR: 0.000454
INFO:tensorflow:Step 16650 | Loss: 0.2194 | Spent: 4.0 secs | LR: 0.000438
INFO:tensorflow:Step 16700 | Loss: 0.2430 | Spent: 4.0 secs | LR: 0.000421
INFO:tensorflow:Step 16750 | Loss: 0.2307 | Spent: 4.0 secs | LR: 0.000405
INFO:tensorflow:Step 16800 | Loss: 0.2530 | Spent: 3.9 secs | LR: 0.000389
INFO:tensorflow:Step 16850 | Loss: 0.2311 | Spent: 3.9 secs | LR: 0.000372
INFO:tensorflow:Step 16900 | Loss: 0.2358 | Spent: 4.0 secs | LR: 0.000356
INFO:tensorflow:Step 16950 | Loss: 0.2254 | Spent: 4.0 secs | LR: 0.000339
INFO:tensorflow:Step 17000 | Loss: 0.2089 | Spent: 4.0 secs | LR: 0.000323
INFO:tensorflow:Step 17050 | Loss: 0.2299 | Spent: 4.0 secs | LR: 0.000307
INFO:tensorflow:Step 17100 | Loss: 0.2464 | Spent: 4.0 secs | LR: 0.000290
INFO:tensorflow:Step 17150 | Loss: 0.2281 | Spent: 4.0 secs | LR: 0.000274
INFO:tensorflow:Step 17200 | Loss: 0.2352 | Spent: 4.0 secs | LR: 0.000257
INFO:tensorflow:Step 17250 | Loss: 0.2465 | Spent: 4.0 secs | LR: 0.000241
INFO:tensorflow:Step 17300 | Loss: 0.2379 | Spent: 4.0 secs | LR: 0.000225
INFO:tensorflow:Step 17350 | Loss: 0.2350 | Spent: 4.0 secs | LR: 0.000208
INFO:tensorflow:Step 17400 | Loss: 0.2219 | Spent: 4.0 secs | LR: 0.000192
INFO:tensorflow:Step 17450 | Loss: 0.2529 | Spent: 4.0 secs | LR: 0.000175
INFO:tensorflow:Step 17500 | Loss: 0.2397 | Spent: 4.0 secs | LR: 0.000159
INFO:tensorflow:Step 17550 | Loss: 0.2410 | Spent: 3.9 secs | LR: 0.000143
INFO:tensorflow:Step 17600 | Loss: 0.2323 | Spent: 4.0 secs | LR: 0.000126
INFO:tensorflow:Step 17650 | Loss: 0.2245 | Spent: 3.9 secs | LR: 0.000110
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.776| [email protected]: 0.950 | [email protected]: 0.916 | [email protected]: 0.636
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 17700 | Loss: 0.2346 | Spent: 35.6 secs | LR: 0.000103
INFO:tensorflow:Step 17750 | Loss: 0.2422 | Spent: 4.0 secs | LR: 0.000111
INFO:tensorflow:Step 17800 | Loss: 0.2553 | Spent: 4.0 secs | LR: 0.000120
INFO:tensorflow:Step 17850 | Loss: 0.2191 | Spent: 4.0 secs | LR: 0.000128
INFO:tensorflow:Step 17900 | Loss: 0.2243 | Spent: 4.0 secs | LR: 0.000136
INFO:tensorflow:Step 17950 | Loss: 0.2452 | Spent: 4.0 secs | LR: 0.000144
INFO:tensorflow:Step 18000 | Loss: 0.2321 | Spent: 4.0 secs | LR: 0.000152
INFO:tensorflow:Step 18050 | Loss: 0.2153 | Spent: 3.9 secs | LR: 0.000161
INFO:tensorflow:Step 18100 | Loss: 0.2204 | Spent: 3.9 secs | LR: 0.000169
INFO:tensorflow:Step 18150 | Loss: 0.2296 | Spent: 4.0 secs | LR: 0.000177
INFO:tensorflow:Step 18200 | Loss: 0.2181 | Spent: 3.9 secs | LR: 0.000185
INFO:tensorflow:Step 18250 | Loss: 0.2310 | Spent: 4.0 secs | LR: 0.000193
INFO:tensorflow:Step 18300 | Loss: 0.2547 | Spent: 4.0 secs | LR: 0.000202
INFO:tensorflow:Step 18350 | Loss: 0.2358 | Spent: 3.9 secs | LR: 0.000210
INFO:tensorflow:Step 18400 | Loss: 0.2179 | Spent: 3.9 secs | LR: 0.000218
INFO:tensorflow:Step 18450 | Loss: 0.2284 | Spent: 3.9 secs | LR: 0.000226
INFO:tensorflow:Step 18500 | Loss: 0.2366 | Spent: 3.9 secs | LR: 0.000235
INFO:tensorflow:Step 18550 | Loss: 0.2366 | Spent: 4.0 secs | LR: 0.000243
INFO:tensorflow:Step 18600 | Loss: 0.2344 | Spent: 3.9 secs | LR: 0.000251
INFO:tensorflow:Step 18650 | Loss: 0.2383 | Spent: 3.9 secs | LR: 0.000259
INFO:tensorflow:Step 18700 | Loss: 0.2271 | Spent: 3.9 secs | LR: 0.000267
INFO:tensorflow:Step 18750 | Loss: 0.2447 | Spent: 4.0 secs | LR: 0.000276
INFO:tensorflow:Step 18800 | Loss: 0.2191 | Spent: 4.0 secs | LR: 0.000284
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.776| [email protected]: 0.950 | [email protected]: 0.915 | [email protected]: 0.635
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 18850 | Loss: 0.2325 | Spent: 35.6 secs | LR: 0.000292
INFO:tensorflow:Step 18900 | Loss: 0.2417 | Spent: 3.9 secs | LR: 0.000300
INFO:tensorflow:Step 18950 | Loss: 0.2299 | Spent: 4.0 secs | LR: 0.000308
INFO:tensorflow:Step 19000 | Loss: 0.2393 | Spent: 3.9 secs | LR: 0.000317
INFO:tensorflow:Step 19050 | Loss: 0.2315 | Spent: 3.9 secs | LR: 0.000325
INFO:tensorflow:Step 19100 | Loss: 0.2240 | Spent: 3.9 secs | LR: 0.000333
INFO:tensorflow:Step 19150 | Loss: 0.2286 | Spent: 3.9 secs | LR: 0.000341
INFO:tensorflow:Step 19200 | Loss: 0.2247 | Spent: 3.9 secs | LR: 0.000349
INFO:tensorflow:Step 19250 | Loss: 0.2474 | Spent: 3.9 secs | LR: 0.000358
INFO:tensorflow:Step 19300 | Loss: 0.2417 | Spent: 3.9 secs | LR: 0.000366
INFO:tensorflow:Step 19350 | Loss: 0.2526 | Spent: 4.0 secs | LR: 0.000374
INFO:tensorflow:Step 19400 | Loss: 0.2410 | Spent: 3.9 secs | LR: 0.000382
INFO:tensorflow:Step 19450 | Loss: 0.2263 | Spent: 3.9 secs | LR: 0.000390
INFO:tensorflow:Step 19500 | Loss: 0.2336 | Spent: 3.9 secs | LR: 0.000399
INFO:tensorflow:Step 19550 | Loss: 0.2281 | Spent: 4.0 secs | LR: 0.000407
INFO:tensorflow:Step 19600 | Loss: 0.2393 | Spent: 3.9 secs | LR: 0.000415
INFO:tensorflow:Step 19650 | Loss: 0.2318 | Spent: 4.0 secs | LR: 0.000423
INFO:tensorflow:Step 19700 | Loss: 0.2327 | Spent: 4.0 secs | LR: 0.000431
INFO:tensorflow:Step 19750 | Loss: 0.2234 | Spent: 3.9 secs | LR: 0.000440
INFO:tensorflow:Step 19800 | Loss: 0.2298 | Spent: 4.0 secs | LR: 0.000448
INFO:tensorflow:Step 19850 | Loss: 0.2434 | Spent: 3.9 secs | LR: 0.000456
INFO:tensorflow:Step 19900 | Loss: 0.2425 | Spent: 3.9 secs | LR: 0.000464
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.774| [email protected]: 0.950 | [email protected]: 0.914 | [email protected]: 0.632
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 19950 | Loss: 0.2356 | Spent: 35.5 secs | LR: 0.000472
INFO:tensorflow:Step 20000 | Loss: 0.2150 | Spent: 3.9 secs | LR: 0.000481
INFO:tensorflow:Step 20050 | Loss: 0.2191 | Spent: 3.9 secs | LR: 0.000489
INFO:tensorflow:Step 20100 | Loss: 0.2513 | Spent: 3.9 secs | LR: 0.000497
INFO:tensorflow:Step 20150 | Loss: 0.2349 | Spent: 3.9 secs | LR: 0.000505
INFO:tensorflow:Step 20200 | Loss: 0.2391 | Spent: 3.9 secs | LR: 0.000513
INFO:tensorflow:Step 20250 | Loss: 0.2327 | Spent: 3.9 secs | LR: 0.000522
INFO:tensorflow:Step 20300 | Loss: 0.2260 | Spent: 3.9 secs | LR: 0.000530
INFO:tensorflow:Step 20350 | Loss: 0.2260 | Spent: 4.0 secs | LR: 0.000538
INFO:tensorflow:Step 20400 | Loss: 0.2563 | Spent: 3.9 secs | LR: 0.000546
INFO:tensorflow:Step 20450 | Loss: 0.2020 | Spent: 3.9 secs | LR: 0.000554
INFO:tensorflow:Step 20500 | Loss: 0.2342 | Spent: 3.9 secs | LR: 0.000563
INFO:tensorflow:Step 20550 | Loss: 0.2271 | Spent: 4.0 secs | LR: 0.000571
INFO:tensorflow:Step 20600 | Loss: 0.2357 | Spent: 3.9 secs | LR: 0.000579
INFO:tensorflow:Step 20650 | Loss: 0.2515 | Spent: 3.9 secs | LR: 0.000587
INFO:tensorflow:Step 20700 | Loss: 0.2332 | Spent: 3.9 secs | LR: 0.000595
INFO:tensorflow:Step 20750 | Loss: 0.2407 | Spent: 3.9 secs | LR: 0.000604
INFO:tensorflow:Step 20800 | Loss: 0.2048 | Spent: 3.9 secs | LR: 0.000612
INFO:tensorflow:Step 20850 | Loss: 0.2408 | Spent: 3.9 secs | LR: 0.000620
INFO:tensorflow:Step 20900 | Loss: 0.2360 | Spent: 3.9 secs | LR: 0.000628
INFO:tensorflow:Step 20950 | Loss: 0.2530 | Spent: 3.9 secs | LR: 0.000636
INFO:tensorflow:Step 21000 | Loss: 0.2322 | Spent: 3.9 secs | LR: 0.000645
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.774| [email protected]: 0.949 | [email protected]: 0.915 | [email protected]: 0.633
INFO:tensorflow:Best MRR: 0.797
Reading ../data/wn18/train.txt
INFO:tensorflow:Step 21050 | Loss: 0.2391 | Spent: 35.3 secs | LR: 0.000653
INFO:tensorflow:Step 21100 | Loss: 0.2313 | Spent: 3.9 secs | LR: 0.000661
INFO:tensorflow:Step 21150 | Loss: 0.2329 | Spent: 3.9 secs | LR: 0.000669
INFO:tensorflow:Step 21200 | Loss: 0.2262 | Spent: 3.9 secs | LR: 0.000677
INFO:tensorflow:Step 21250 | Loss: 0.2294 | Spent: 3.9 secs | LR: 0.000686
INFO:tensorflow:Step 21300 | Loss: 0.2140 | Spent: 3.9 secs | LR: 0.000694
INFO:tensorflow:Step 21350 | Loss: 0.2583 | Spent: 3.9 secs | LR: 0.000702
INFO:tensorflow:Step 21400 | Loss: 0.2466 | Spent: 3.9 secs | LR: 0.000710
INFO:tensorflow:Step 21450 | Loss: 0.2393 | Spent: 3.9 secs | LR: 0.000718
INFO:tensorflow:Step 21500 | Loss: 0.2234 | Spent: 3.9 secs | LR: 0.000727
INFO:tensorflow:Step 21550 | Loss: 0.2398 | Spent: 3.9 secs | LR: 0.000735
INFO:tensorflow:Step 21600 | Loss: 0.2486 | Spent: 3.9 secs | LR: 0.000743
INFO:tensorflow:Step 21650 | Loss: 0.2337 | Spent: 3.9 secs | LR: 0.000751
INFO:tensorflow:Step 21700 | Loss: 0.2246 | Spent: 3.9 secs | LR: 0.000759
INFO:tensorflow:Step 21750 | Loss: 0.2170 | Spent: 4.0 secs | LR: 0.000768
INFO:tensorflow:Step 21800 | Loss: 0.2467 | Spent: 3.9 secs | LR: 0.000776
INFO:tensorflow:Step 21850 | Loss: 0.2467 | Spent: 3.9 secs | LR: 0.000784
INFO:tensorflow:Step 21900 | Loss: 0.2319 | Spent: 4.0 secs | LR: 0.000792
INFO:tensorflow:Step 21950 | Loss: 0.2354 | Spent: 3.9 secs | LR: 0.000800
INFO:tensorflow:Step 22000 | Loss: 0.2435 | Spent: 3.9 secs | LR: 0.000809
INFO:tensorflow:Step 22050 | Loss: 0.2256 | Spent: 3.9 secs | LR: 0.000817
INFO:tensorflow:Step 22100 | Loss: 0.2251 | Spent: 3.9 secs | LR: 0.000825
Reading ../data/wn18/test.txt
INFO:tensorflow:MRR: 0.771| [email protected]: 0.947 | [email protected]: 0.911 | [email protected]: 0.631
INFO:tensorflow:Best MRR: 0.797
10 times not improve the best result, therefore stop training