In [ ]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_matching/joint/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [ ]:
%tensorflow_version 2.x
!pip install transformers
Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.4.0)
Requirement already satisfied: dataclasses; python_version < "3.7" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)
Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.94)
Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)
Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)
Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)
Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)
Requirement already satisfied: tokenizers==0.9.2 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.9.2)
Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)
Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers) (3.12.4)
Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)
Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)
Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)
Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.17.0)
Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers) (50.3.2)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)
Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)
In [ ]:
from transformers import BertTokenizer, TFBertModel
from sklearn.metrics import classification_report

import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pprint
import logging
import time
import math
import json
import csv

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.3.0
WARNING:tensorflow:From <ipython-input-3-1f766395fbf4>:15: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU Enabled: True
In [ ]:
params = {
  'pretrain_path': 'bert-base-chinese',
  'train_path': '../data/train.json',
  'test_path': '../data/dev.json',
  'batch_size': 32,
  'max_len': 128,
  'buffer_size': 34334 + 100000,
  'init_lr': 1e-5,
  'max_lr': 4e-5,
  'n_epochs': 12,
  'clip_norm': 5.,
  'label_smooth': .0,
  'num_patience': 5,
}

tokenizer = BertTokenizer.from_pretrained(params['pretrain_path'],
                                          lowercase = True,
                                          add_special_tokens = True)
In [ ]:
def get_vocab(f_path):
  k2v = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      k2v[line] = i
  return k2v
In [ ]:
def data_gen_cs():
  f_path = '../data/test.csv'
  with open(f_path) as f:
    print('Reading', f_path)
    for i, line in enumerate(csv.reader(f, delimiter=',')):
      if i == 0:
        continue
      text1, text2, label = line
      if len(text1) + len(text2) + 3 > params['max_len']:
        _max_len = (params['max_len'] - 3) // 2
        text1 = text1[:_max_len]
        text2 = text2[:_max_len]
      text1 = list(text1)
      text2 = list(text2)
      text = ['[CLS]'] + text1 + ['[SEP]'] + text2 + ['[SEP]']
      seg = [0] + [0] * len(text1) + [0] + [1] * len(text2) + [1]
      text = tokenizer.convert_tokens_to_ids(text)
      yield ((text, seg), int(label))


def data_gen_js():
  f_path = '../data/dev.json'
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = json.loads(line.rstrip())
      text1, text2, label = line['sentence1'], line['sentence2'], line['label']
      if len(text1) + len(text2) + 3 > params['max_len']:
        _max_len = (params['max_len'] - 3) // 2
        text1 = text1[:_max_len]
        text2 = text2[:_max_len]
      text1 = list(text1)
      text2 = list(text2)
      text = ['[CLS]'] + text1 + ['[SEP]'] + text2 + ['[SEP]']
      seg = [0] + [0] * len(text1) + [0] + [1] * len(text2) + [1]
      text = tokenizer.convert_tokens_to_ids(text)
      yield ((text, seg), int(label))


def joint_data_gen():
  f_path = '../data/train.csv'
  with open(f_path) as f:
    print('Reading', f_path)
    for i, line in enumerate(csv.reader(f, delimiter=',')):
      if i == 0:
        continue
      text1, text2, label = line
      if len(text1) + len(text2) + 3 > params['max_len']:
        _max_len = (params['max_len'] - 3) // 2
        text1 = text1[:_max_len]
        text2 = text2[:_max_len]
      text1 = list(text1)
      text2 = list(text2)
      text = ['[CLS]'] + text1 + ['[SEP]'] + text2 + ['[SEP]']
      seg = [0] + [0] * len(text1) + [0] + [1] * len(text2) + [1]
      text = tokenizer.convert_tokens_to_ids(text)
      yield ((text, seg), int(label))
  f_path = '../data/train.json'
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = json.loads(line.rstrip())
      text1, text2, label = line['sentence1'], line['sentence2'], line['label']
      if len(text1) + len(text2) + 3 > params['max_len']:
        _max_len = (params['max_len'] - 3) // 2
        text1 = text1[:_max_len]
        text2 = text2[:_max_len]
      text1 = list(text1)
      text2 = list(text2)
      text = ['[CLS]'] + text1 + ['[SEP]'] + text2 + ['[SEP]']
      seg = [0] + [0] * len(text1) + [0] + [1] * len(text2) + [1]
      text = tokenizer.convert_tokens_to_ids(text)
      yield ((text, seg), int(label))
In [ ]:
def get_datasets(params):
  _shapes = (([None], [None]), ())
  _types = ((tf.int32, tf.int32), tf.int32)
  _pads = ((0, 0), -1)
  
  ds_train = tf.data.Dataset.from_generator(
    lambda: joint_data_gen(),
    output_shapes = _shapes,
    output_types = _types,)
  ds_train = ds_train.shuffle(params['buffer_size'])
  ds_train = ds_train.padded_batch(params['batch_size'], _shapes, _pads)
  ds_train = ds_train.prefetch(tf.data.experimental.AUTOTUNE)

  ds_test_js = tf.data.Dataset.from_generator(
    lambda: data_gen_js(),
    output_shapes = _shapes,
    output_types = _types,)
  ds_test_js = ds_test_js.padded_batch(params['batch_size'], _shapes, _pads)
  ds_test_js = ds_test_js.prefetch(tf.data.experimental.AUTOTUNE)

  ds_test_cs = tf.data.Dataset.from_generator(
    lambda: data_gen_cs(),
    output_shapes = _shapes,
    output_types = _types,)
  ds_test_cs = ds_test_cs.padded_batch(params['batch_size'], _shapes, _pads)
  ds_test_cs = ds_test_cs.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds_train, ds_test_js, ds_test_cs
In [ ]:
# input stream ids check
(text, seg), _ = next(joint_data_gen())
print(text)
print(seg)
Reading ../data/train.csv
[101, 4500, 2544, 928, 6963, 127, 2399, 8024, 2544, 928, 3766, 3300, 2544, 5108, 6587, 1216, 5543, 102, 125, 511, 100, 100, 1384, 4772, 3341, 2544, 5108, 6587, 102]
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
In [ ]:
class BertFinetune(tf.keras.Model):
  def __init__(self, params):
    super(BertFinetune, self).__init__()
    self.bert = TFBertModel.from_pretrained(params['pretrain_path'],
                                            trainable = True)
    self.drop_1 = tf.keras.layers.Dropout(.1)
    self.fc = tf.keras.layers.Dense(300, tf.nn.swish, name='down_stream/fc')
    self.drop_2 = tf.keras.layers.Dropout(.1)
    self.out = tf.keras.layers.Dense(1, name='down_stream/out')

  def call(self, bert_inputs, training):
    bert_inputs = [tf.cast(inp, tf.int32) for inp in bert_inputs]
    x = self.bert(bert_inputs, training=training)
    x = x[1]
    x = self.drop_1(x, training=training)
    x = self.fc(x)
    x = self.drop_2(x, training=training)
    x = self.out(x)
    x = tf.squeeze(x, 1)
    return x
In [ ]:
model = BertFinetune(params)
model.build([[None, None], [None, None], [None, None]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])
Some layers from the model checkpoint at bert-base-chinese were not used when initializing TFBertModel: ['nsp___cls', 'mlm___cls']
- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
All the layers of TFBertModel were initialized from the model checkpoint at bert-base-chinese.
If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.
[('tf_bert_model/bert/embeddings/word_embeddings/weight:0',
  TensorShape([21128, 768])),
 ('tf_bert_model/bert/embeddings/position_embeddings/embeddings:0',
  TensorShape([512, 768])),
 ('tf_bert_model/bert/embeddings/token_type_embeddings/embeddings:0',
  TensorShape([2, 768])),
 ('tf_bert_model/bert/embeddings/LayerNorm/gamma:0', TensorShape([768])),
 ('tf_bert_model/bert/embeddings/LayerNorm/beta:0', TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._0/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._0/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._1/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._1/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._2/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._2/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._3/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._3/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._4/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._4/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._5/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._5/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._6/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._6/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._7/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._7/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._8/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._8/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._9/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._9/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._10/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._10/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._11/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._11/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/pooler/dense/kernel:0', TensorShape([768, 768])),
 ('tf_bert_model/bert/pooler/dense/bias:0', TensorShape([768])),
 ('down_stream/fc/kernel:0', TensorShape([768, 300])),
 ('down_stream/fc/bias:0', TensorShape([300])),
 ('down_stream/out/kernel:0', TensorShape([300, 1])),
 ('down_stream/out/bias:0', TensorShape([1]))]
In [ ]:
def label_smoothing(label, smooth):
  if smooth > 0.:
    return label * (1 - smooth) + 0.5 * smooth
  else:
    return label
In [12]:
decay_lr = tfa.optimizers.Triangular2CyclicalLearningRate(
  initial_learning_rate = params['init_lr'],
  maximal_learning_rate = params['max_lr'],
  step_size = 2 * params['buffer_size'] // params['batch_size'],)
optim = tf.optimizers.Adam(params['init_lr'])
global_step = 0

best_acc, best_acc1, best_acc2 = .0, .0, .0
count = 0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

for _ in range(params['n_epochs']):
  ds_train, ds_test_js, ds_test_cs = get_datasets(params)

  # TRAINING
  for ((text, seg), labels) in ds_train:
    with tf.GradientTape() as tape:
      logits = model([text, tf.sign(text), seg], training=True)
      labels = tf.cast(labels, tf.float32)
      num_neg = tf.reduce_sum(tf.cast(tf.equal(labels, 0.), tf.float32)).numpy()
      num_pos = tf.reduce_sum(labels).numpy()
      if num_pos == 0.:
        pos_weight = 1.
      else:
        pos_weight = num_neg / num_pos
      loss = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(
        labels = label_smoothing(labels, params['label_smooth']),
        logits = logits,
        pos_weight = pos_weight))
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    optim.apply_gradients(zip(grads, model.trainable_variables))
    
    if global_step % 100 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
        global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION 1
  m = tf.keras.metrics.Accuracy()
  intent_true = []
  intent_pred = []

  for ((text, seg), labels) in ds_test_cs:
    logits = tf.sigmoid(model([text, tf.sign(text), seg], training=False))
    y_pred = tf.cast(tf.math.greater_equal(logits, .5), tf.int32)
    m.update_state(y_true=labels, y_pred=y_pred)
    intent_true += labels.numpy().flatten().tolist()
    intent_pred += y_pred.numpy().flatten().tolist()

  acc_1 = m.result().numpy()
  logger.info('测试集:微众银行智能客服')
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc_1))
  logger.info('\n'+classification_report(y_true = intent_true,
                                         y_pred = intent_pred,
                                         labels = [0, 1],
                                         target_names = ['Not Matched', 'Matched'],
                                         digits = 3))

  # EVALUATION 2
  m = tf.keras.metrics.Accuracy()
  intent_true = []
  intent_pred = []

  for ((text, seg), labels) in ds_test_js:
    logits = tf.sigmoid(model([text, tf.sign(text), seg], training=False))
    y_pred = tf.cast(tf.math.greater_equal(logits, .5), tf.int32)
    m.update_state(y_true=labels, y_pred=y_pred)
    intent_true += labels.numpy().flatten().tolist()
    intent_pred += y_pred.numpy().flatten().tolist()

  acc_2 = m.result().numpy()
  logger.info('测试集:蚂蚁金融语义相似度')
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc_2))
  logger.info('\n'+classification_report(y_true = intent_true,
                                         y_pred = intent_pred,
                                         labels = [0, 1],
                                         target_names = ['Not Matched', 'Matched'],
                                         digits = 3))

  # Define Where To Save Model and Stop Training
  acc = (acc_1 + acc_2) / 2.
  if acc > best_acc:
    best_acc = acc
    best_acc1 = acc_1
    best_acc2 = acc_2
    # you can save model here
    count = 0
  else:
    count += 1
  logger.info("Best | Accuracy 1: {:.3f} | Accuracy 2: {:.3f}".format(best_acc1, best_acc2))

  if count == params['num_patience']:
    print(params['num_patience'], "times not improve the best result, therefore stop training")
    break
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 0 | Loss: 0.7939 | Spent: 26.1 secs | LR: 0.000010
INFO:tensorflow:Step 100 | Loss: 0.6701 | Spent: 49.2 secs | LR: 0.000010
INFO:tensorflow:Step 200 | Loss: 0.4445 | Spent: 48.4 secs | LR: 0.000011
INFO:tensorflow:Step 300 | Loss: 0.4934 | Spent: 48.3 secs | LR: 0.000011
INFO:tensorflow:Step 400 | Loss: 0.7102 | Spent: 48.4 secs | LR: 0.000011
INFO:tensorflow:Step 500 | Loss: 0.8648 | Spent: 47.0 secs | LR: 0.000012
INFO:tensorflow:Step 600 | Loss: 0.5089 | Spent: 47.0 secs | LR: 0.000012
INFO:tensorflow:Step 700 | Loss: 0.4457 | Spent: 48.1 secs | LR: 0.000013
INFO:tensorflow:Step 800 | Loss: 0.4617 | Spent: 49.0 secs | LR: 0.000013
INFO:tensorflow:Step 900 | Loss: 0.3721 | Spent: 49.1 secs | LR: 0.000013
INFO:tensorflow:Step 1000 | Loss: 0.4169 | Spent: 50.7 secs | LR: 0.000014
INFO:tensorflow:Step 1100 | Loss: 0.5750 | Spent: 48.6 secs | LR: 0.000014
INFO:tensorflow:Step 1200 | Loss: 0.6129 | Spent: 48.4 secs | LR: 0.000014
INFO:tensorflow:Step 1300 | Loss: 0.3495 | Spent: 50.1 secs | LR: 0.000015
INFO:tensorflow:Step 1400 | Loss: 0.5017 | Spent: 50.3 secs | LR: 0.000015
INFO:tensorflow:Step 1500 | Loss: 0.4353 | Spent: 49.1 secs | LR: 0.000015
INFO:tensorflow:Step 1600 | Loss: 0.5714 | Spent: 48.3 secs | LR: 0.000016
INFO:tensorflow:Step 1700 | Loss: 0.5312 | Spent: 50.1 secs | LR: 0.000016
INFO:tensorflow:Step 1800 | Loss: 0.3502 | Spent: 49.8 secs | LR: 0.000016
INFO:tensorflow:Step 1900 | Loss: 0.4622 | Spent: 49.6 secs | LR: 0.000017
INFO:tensorflow:Step 2000 | Loss: 1.0058 | Spent: 48.8 secs | LR: 0.000017
INFO:tensorflow:Step 2100 | Loss: 0.3331 | Spent: 48.0 secs | LR: 0.000018
INFO:tensorflow:Step 2200 | Loss: 0.4306 | Spent: 50.8 secs | LR: 0.000018
INFO:tensorflow:Step 2300 | Loss: 0.6984 | Spent: 50.1 secs | LR: 0.000018
INFO:tensorflow:Step 2400 | Loss: 0.4039 | Spent: 48.8 secs | LR: 0.000019
INFO:tensorflow:Step 2500 | Loss: 0.5073 | Spent: 49.3 secs | LR: 0.000019
INFO:tensorflow:Step 2600 | Loss: 0.3653 | Spent: 48.7 secs | LR: 0.000019
INFO:tensorflow:Step 2700 | Loss: 0.2664 | Spent: 50.2 secs | LR: 0.000020
INFO:tensorflow:Step 2800 | Loss: 0.3375 | Spent: 48.1 secs | LR: 0.000020
INFO:tensorflow:Step 2900 | Loss: 0.1935 | Spent: 47.6 secs | LR: 0.000020
INFO:tensorflow:Step 3000 | Loss: 0.3722 | Spent: 48.7 secs | LR: 0.000021
INFO:tensorflow:Step 3100 | Loss: 0.4484 | Spent: 48.1 secs | LR: 0.000021
INFO:tensorflow:Step 3200 | Loss: 0.4172 | Spent: 48.5 secs | LR: 0.000021
INFO:tensorflow:Step 3300 | Loss: 0.2611 | Spent: 48.7 secs | LR: 0.000022
INFO:tensorflow:Step 3400 | Loss: 0.3075 | Spent: 48.6 secs | LR: 0.000022
INFO:tensorflow:Step 3500 | Loss: 0.7058 | Spent: 50.6 secs | LR: 0.000023
INFO:tensorflow:Step 3600 | Loss: 0.2819 | Spent: 52.1 secs | LR: 0.000023
INFO:tensorflow:Step 3700 | Loss: 0.4129 | Spent: 49.3 secs | LR: 0.000023
INFO:tensorflow:Step 3800 | Loss: 0.3364 | Spent: 48.4 secs | LR: 0.000024
INFO:tensorflow:Step 3900 | Loss: 0.4655 | Spent: 49.2 secs | LR: 0.000024
INFO:tensorflow:Step 4000 | Loss: 0.6081 | Spent: 46.9 secs | LR: 0.000024
INFO:tensorflow:Step 4100 | Loss: 0.4207 | Spent: 47.8 secs | LR: 0.000025
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.832
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.838     0.822     0.830      5000
     Matched      0.826     0.841     0.833      5000

    accuracy                          0.832     10000
   macro avg      0.832     0.832     0.832     10000
weighted avg      0.832     0.832     0.832     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.729
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.771     0.864     0.815      2978
     Matched      0.586     0.428     0.494      1338

    accuracy                          0.729      4316
   macro avg      0.678     0.646     0.655      4316
weighted avg      0.713     0.729     0.715      4316

INFO:tensorflow:Best | Accuracy 1: 0.832 | Accuracy 2: 0.729
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 4200 | Loss: 0.3069 | Spent: 141.1 secs | LR: 0.000025
INFO:tensorflow:Step 4300 | Loss: 0.1757 | Spent: 48.7 secs | LR: 0.000025
INFO:tensorflow:Step 4400 | Loss: 0.2419 | Spent: 48.7 secs | LR: 0.000026
INFO:tensorflow:Step 4500 | Loss: 0.3682 | Spent: 50.0 secs | LR: 0.000026
INFO:tensorflow:Step 4600 | Loss: 0.3594 | Spent: 49.5 secs | LR: 0.000026
INFO:tensorflow:Step 4700 | Loss: 0.2605 | Spent: 48.3 secs | LR: 0.000027
INFO:tensorflow:Step 4800 | Loss: 0.2932 | Spent: 49.2 secs | LR: 0.000027
INFO:tensorflow:Step 4900 | Loss: 0.2818 | Spent: 48.4 secs | LR: 0.000028
INFO:tensorflow:Step 5000 | Loss: 0.4844 | Spent: 48.5 secs | LR: 0.000028
INFO:tensorflow:Step 5100 | Loss: 0.3414 | Spent: 48.7 secs | LR: 0.000028
INFO:tensorflow:Step 5200 | Loss: 0.4281 | Spent: 49.3 secs | LR: 0.000029
INFO:tensorflow:Step 5300 | Loss: 0.2954 | Spent: 48.5 secs | LR: 0.000029
INFO:tensorflow:Step 5400 | Loss: 0.1430 | Spent: 49.8 secs | LR: 0.000029
INFO:tensorflow:Step 5500 | Loss: 0.2885 | Spent: 48.0 secs | LR: 0.000030
INFO:tensorflow:Step 5600 | Loss: 0.2596 | Spent: 48.2 secs | LR: 0.000030
INFO:tensorflow:Step 5700 | Loss: 0.3375 | Spent: 46.9 secs | LR: 0.000030
INFO:tensorflow:Step 5800 | Loss: 0.2543 | Spent: 49.9 secs | LR: 0.000031
INFO:tensorflow:Step 5900 | Loss: 0.4220 | Spent: 48.2 secs | LR: 0.000031
INFO:tensorflow:Step 6000 | Loss: 0.3948 | Spent: 49.5 secs | LR: 0.000031
INFO:tensorflow:Step 6100 | Loss: 0.4314 | Spent: 49.0 secs | LR: 0.000032
INFO:tensorflow:Step 6200 | Loss: 0.2747 | Spent: 49.2 secs | LR: 0.000032
INFO:tensorflow:Step 6300 | Loss: 0.7042 | Spent: 49.0 secs | LR: 0.000033
INFO:tensorflow:Step 6400 | Loss: 0.3305 | Spent: 46.8 secs | LR: 0.000033
INFO:tensorflow:Step 6500 | Loss: 0.3573 | Spent: 49.6 secs | LR: 0.000033
INFO:tensorflow:Step 6600 | Loss: 0.2980 | Spent: 49.2 secs | LR: 0.000034
INFO:tensorflow:Step 6700 | Loss: 0.4283 | Spent: 48.2 secs | LR: 0.000034
INFO:tensorflow:Step 6800 | Loss: 0.2269 | Spent: 48.7 secs | LR: 0.000034
INFO:tensorflow:Step 6900 | Loss: 0.2173 | Spent: 47.7 secs | LR: 0.000035
INFO:tensorflow:Step 7000 | Loss: 0.1204 | Spent: 47.0 secs | LR: 0.000035
INFO:tensorflow:Step 7100 | Loss: 0.6526 | Spent: 48.9 secs | LR: 0.000035
INFO:tensorflow:Step 7200 | Loss: 0.2126 | Spent: 49.6 secs | LR: 0.000036
INFO:tensorflow:Step 7300 | Loss: 0.4695 | Spent: 49.6 secs | LR: 0.000036
INFO:tensorflow:Step 7400 | Loss: 0.3025 | Spent: 49.7 secs | LR: 0.000036
INFO:tensorflow:Step 7500 | Loss: 0.5156 | Spent: 50.0 secs | LR: 0.000037
INFO:tensorflow:Step 7600 | Loss: 0.3309 | Spent: 50.4 secs | LR: 0.000037
INFO:tensorflow:Step 7700 | Loss: 0.1965 | Spent: 47.5 secs | LR: 0.000038
INFO:tensorflow:Step 7800 | Loss: 0.3242 | Spent: 50.1 secs | LR: 0.000038
INFO:tensorflow:Step 7900 | Loss: 0.4265 | Spent: 48.3 secs | LR: 0.000038
INFO:tensorflow:Step 8000 | Loss: 0.3137 | Spent: 47.5 secs | LR: 0.000039
INFO:tensorflow:Step 8100 | Loss: 0.3275 | Spent: 48.4 secs | LR: 0.000039
INFO:tensorflow:Step 8200 | Loss: 0.2631 | Spent: 49.2 secs | LR: 0.000039
INFO:tensorflow:Step 8300 | Loss: 0.2880 | Spent: 50.9 secs | LR: 0.000040
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.838
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.827     0.854     0.840      5000
     Matched      0.849     0.822     0.835      5000

    accuracy                          0.838     10000
   macro avg      0.838     0.838     0.838     10000
weighted avg      0.838     0.838     0.838     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.711
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.800     0.775     0.787      2978
     Matched      0.532     0.570     0.550      1338

    accuracy                          0.711      4316
   macro avg      0.666     0.672     0.669      4316
weighted avg      0.717     0.711     0.714      4316

INFO:tensorflow:Best | Accuracy 1: 0.832 | Accuracy 2: 0.729
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 8400 | Loss: 0.4003 | Spent: 135.9 secs | LR: 0.000040
INFO:tensorflow:Step 8500 | Loss: 0.1837 | Spent: 48.3 secs | LR: 0.000040
INFO:tensorflow:Step 8600 | Loss: 0.2185 | Spent: 48.1 secs | LR: 0.000039
INFO:tensorflow:Step 8700 | Loss: 0.2337 | Spent: 50.6 secs | LR: 0.000039
INFO:tensorflow:Step 8800 | Loss: 0.4523 | Spent: 48.1 secs | LR: 0.000039
INFO:tensorflow:Step 8900 | Loss: 0.3097 | Spent: 48.8 secs | LR: 0.000038
INFO:tensorflow:Step 9000 | Loss: 0.1542 | Spent: 48.1 secs | LR: 0.000038
INFO:tensorflow:Step 9100 | Loss: 0.2116 | Spent: 50.4 secs | LR: 0.000037
INFO:tensorflow:Step 9200 | Loss: 0.2328 | Spent: 49.9 secs | LR: 0.000037
INFO:tensorflow:Step 9300 | Loss: 0.3879 | Spent: 49.0 secs | LR: 0.000037
INFO:tensorflow:Step 9400 | Loss: 0.2513 | Spent: 49.1 secs | LR: 0.000036
INFO:tensorflow:Step 9500 | Loss: 0.1754 | Spent: 49.8 secs | LR: 0.000036
INFO:tensorflow:Step 9600 | Loss: 0.2443 | Spent: 48.1 secs | LR: 0.000036
INFO:tensorflow:Step 9700 | Loss: 0.2303 | Spent: 48.9 secs | LR: 0.000035
INFO:tensorflow:Step 9800 | Loss: 0.1616 | Spent: 47.0 secs | LR: 0.000035
INFO:tensorflow:Step 9900 | Loss: 0.2708 | Spent: 49.4 secs | LR: 0.000035
INFO:tensorflow:Step 10000 | Loss: 0.2084 | Spent: 48.0 secs | LR: 0.000034
INFO:tensorflow:Step 10100 | Loss: 0.2926 | Spent: 48.8 secs | LR: 0.000034
INFO:tensorflow:Step 10200 | Loss: 0.4039 | Spent: 48.8 secs | LR: 0.000034
INFO:tensorflow:Step 10300 | Loss: 0.3099 | Spent: 48.0 secs | LR: 0.000033
INFO:tensorflow:Step 10400 | Loss: 0.2702 | Spent: 47.6 secs | LR: 0.000033
INFO:tensorflow:Step 10500 | Loss: 0.2636 | Spent: 49.8 secs | LR: 0.000032
INFO:tensorflow:Step 10600 | Loss: 0.1920 | Spent: 50.0 secs | LR: 0.000032
INFO:tensorflow:Step 10700 | Loss: 0.1300 | Spent: 48.2 secs | LR: 0.000032
INFO:tensorflow:Step 10800 | Loss: 0.2579 | Spent: 48.8 secs | LR: 0.000031
INFO:tensorflow:Step 10900 | Loss: 0.2237 | Spent: 48.3 secs | LR: 0.000031
INFO:tensorflow:Step 11000 | Loss: 0.2614 | Spent: 48.3 secs | LR: 0.000031
INFO:tensorflow:Step 11100 | Loss: 0.5065 | Spent: 48.7 secs | LR: 0.000030
INFO:tensorflow:Step 11200 | Loss: 0.1964 | Spent: 48.3 secs | LR: 0.000030
INFO:tensorflow:Step 11300 | Loss: 0.3983 | Spent: 48.8 secs | LR: 0.000030
INFO:tensorflow:Step 11400 | Loss: 0.1771 | Spent: 47.5 secs | LR: 0.000029
INFO:tensorflow:Step 11500 | Loss: 0.2650 | Spent: 48.7 secs | LR: 0.000029
INFO:tensorflow:Step 11600 | Loss: 0.3391 | Spent: 47.9 secs | LR: 0.000029
INFO:tensorflow:Step 11700 | Loss: 0.2270 | Spent: 49.4 secs | LR: 0.000028
INFO:tensorflow:Step 11800 | Loss: 0.2361 | Spent: 50.0 secs | LR: 0.000028
INFO:tensorflow:Step 11900 | Loss: 0.4915 | Spent: 48.3 secs | LR: 0.000027
INFO:tensorflow:Step 12000 | Loss: 0.3200 | Spent: 48.1 secs | LR: 0.000027
INFO:tensorflow:Step 12100 | Loss: 0.3080 | Spent: 47.8 secs | LR: 0.000027
INFO:tensorflow:Step 12200 | Loss: 0.2805 | Spent: 46.1 secs | LR: 0.000026
INFO:tensorflow:Step 12300 | Loss: 0.3652 | Spent: 50.1 secs | LR: 0.000026
INFO:tensorflow:Step 12400 | Loss: 0.2778 | Spent: 49.2 secs | LR: 0.000026
INFO:tensorflow:Step 12500 | Loss: 0.1754 | Spent: 50.3 secs | LR: 0.000025
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.840
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.806     0.895     0.848      5000
     Matched      0.882     0.785     0.830      5000

    accuracy                          0.840     10000
   macro avg      0.844     0.840     0.839     10000
weighted avg      0.844     0.840     0.839     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.725
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.795     0.811     0.803      2978
     Matched      0.560     0.534     0.546      1338

    accuracy                          0.725      4316
   macro avg      0.677     0.672     0.675      4316
weighted avg      0.722     0.725     0.723      4316

INFO:tensorflow:Best | Accuracy 1: 0.840 | Accuracy 2: 0.725
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 12600 | Loss: 0.1279 | Spent: 134.7 secs | LR: 0.000025
INFO:tensorflow:Step 12700 | Loss: 0.6020 | Spent: 48.2 secs | LR: 0.000025
INFO:tensorflow:Step 12800 | Loss: 0.0343 | Spent: 48.5 secs | LR: 0.000024
INFO:tensorflow:Step 12900 | Loss: 0.1322 | Spent: 48.4 secs | LR: 0.000024
INFO:tensorflow:Step 13000 | Loss: 0.1484 | Spent: 47.9 secs | LR: 0.000024
INFO:tensorflow:Step 13100 | Loss: 0.0666 | Spent: 49.5 secs | LR: 0.000023
INFO:tensorflow:Step 13200 | Loss: 0.3189 | Spent: 47.4 secs | LR: 0.000023
INFO:tensorflow:Step 13300 | Loss: 0.2616 | Spent: 49.1 secs | LR: 0.000022
INFO:tensorflow:Step 13400 | Loss: 0.4163 | Spent: 47.6 secs | LR: 0.000022
INFO:tensorflow:Step 13500 | Loss: 0.0645 | Spent: 48.2 secs | LR: 0.000022
INFO:tensorflow:Step 13600 | Loss: 0.1182 | Spent: 47.9 secs | LR: 0.000021
INFO:tensorflow:Step 13700 | Loss: 0.1517 | Spent: 48.5 secs | LR: 0.000021
INFO:tensorflow:Step 13800 | Loss: 0.2372 | Spent: 49.5 secs | LR: 0.000021
INFO:tensorflow:Step 13900 | Loss: 0.4404 | Spent: 50.4 secs | LR: 0.000020
INFO:tensorflow:Step 14000 | Loss: 0.0910 | Spent: 51.1 secs | LR: 0.000020
INFO:tensorflow:Step 14100 | Loss: 0.2365 | Spent: 47.5 secs | LR: 0.000020
INFO:tensorflow:Step 14200 | Loss: 0.5014 | Spent: 46.9 secs | LR: 0.000019
INFO:tensorflow:Step 14300 | Loss: 0.0449 | Spent: 47.7 secs | LR: 0.000019
INFO:tensorflow:Step 14400 | Loss: 0.2469 | Spent: 48.9 secs | LR: 0.000019
INFO:tensorflow:Step 14500 | Loss: 0.4402 | Spent: 47.5 secs | LR: 0.000018
INFO:tensorflow:Step 14600 | Loss: 0.0993 | Spent: 47.5 secs | LR: 0.000018
INFO:tensorflow:Step 14700 | Loss: 0.0916 | Spent: 49.3 secs | LR: 0.000017
INFO:tensorflow:Step 14800 | Loss: 0.0711 | Spent: 50.4 secs | LR: 0.000017
INFO:tensorflow:Step 14900 | Loss: 0.0712 | Spent: 47.8 secs | LR: 0.000017
INFO:tensorflow:Step 15000 | Loss: 0.1871 | Spent: 47.1 secs | LR: 0.000016
INFO:tensorflow:Step 15100 | Loss: 0.0626 | Spent: 48.6 secs | LR: 0.000016
INFO:tensorflow:Step 15200 | Loss: 0.1546 | Spent: 47.9 secs | LR: 0.000016
INFO:tensorflow:Step 15300 | Loss: 0.0801 | Spent: 47.4 secs | LR: 0.000015
INFO:tensorflow:Step 15400 | Loss: 0.4010 | Spent: 49.7 secs | LR: 0.000015
INFO:tensorflow:Step 15500 | Loss: 0.1180 | Spent: 48.5 secs | LR: 0.000015
INFO:tensorflow:Step 15600 | Loss: 0.0868 | Spent: 48.8 secs | LR: 0.000014
INFO:tensorflow:Step 15700 | Loss: 0.1816 | Spent: 46.5 secs | LR: 0.000014
INFO:tensorflow:Step 15800 | Loss: 0.1614 | Spent: 48.9 secs | LR: 0.000014
INFO:tensorflow:Step 15900 | Loss: 0.3362 | Spent: 48.2 secs | LR: 0.000013
INFO:tensorflow:Step 16000 | Loss: 0.2098 | Spent: 49.9 secs | LR: 0.000013
INFO:tensorflow:Step 16100 | Loss: 0.0300 | Spent: 50.9 secs | LR: 0.000012
INFO:tensorflow:Step 16200 | Loss: 0.0727 | Spent: 50.7 secs | LR: 0.000012
INFO:tensorflow:Step 16300 | Loss: 0.1058 | Spent: 49.6 secs | LR: 0.000012
INFO:tensorflow:Step 16400 | Loss: 0.4964 | Spent: 48.7 secs | LR: 0.000011
INFO:tensorflow:Step 16500 | Loss: 0.0400 | Spent: 47.9 secs | LR: 0.000011
INFO:tensorflow:Step 16600 | Loss: 0.3161 | Spent: 49.4 secs | LR: 0.000011
INFO:tensorflow:Step 16700 | Loss: 0.2731 | Spent: 48.2 secs | LR: 0.000010
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.845
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.818     0.889     0.852      5000
     Matched      0.879     0.802     0.838      5000

    accuracy                          0.845     10000
   macro avg      0.848     0.845     0.845     10000
weighted avg      0.848     0.845     0.845     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.737
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.795     0.834     0.814      2978
     Matched      0.586     0.522     0.552      1338

    accuracy                          0.737      4316
   macro avg      0.691     0.678     0.683      4316
weighted avg      0.730     0.737     0.733      4316

INFO:tensorflow:Best | Accuracy 1: 0.845 | Accuracy 2: 0.737
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 16800 | Loss: 0.0773 | Spent: 136.1 secs | LR: 0.000010
INFO:tensorflow:Step 16900 | Loss: 0.0411 | Spent: 50.7 secs | LR: 0.000010
INFO:tensorflow:Step 17000 | Loss: 0.1088 | Spent: 47.6 secs | LR: 0.000010
INFO:tensorflow:Step 17100 | Loss: 0.0411 | Spent: 49.1 secs | LR: 0.000011
INFO:tensorflow:Step 17200 | Loss: 0.0636 | Spent: 47.0 secs | LR: 0.000011
INFO:tensorflow:Step 17300 | Loss: 0.0910 | Spent: 47.7 secs | LR: 0.000011
INFO:tensorflow:Step 17400 | Loss: 0.3244 | Spent: 49.0 secs | LR: 0.000011
INFO:tensorflow:Step 17500 | Loss: 0.2961 | Spent: 47.6 secs | LR: 0.000011
INFO:tensorflow:Step 17600 | Loss: 0.0758 | Spent: 48.7 secs | LR: 0.000011
INFO:tensorflow:Step 17700 | Loss: 0.0951 | Spent: 49.3 secs | LR: 0.000012
INFO:tensorflow:Step 17800 | Loss: 0.0462 | Spent: 49.0 secs | LR: 0.000012
INFO:tensorflow:Step 17900 | Loss: 0.0858 | Spent: 48.4 secs | LR: 0.000012
INFO:tensorflow:Step 18000 | Loss: 0.2328 | Spent: 47.4 secs | LR: 0.000012
INFO:tensorflow:Step 18100 | Loss: 0.0675 | Spent: 50.9 secs | LR: 0.000012
INFO:tensorflow:Step 18200 | Loss: 0.0260 | Spent: 47.0 secs | LR: 0.000013
INFO:tensorflow:Step 18300 | Loss: 0.1087 | Spent: 49.2 secs | LR: 0.000013
INFO:tensorflow:Step 18400 | Loss: 0.0701 | Spent: 47.4 secs | LR: 0.000013
INFO:tensorflow:Step 18500 | Loss: 0.0349 | Spent: 49.0 secs | LR: 0.000013
INFO:tensorflow:Step 18600 | Loss: 0.0521 | Spent: 49.2 secs | LR: 0.000013
INFO:tensorflow:Step 18700 | Loss: 0.0980 | Spent: 48.0 secs | LR: 0.000013
INFO:tensorflow:Step 18800 | Loss: 0.0413 | Spent: 49.1 secs | LR: 0.000014
INFO:tensorflow:Step 18900 | Loss: 0.1280 | Spent: 49.3 secs | LR: 0.000014
INFO:tensorflow:Step 19000 | Loss: 0.0737 | Spent: 47.8 secs | LR: 0.000014
INFO:tensorflow:Step 19100 | Loss: 0.0976 | Spent: 50.2 secs | LR: 0.000014
INFO:tensorflow:Step 19200 | Loss: 0.0204 | Spent: 49.9 secs | LR: 0.000014
INFO:tensorflow:Step 19300 | Loss: 0.3530 | Spent: 49.3 secs | LR: 0.000014
INFO:tensorflow:Step 19400 | Loss: 0.0707 | Spent: 49.3 secs | LR: 0.000015
INFO:tensorflow:Step 19500 | Loss: 0.0598 | Spent: 49.2 secs | LR: 0.000015
INFO:tensorflow:Step 19600 | Loss: 0.2094 | Spent: 48.2 secs | LR: 0.000015
INFO:tensorflow:Step 19700 | Loss: 0.0740 | Spent: 49.1 secs | LR: 0.000015
INFO:tensorflow:Step 19800 | Loss: 0.0822 | Spent: 48.6 secs | LR: 0.000015
INFO:tensorflow:Step 19900 | Loss: 0.0791 | Spent: 49.2 secs | LR: 0.000016
INFO:tensorflow:Step 20000 | Loss: 0.1075 | Spent: 49.6 secs | LR: 0.000016
INFO:tensorflow:Step 20100 | Loss: 0.1417 | Spent: 49.0 secs | LR: 0.000016
INFO:tensorflow:Step 20200 | Loss: 0.1078 | Spent: 49.0 secs | LR: 0.000016
INFO:tensorflow:Step 20300 | Loss: 0.0884 | Spent: 51.7 secs | LR: 0.000016
INFO:tensorflow:Step 20400 | Loss: 0.0571 | Spent: 50.6 secs | LR: 0.000016
INFO:tensorflow:Step 20500 | Loss: 0.2262 | Spent: 50.7 secs | LR: 0.000017
INFO:tensorflow:Step 20600 | Loss: 0.0098 | Spent: 50.6 secs | LR: 0.000017
INFO:tensorflow:Step 20700 | Loss: 0.1404 | Spent: 49.2 secs | LR: 0.000017
INFO:tensorflow:Step 20800 | Loss: 0.1979 | Spent: 48.3 secs | LR: 0.000017
INFO:tensorflow:Step 20900 | Loss: 0.0287 | Spent: 48.8 secs | LR: 0.000017
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.844
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.821     0.879     0.849      5000
     Matched      0.870     0.808     0.838      5000

    accuracy                          0.844     10000
   macro avg      0.845     0.844     0.843     10000
weighted avg      0.845     0.844     0.843     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.740
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.805     0.822     0.814      2978
     Matched      0.585     0.558     0.571      1338

    accuracy                          0.740      4316
   macro avg      0.695     0.690     0.692      4316
weighted avg      0.737     0.740     0.738      4316

INFO:tensorflow:Best | Accuracy 1: 0.844 | Accuracy 2: 0.740
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 21000 | Loss: 0.0166 | Spent: 140.8 secs | LR: 0.000018
INFO:tensorflow:Step 21100 | Loss: 0.1058 | Spent: 51.9 secs | LR: 0.000018
INFO:tensorflow:Step 21200 | Loss: 0.1033 | Spent: 49.9 secs | LR: 0.000018
INFO:tensorflow:Step 21300 | Loss: 0.1643 | Spent: 49.2 secs | LR: 0.000018
INFO:tensorflow:Step 21400 | Loss: 0.1922 | Spent: 48.9 secs | LR: 0.000018
INFO:tensorflow:Step 21500 | Loss: 0.2153 | Spent: 49.3 secs | LR: 0.000018
INFO:tensorflow:Step 21600 | Loss: 0.0993 | Spent: 48.8 secs | LR: 0.000019
INFO:tensorflow:Step 21700 | Loss: 0.4159 | Spent: 48.9 secs | LR: 0.000019
INFO:tensorflow:Step 21800 | Loss: 0.0217 | Spent: 48.0 secs | LR: 0.000019
INFO:tensorflow:Step 21900 | Loss: 0.1145 | Spent: 49.7 secs | LR: 0.000019
INFO:tensorflow:Step 22000 | Loss: 0.0711 | Spent: 50.8 secs | LR: 0.000019
INFO:tensorflow:Step 22100 | Loss: 0.1118 | Spent: 48.1 secs | LR: 0.000019
INFO:tensorflow:Step 22200 | Loss: 0.0617 | Spent: 49.8 secs | LR: 0.000020
INFO:tensorflow:Step 22300 | Loss: 0.0298 | Spent: 50.9 secs | LR: 0.000020
INFO:tensorflow:Step 22400 | Loss: 0.0080 | Spent: 49.6 secs | LR: 0.000020
INFO:tensorflow:Step 22500 | Loss: 0.0151 | Spent: 49.2 secs | LR: 0.000020
INFO:tensorflow:Step 22600 | Loss: 0.0656 | Spent: 49.0 secs | LR: 0.000020
INFO:tensorflow:Step 22700 | Loss: 0.1939 | Spent: 50.0 secs | LR: 0.000021
INFO:tensorflow:Step 22800 | Loss: 0.1671 | Spent: 51.3 secs | LR: 0.000021
INFO:tensorflow:Step 22900 | Loss: 0.0265 | Spent: 48.8 secs | LR: 0.000021
INFO:tensorflow:Step 23000 | Loss: 0.0590 | Spent: 48.9 secs | LR: 0.000021
INFO:tensorflow:Step 23100 | Loss: 0.2052 | Spent: 49.1 secs | LR: 0.000021
INFO:tensorflow:Step 23200 | Loss: 0.1298 | Spent: 49.7 secs | LR: 0.000021
INFO:tensorflow:Step 23300 | Loss: 0.0591 | Spent: 48.5 secs | LR: 0.000022
INFO:tensorflow:Step 23400 | Loss: 0.0939 | Spent: 49.3 secs | LR: 0.000022
INFO:tensorflow:Step 23500 | Loss: 0.1421 | Spent: 52.7 secs | LR: 0.000022
INFO:tensorflow:Step 23600 | Loss: 0.1144 | Spent: 49.6 secs | LR: 0.000022
INFO:tensorflow:Step 23700 | Loss: 0.0847 | Spent: 48.9 secs | LR: 0.000022
INFO:tensorflow:Step 23800 | Loss: 0.2067 | Spent: 49.2 secs | LR: 0.000023
INFO:tensorflow:Step 23900 | Loss: 0.3290 | Spent: 49.0 secs | LR: 0.000023
INFO:tensorflow:Step 24000 | Loss: 0.2680 | Spent: 49.8 secs | LR: 0.000023
INFO:tensorflow:Step 24100 | Loss: 0.1183 | Spent: 47.7 secs | LR: 0.000023
INFO:tensorflow:Step 24200 | Loss: 0.1180 | Spent: 48.9 secs | LR: 0.000023
INFO:tensorflow:Step 24300 | Loss: 0.1241 | Spent: 50.7 secs | LR: 0.000023
INFO:tensorflow:Step 24400 | Loss: 0.1371 | Spent: 49.2 secs | LR: 0.000024
INFO:tensorflow:Step 24500 | Loss: 0.0402 | Spent: 48.6 secs | LR: 0.000024
INFO:tensorflow:Step 24600 | Loss: 0.2175 | Spent: 48.5 secs | LR: 0.000024
INFO:tensorflow:Step 24700 | Loss: 0.1798 | Spent: 50.2 secs | LR: 0.000024
INFO:tensorflow:Step 24800 | Loss: 0.0661 | Spent: 50.1 secs | LR: 0.000024
INFO:tensorflow:Step 24900 | Loss: 0.1223 | Spent: 49.0 secs | LR: 0.000024
INFO:tensorflow:Step 25000 | Loss: 0.2119 | Spent: 49.1 secs | LR: 0.000025
INFO:tensorflow:Step 25100 | Loss: 0.0598 | Spent: 48.3 secs | LR: 0.000025
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.837
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.797     0.905     0.848      5000
     Matched      0.890     0.770     0.826      5000

    accuracy                          0.838     10000
   macro avg      0.844     0.838     0.837     10000
weighted avg      0.844     0.838     0.837     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.737
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.784     0.855     0.818      2978
     Matched      0.595     0.476     0.529      1338

    accuracy                          0.737      4316
   macro avg      0.690     0.665     0.673      4316
weighted avg      0.726     0.737     0.728      4316

INFO:tensorflow:Best | Accuracy 1: 0.844 | Accuracy 2: 0.740
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 25200 | Loss: 0.0295 | Spent: 139.0 secs | LR: 0.000025
INFO:tensorflow:Step 25300 | Loss: 0.1723 | Spent: 48.5 secs | LR: 0.000025
INFO:tensorflow:Step 25400 | Loss: 0.1681 | Spent: 49.4 secs | LR: 0.000025
INFO:tensorflow:Step 25500 | Loss: 0.0167 | Spent: 49.3 secs | LR: 0.000024
INFO:tensorflow:Step 25600 | Loss: 0.0883 | Spent: 49.2 secs | LR: 0.000024
INFO:tensorflow:Step 25700 | Loss: 0.1441 | Spent: 49.8 secs | LR: 0.000024
INFO:tensorflow:Step 25800 | Loss: 0.1705 | Spent: 48.5 secs | LR: 0.000024
INFO:tensorflow:Step 25900 | Loss: 0.0313 | Spent: 52.2 secs | LR: 0.000024
INFO:tensorflow:Step 26000 | Loss: 0.0662 | Spent: 49.2 secs | LR: 0.000024
INFO:tensorflow:Step 26100 | Loss: 0.0594 | Spent: 50.2 secs | LR: 0.000023
INFO:tensorflow:Step 26200 | Loss: 0.0533 | Spent: 49.3 secs | LR: 0.000023
INFO:tensorflow:Step 26300 | Loss: 0.1504 | Spent: 48.7 secs | LR: 0.000023
INFO:tensorflow:Step 26400 | Loss: 0.0809 | Spent: 47.1 secs | LR: 0.000023
INFO:tensorflow:Step 26500 | Loss: 0.2219 | Spent: 47.2 secs | LR: 0.000023
INFO:tensorflow:Step 26600 | Loss: 0.0267 | Spent: 48.1 secs | LR: 0.000022
INFO:tensorflow:Step 26700 | Loss: 0.1085 | Spent: 50.2 secs | LR: 0.000022
INFO:tensorflow:Step 26800 | Loss: 0.0230 | Spent: 49.2 secs | LR: 0.000022
INFO:tensorflow:Step 26900 | Loss: 0.1853 | Spent: 50.1 secs | LR: 0.000022
INFO:tensorflow:Step 27000 | Loss: 0.1409 | Spent: 49.7 secs | LR: 0.000022
INFO:tensorflow:Step 27100 | Loss: 0.0109 | Spent: 50.8 secs | LR: 0.000022
INFO:tensorflow:Step 27200 | Loss: 0.0858 | Spent: 50.5 secs | LR: 0.000021
INFO:tensorflow:Step 27300 | Loss: 0.1745 | Spent: 48.4 secs | LR: 0.000021
INFO:tensorflow:Step 27400 | Loss: 0.0621 | Spent: 50.0 secs | LR: 0.000021
INFO:tensorflow:Step 27500 | Loss: 0.3353 | Spent: 50.5 secs | LR: 0.000021
INFO:tensorflow:Step 27600 | Loss: 0.0912 | Spent: 48.5 secs | LR: 0.000021
INFO:tensorflow:Step 27700 | Loss: 0.1200 | Spent: 48.4 secs | LR: 0.000021
INFO:tensorflow:Step 27800 | Loss: 0.0277 | Spent: 49.7 secs | LR: 0.000020
INFO:tensorflow:Step 27900 | Loss: 0.2217 | Spent: 49.4 secs | LR: 0.000020
INFO:tensorflow:Step 28000 | Loss: 0.0308 | Spent: 48.8 secs | LR: 0.000020
INFO:tensorflow:Step 28100 | Loss: 0.0824 | Spent: 48.9 secs | LR: 0.000020
INFO:tensorflow:Step 28200 | Loss: 0.1751 | Spent: 48.8 secs | LR: 0.000020
INFO:tensorflow:Step 28300 | Loss: 0.0791 | Spent: 48.0 secs | LR: 0.000019
INFO:tensorflow:Step 28400 | Loss: 0.0386 | Spent: 48.3 secs | LR: 0.000019
INFO:tensorflow:Step 28500 | Loss: 0.0452 | Spent: 50.5 secs | LR: 0.000019
INFO:tensorflow:Step 28600 | Loss: 0.0405 | Spent: 47.3 secs | LR: 0.000019
INFO:tensorflow:Step 28700 | Loss: 0.2410 | Spent: 48.1 secs | LR: 0.000019
INFO:tensorflow:Step 28800 | Loss: 0.1476 | Spent: 49.0 secs | LR: 0.000019
INFO:tensorflow:Step 28900 | Loss: 0.0359 | Spent: 47.6 secs | LR: 0.000018
INFO:tensorflow:Step 29000 | Loss: 0.1070 | Spent: 49.8 secs | LR: 0.000018
INFO:tensorflow:Step 29100 | Loss: 0.0771 | Spent: 49.1 secs | LR: 0.000018
INFO:tensorflow:Step 29200 | Loss: 0.0518 | Spent: 47.7 secs | LR: 0.000018
INFO:tensorflow:Step 29300 | Loss: 0.0376 | Spent: 47.0 secs | LR: 0.000018
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.835
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.798     0.896     0.844      5000
     Matched      0.881     0.774     0.824      5000

    accuracy                          0.835     10000
   macro avg      0.840     0.835     0.834     10000
weighted avg      0.840     0.835     0.834     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.729
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.801     0.808     0.804      2978
     Matched      0.564     0.553     0.558      1338

    accuracy                          0.729      4316
   macro avg      0.682     0.680     0.681      4316
weighted avg      0.727     0.729     0.728      4316

INFO:tensorflow:Best | Accuracy 1: 0.844 | Accuracy 2: 0.740
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 29400 | Loss: 0.0573 | Spent: 136.9 secs | LR: 0.000017
INFO:tensorflow:Step 29500 | Loss: 0.0171 | Spent: 48.8 secs | LR: 0.000017
INFO:tensorflow:Step 29600 | Loss: 0.0117 | Spent: 47.1 secs | LR: 0.000017
INFO:tensorflow:Step 29700 | Loss: 0.0562 | Spent: 49.2 secs | LR: 0.000017
INFO:tensorflow:Step 29800 | Loss: 0.0266 | Spent: 49.9 secs | LR: 0.000017
INFO:tensorflow:Step 29900 | Loss: 0.1288 | Spent: 50.0 secs | LR: 0.000017
INFO:tensorflow:Step 30000 | Loss: 0.0078 | Spent: 48.4 secs | LR: 0.000016
INFO:tensorflow:Step 30100 | Loss: 0.0047 | Spent: 49.1 secs | LR: 0.000016
INFO:tensorflow:Step 30200 | Loss: 0.2837 | Spent: 49.5 secs | LR: 0.000016
INFO:tensorflow:Step 30300 | Loss: 0.0024 | Spent: 48.2 secs | LR: 0.000016
INFO:tensorflow:Step 30400 | Loss: 0.0264 | Spent: 51.2 secs | LR: 0.000016
INFO:tensorflow:Step 30500 | Loss: 0.0353 | Spent: 50.1 secs | LR: 0.000016
INFO:tensorflow:Step 30600 | Loss: 0.1308 | Spent: 48.6 secs | LR: 0.000015
INFO:tensorflow:Step 30700 | Loss: 0.1577 | Spent: 47.8 secs | LR: 0.000015
INFO:tensorflow:Step 30800 | Loss: 0.0190 | Spent: 49.1 secs | LR: 0.000015
INFO:tensorflow:Step 30900 | Loss: 0.0695 | Spent: 49.3 secs | LR: 0.000015
INFO:tensorflow:Step 31000 | Loss: 0.0516 | Spent: 48.2 secs | LR: 0.000015
INFO:tensorflow:Step 31100 | Loss: 0.0249 | Spent: 47.2 secs | LR: 0.000014
INFO:tensorflow:Step 31200 | Loss: 0.0936 | Spent: 48.7 secs | LR: 0.000014
INFO:tensorflow:Step 31300 | Loss: 0.0240 | Spent: 49.1 secs | LR: 0.000014
INFO:tensorflow:Step 31400 | Loss: 0.0297 | Spent: 47.8 secs | LR: 0.000014
INFO:tensorflow:Step 31500 | Loss: 0.0043 | Spent: 47.3 secs | LR: 0.000014
INFO:tensorflow:Step 31600 | Loss: 0.1027 | Spent: 48.0 secs | LR: 0.000014
INFO:tensorflow:Step 31700 | Loss: 0.1383 | Spent: 48.0 secs | LR: 0.000013
INFO:tensorflow:Step 31800 | Loss: 0.0033 | Spent: 48.7 secs | LR: 0.000013
INFO:tensorflow:Step 31900 | Loss: 0.0089 | Spent: 48.1 secs | LR: 0.000013
INFO:tensorflow:Step 32000 | Loss: 0.0192 | Spent: 50.0 secs | LR: 0.000013
INFO:tensorflow:Step 32100 | Loss: 0.0274 | Spent: 48.0 secs | LR: 0.000013
INFO:tensorflow:Step 32200 | Loss: 0.0462 | Spent: 48.0 secs | LR: 0.000012
INFO:tensorflow:Step 32300 | Loss: 0.0882 | Spent: 46.5 secs | LR: 0.000012
INFO:tensorflow:Step 32400 | Loss: 0.1993 | Spent: 49.9 secs | LR: 0.000012
INFO:tensorflow:Step 32500 | Loss: 0.1767 | Spent: 48.3 secs | LR: 0.000012
INFO:tensorflow:Step 32600 | Loss: 0.0575 | Spent: 50.1 secs | LR: 0.000012
INFO:tensorflow:Step 32700 | Loss: 0.0161 | Spent: 47.4 secs | LR: 0.000012
INFO:tensorflow:Step 32800 | Loss: 0.1007 | Spent: 48.9 secs | LR: 0.000011
INFO:tensorflow:Step 32900 | Loss: 0.0926 | Spent: 50.3 secs | LR: 0.000011
INFO:tensorflow:Step 33000 | Loss: 0.0073 | Spent: 47.0 secs | LR: 0.000011
INFO:tensorflow:Step 33100 | Loss: 0.0939 | Spent: 48.7 secs | LR: 0.000011
INFO:tensorflow:Step 33200 | Loss: 0.0528 | Spent: 49.5 secs | LR: 0.000011
INFO:tensorflow:Step 33300 | Loss: 0.0829 | Spent: 48.1 secs | LR: 0.000011
INFO:tensorflow:Step 33400 | Loss: 0.1292 | Spent: 51.2 secs | LR: 0.000010
INFO:tensorflow:Step 33500 | Loss: 0.0428 | Spent: 47.1 secs | LR: 0.000010
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.842
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.815     0.886     0.849      5000
     Matched      0.875     0.798     0.835      5000

    accuracy                          0.842     10000
   macro avg      0.845     0.842     0.842     10000
weighted avg      0.845     0.842     0.842     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.730
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.788     0.832     0.810      2978
     Matched      0.574     0.503     0.536      1338

    accuracy                          0.730      4316
   macro avg      0.681     0.668     0.673      4316
weighted avg      0.722     0.730     0.725      4316

INFO:tensorflow:Best | Accuracy 1: 0.844 | Accuracy 2: 0.740
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 33600 | Loss: 0.0126 | Spent: 136.8 secs | LR: 0.000010
INFO:tensorflow:Step 33700 | Loss: 0.0264 | Spent: 49.5 secs | LR: 0.000010
INFO:tensorflow:Step 33800 | Loss: 0.0568 | Spent: 49.7 secs | LR: 0.000010
INFO:tensorflow:Step 33900 | Loss: 0.5045 | Spent: 47.4 secs | LR: 0.000010
INFO:tensorflow:Step 34000 | Loss: 0.0043 | Spent: 48.9 secs | LR: 0.000010
INFO:tensorflow:Step 34100 | Loss: 0.0003 | Spent: 47.3 secs | LR: 0.000010
INFO:tensorflow:Step 34200 | Loss: 0.0013 | Spent: 48.6 secs | LR: 0.000011
INFO:tensorflow:Step 34300 | Loss: 0.0498 | Spent: 48.6 secs | LR: 0.000011
INFO:tensorflow:Step 34400 | Loss: 0.0042 | Spent: 50.3 secs | LR: 0.000011
INFO:tensorflow:Step 34500 | Loss: 0.0014 | Spent: 48.2 secs | LR: 0.000011
INFO:tensorflow:Step 34600 | Loss: 0.0844 | Spent: 49.4 secs | LR: 0.000011
INFO:tensorflow:Step 34700 | Loss: 0.0179 | Spent: 49.9 secs | LR: 0.000011
INFO:tensorflow:Step 34800 | Loss: 0.0017 | Spent: 49.7 secs | LR: 0.000011
INFO:tensorflow:Step 34900 | Loss: 0.0030 | Spent: 49.0 secs | LR: 0.000011
INFO:tensorflow:Step 35000 | Loss: 0.0030 | Spent: 47.6 secs | LR: 0.000011
INFO:tensorflow:Step 35100 | Loss: 0.0027 | Spent: 48.7 secs | LR: 0.000011
INFO:tensorflow:Step 35200 | Loss: 0.0064 | Spent: 48.0 secs | LR: 0.000011
INFO:tensorflow:Step 35300 | Loss: 0.0033 | Spent: 48.8 secs | LR: 0.000012
INFO:tensorflow:Step 35400 | Loss: 0.1118 | Spent: 48.2 secs | LR: 0.000012
INFO:tensorflow:Step 35500 | Loss: 0.0366 | Spent: 48.0 secs | LR: 0.000012
INFO:tensorflow:Step 35600 | Loss: 0.0036 | Spent: 49.0 secs | LR: 0.000012
INFO:tensorflow:Step 35700 | Loss: 0.0014 | Spent: 47.6 secs | LR: 0.000012
INFO:tensorflow:Step 35800 | Loss: 0.0037 | Spent: 47.8 secs | LR: 0.000012
INFO:tensorflow:Step 35900 | Loss: 0.0028 | Spent: 49.5 secs | LR: 0.000012
INFO:tensorflow:Step 36000 | Loss: 0.0312 | Spent: 48.7 secs | LR: 0.000012
INFO:tensorflow:Step 36100 | Loss: 0.0024 | Spent: 47.7 secs | LR: 0.000012
INFO:tensorflow:Step 36200 | Loss: 0.0036 | Spent: 48.2 secs | LR: 0.000012
INFO:tensorflow:Step 36300 | Loss: 0.0665 | Spent: 47.1 secs | LR: 0.000012
INFO:tensorflow:Step 36400 | Loss: 0.0005 | Spent: 48.7 secs | LR: 0.000013
INFO:tensorflow:Step 36500 | Loss: 0.1741 | Spent: 48.5 secs | LR: 0.000013
INFO:tensorflow:Step 36600 | Loss: 0.0674 | Spent: 48.3 secs | LR: 0.000013
INFO:tensorflow:Step 36700 | Loss: 0.0125 | Spent: 49.0 secs | LR: 0.000013
INFO:tensorflow:Step 36800 | Loss: 0.0288 | Spent: 49.3 secs | LR: 0.000013
INFO:tensorflow:Step 36900 | Loss: 0.0142 | Spent: 47.5 secs | LR: 0.000013
INFO:tensorflow:Step 37000 | Loss: 0.0198 | Spent: 47.5 secs | LR: 0.000013
INFO:tensorflow:Step 37100 | Loss: 0.0101 | Spent: 48.1 secs | LR: 0.000013
INFO:tensorflow:Step 37200 | Loss: 0.0415 | Spent: 49.1 secs | LR: 0.000013
INFO:tensorflow:Step 37300 | Loss: 0.0240 | Spent: 48.8 secs | LR: 0.000013
INFO:tensorflow:Step 37400 | Loss: 0.0071 | Spent: 50.5 secs | LR: 0.000013
INFO:tensorflow:Step 37500 | Loss: 0.0303 | Spent: 50.6 secs | LR: 0.000014
INFO:tensorflow:Step 37600 | Loss: 0.0215 | Spent: 49.6 secs | LR: 0.000014
INFO:tensorflow:Step 37700 | Loss: 0.0514 | Spent: 49.6 secs | LR: 0.000014
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.838
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.799     0.903     0.848      5000
     Matched      0.889     0.773     0.827      5000

    accuracy                          0.838     10000
   macro avg      0.844     0.838     0.837     10000
weighted avg      0.844     0.838     0.837     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.731
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.786     0.838     0.811      2978
     Matched      0.577     0.493     0.532      1338

    accuracy                          0.731      4316
   macro avg      0.682     0.666     0.672      4316
weighted avg      0.722     0.731     0.725      4316

INFO:tensorflow:Best | Accuracy 1: 0.844 | Accuracy 2: 0.740
Reading ../data/train.csv
Reading ../data/train.json
INFO:tensorflow:Step 37800 | Loss: 0.0093 | Spent: 135.9 secs | LR: 0.000014
INFO:tensorflow:Step 37900 | Loss: 0.0113 | Spent: 47.8 secs | LR: 0.000014
INFO:tensorflow:Step 38000 | Loss: 0.1536 | Spent: 47.8 secs | LR: 0.000014
INFO:tensorflow:Step 38100 | Loss: 0.1189 | Spent: 48.2 secs | LR: 0.000014
INFO:tensorflow:Step 38200 | Loss: 0.0004 | Spent: 48.7 secs | LR: 0.000014
INFO:tensorflow:Step 38300 | Loss: 0.0031 | Spent: 48.6 secs | LR: 0.000014
INFO:tensorflow:Step 38400 | Loss: 0.0139 | Spent: 49.1 secs | LR: 0.000014
INFO:tensorflow:Step 38500 | Loss: 0.0120 | Spent: 47.7 secs | LR: 0.000014
INFO:tensorflow:Step 38600 | Loss: 0.0010 | Spent: 48.1 secs | LR: 0.000014
INFO:tensorflow:Step 38700 | Loss: 0.0091 | Spent: 46.9 secs | LR: 0.000015
INFO:tensorflow:Step 38800 | Loss: 0.0009 | Spent: 46.3 secs | LR: 0.000015
INFO:tensorflow:Step 38900 | Loss: 0.1179 | Spent: 45.2 secs | LR: 0.000015
INFO:tensorflow:Step 39000 | Loss: 0.0017 | Spent: 48.2 secs | LR: 0.000015
INFO:tensorflow:Step 39100 | Loss: 0.0995 | Spent: 50.5 secs | LR: 0.000015
INFO:tensorflow:Step 39200 | Loss: 0.1138 | Spent: 50.3 secs | LR: 0.000015
INFO:tensorflow:Step 39300 | Loss: 0.0056 | Spent: 49.2 secs | LR: 0.000015
INFO:tensorflow:Step 39400 | Loss: 0.0031 | Spent: 50.5 secs | LR: 0.000015
INFO:tensorflow:Step 39500 | Loss: 0.0007 | Spent: 48.8 secs | LR: 0.000015
INFO:tensorflow:Step 39600 | Loss: 0.0013 | Spent: 47.8 secs | LR: 0.000015
INFO:tensorflow:Step 39700 | Loss: 0.0074 | Spent: 49.1 secs | LR: 0.000015
INFO:tensorflow:Step 39800 | Loss: 0.0157 | Spent: 49.0 secs | LR: 0.000016
INFO:tensorflow:Step 39900 | Loss: 0.0223 | Spent: 48.4 secs | LR: 0.000016
INFO:tensorflow:Step 40000 | Loss: 0.0069 | Spent: 47.3 secs | LR: 0.000016
INFO:tensorflow:Step 40100 | Loss: 0.1438 | Spent: 49.6 secs | LR: 0.000016
INFO:tensorflow:Step 40200 | Loss: 0.0888 | Spent: 48.6 secs | LR: 0.000016
INFO:tensorflow:Step 40300 | Loss: 0.1315 | Spent: 48.1 secs | LR: 0.000016
INFO:tensorflow:Step 40400 | Loss: 0.0071 | Spent: 48.6 secs | LR: 0.000016
INFO:tensorflow:Step 40500 | Loss: 0.0484 | Spent: 48.4 secs | LR: 0.000016
INFO:tensorflow:Step 40600 | Loss: 0.0044 | Spent: 48.3 secs | LR: 0.000016
INFO:tensorflow:Step 40700 | Loss: 0.0446 | Spent: 48.3 secs | LR: 0.000016
INFO:tensorflow:Step 40800 | Loss: 0.0014 | Spent: 48.8 secs | LR: 0.000016
INFO:tensorflow:Step 40900 | Loss: 0.0198 | Spent: 47.3 secs | LR: 0.000017
INFO:tensorflow:Step 41000 | Loss: 0.0972 | Spent: 48.4 secs | LR: 0.000017
INFO:tensorflow:Step 41100 | Loss: 0.0202 | Spent: 51.5 secs | LR: 0.000017
INFO:tensorflow:Step 41200 | Loss: 0.0035 | Spent: 48.2 secs | LR: 0.000017
INFO:tensorflow:Step 41300 | Loss: 0.0453 | Spent: 49.5 secs | LR: 0.000017
INFO:tensorflow:Step 41400 | Loss: 0.0127 | Spent: 48.2 secs | LR: 0.000017
INFO:tensorflow:Step 41500 | Loss: 0.0052 | Spent: 49.5 secs | LR: 0.000017
INFO:tensorflow:Step 41600 | Loss: 0.3876 | Spent: 50.5 secs | LR: 0.000017
INFO:tensorflow:Step 41700 | Loss: 0.0216 | Spent: 48.2 secs | LR: 0.000017
INFO:tensorflow:Step 41800 | Loss: 0.0117 | Spent: 48.8 secs | LR: 0.000017
INFO:tensorflow:Step 41900 | Loss: 0.0051 | Spent: 47.4 secs | LR: 0.000017
Reading ../data/test.csv
INFO:tensorflow:测试集:微众银行智能客服
INFO:tensorflow:Evaluation: Testing Accuracy: 0.832
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.798     0.890     0.842      5000
     Matched      0.876     0.775     0.822      5000

    accuracy                          0.832     10000
   macro avg      0.837     0.832     0.832     10000
weighted avg      0.837     0.832     0.832     10000

Reading ../data/dev.json
INFO:tensorflow:测试集:蚂蚁金融语义相似度
INFO:tensorflow:Evaluation: Testing Accuracy: 0.731
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.785     0.842     0.812      2978
     Matched      0.579     0.486     0.528      1338

    accuracy                          0.731      4316
   macro avg      0.682     0.664     0.670      4316
weighted avg      0.721     0.731     0.724      4316

INFO:tensorflow:Best | Accuracy 1: 0.844 | Accuracy 2: 0.740
5 times not improve the best result, therefore stop training