In [1]:
!wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.0-linux-x86_64.tar.gz
!tar -xzf elasticsearch-7.9.0-linux-x86_64.tar.gz
!chown -R daemon:daemon elasticsearch-7.9.0
--2020-11-16 08:37:25--  https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.0-linux-x86_64.tar.gz
Resolving artifacts.elastic.co (artifacts.elastic.co)... 151.101.2.222, 151.101.66.222, 151.101.130.222, ...
Connecting to artifacts.elastic.co (artifacts.elastic.co)|151.101.2.222|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 319531145 (305M) [application/x-gzip]
Saving to: ‘elasticsearch-7.9.0-linux-x86_64.tar.gz’

elasticsearch-7.9.0 100%[===================>] 304.73M  23.7MB/s    in 12s     

2020-11-16 08:37:38 (24.8 MB/s) - ‘elasticsearch-7.9.0-linux-x86_64.tar.gz’ saved [319531145/319531145]

In [2]:
!pip install elasticsearch -q
!pip install tensorflow_text -q
     |████████████████████████████████| 327kB 4.3MB/s 
     |████████████████████████████████| 2.6MB 4.3MB/s 
In [3]:
from subprocess import Popen, PIPE, STDOUT
from elasticsearch import Elasticsearch
from elasticsearch import helpers

import os
import time
import pprint
import tensorflow_hub as hub
import numpy as np
import tensorflow_text
In [4]:
es_server = Popen(
  ['elasticsearch-7.9.0/bin/elasticsearch'], 
  stdout = PIPE, stderr = STDOUT,
  preexec_fn = lambda: os.setuid(1))
In [5]:
!curl -X GET "localhost:9200/"
{
  "name" : "9fcc90d6da6c",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "vMZR7jeTRRq1Z7mXiaQJjw",
  "version" : {
    "number" : "7.9.0",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "a479a2a7fce0389512d6a9361301708b92dff667",
    "build_date" : "2020-08-11T21:36:48.204330Z",
    "build_snapshot" : false,
    "lucene_version" : "8.6.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}
In [6]:
def gen_data():
  with open('/content/gdrive/My Drive/finch/es/free_chat/data/basic.txt') as f:
    for line in f:
      line = line.rstrip()
      q, a = line.split('<SEP>')
      yield {
        '_index': 'chatbot',
        'question': q,
        'answer': a,
        'question_embedding': encoder(q)[0].numpy(),}
In [7]:
from google.colab import drive
drive.mount('/content/gdrive')
Mounted at /content/gdrive
In [8]:
es = Elasticsearch()
print(es.ping())

es.indices.create(index='chatbot')

mapping = {
  'properties': {
    'question': {
      'type': 'text',
    },
    'question_embedding': {
      'type': 'dense_vector',
      'dims': 512,}}}
es.indices.put_mapping(body=mapping, index='chatbot')

encoder = hub.load('https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3')

helpers.bulk(es, gen_data())
True
Out[8]:
(366, [])
In [10]:
correct, total = 0., 0.
while True:
  text_inp = input('Input:')
  if text_inp == '|quit':
    break
  t0 = time.time()
  query_vector = encoder([text_inp])[0].numpy()
  script_query = {
    'script_score': {
      'query': {'match_all': {}},
      'script': {
        'source': "cosineSimilarity(params.query_vector, doc['question_embedding']) + 1.0",
        'params': {'query_vector': query_vector},}}}
  dsl = {
    'query': script_query,
    '_source': {'excludes': ['question_embedding']},}
  hits = es.search(index='chatbot', body=dsl)['hits']['hits']
  print('Match:', hits[0]['_source'])
  print('%.2f sec' % (time.time() - t0))
  text_inp = input('Is the answer correct?')
  if text_inp == 'yes':
    correct += 1.
  total += 1
  print()
print('Correct: {} | Total: {} | Accuracy: {:.3f}'.format(correct, total, correct/total))
Input:早安
/usr/local/lib/python3.6/dist-packages/elasticsearch/connection/base.py:190: ElasticsearchDeprecationWarning: The vector functions of the form function(query, doc['field']) are deprecated, and the form function(query, 'field') should be used instead. For example, cosineSimilarity(query, doc['field']) is replaced by cosineSimilarity(query, 'field').
  warnings.warn(message, category=ElasticsearchDeprecationWarning)
Match: {'question': '早安', 'answer': '早安'}
0.06 sec
Is the answer correct?yes

Input:天气
Match: {'question': '天气', 'answer': '天气还行 要查具体的吗'}
0.06 sec
Is the answer correct?yes

Input:几点了
Match: {'question': '现在几点了', 'answer': '要查一下现在的具体时间吗'}
0.06 sec
Is the answer correct?yes

Input:冷
Match: {'question': '冷', 'answer': '多穿衣服哈'}
0.06 sec
Is the answer correct?yes

Input:热死了
Match: {'question': '有点热', 'answer': '对啊, 热死人了'}
0.06 sec
Is the answer correct?yes

Input:激动死了
Match: {'question': '太激动了', 'answer': '发生什么好事了 说来听听'}
0.06 sec
Is the answer correct?yes

Input:我伤心
Match: {'question': '我好伤心', 'answer': '怎么了 说来听听'}
0.06 sec
Is the answer correct?yes

Input:厉害了
Match: {'question': '你好厉害啊', 'answer': '这么夸我我会骄傲的'}
0.07 sec
Is the answer correct?yes

Input:我爱你
Match: {'question': '我爱你', 'answer': '我也爱你'}
0.06 sec
Is the answer correct?yes

Input:我喜欢你
Match: {'question': '我爱你', 'answer': '我也爱你'}
0.07 sec
Is the answer correct?yes

Input:|quit
Correct: 10.0 | Total: 10.0 | Accuracy: 1.000