In [1]:
# Copyright 2019 Google Inc.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
In [14]:
!pip install tqdm bert-for-tf2
Requirement already satisfied: tqdm in /opt/anaconda3/envs/bentoml-dev-py36/lib/python3.6/site-packages (4.45.0)
Requirement already satisfied: bert-for-tf2 in /opt/anaconda3/envs/bentoml-dev-py36/lib/python3.6/site-packages (0.14.1)
Requirement already satisfied: params-flow>=0.8.0 in /home/bentoml/.local/lib/python3.6/site-packages (from bert-for-tf2) (0.8.0)
Requirement already satisfied: py-params>=0.9.6 in /home/bentoml/.local/lib/python3.6/site-packages (from bert-for-tf2) (0.9.7)
Requirement already satisfied: numpy in /opt/anaconda3/envs/bentoml-dev-py36/lib/python3.6/site-packages (from params-flow>=0.8.0->bert-for-tf2) (1.18.1)
In [1]:
import os
import sys
import math
import datetime
from tqdm import tqdm
import pandas as pd
import numpy as np
import tensorflow as tf

# tf.config.set_visible_devices([], 'GPU')  # disable GPU
In [2]:
print("Tensorflow: ", tf.__version__)
print("Python: ", sys.version)
print("GPU: ", tf.test.is_gpu_available())
assert sys.version_info.major == 3 and sys.version_info.minor == 6  # required by clipper benchmark
Tensorflow:  2.1.0
Python:  3.6.10 |Anaconda, Inc.| (default, Jan  7 2020, 21:14:29) 
[GCC 7.3.0]
WARNING:tensorflow:From <ipython-input-2-7068fd7facab>:3: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU:  True
In [3]:
import bert
from bert import BertModelLayer
from bert.loader import StockBertConfig, map_stock_config_to_params, load_stock_weights
from bert.tokenization.bert_tokenization import FullTokenizer
from tensorflow import keras
import os
import re
In [4]:
from tensorflow import keras
import os
import re

# Load all files from a directory in a DataFrame.
def load_directory_data(directory):
  data = {}
  data["sentence"] = []
  data["sentiment"] = []
  for file_path in tqdm(os.listdir(directory), desc=os.path.basename(directory)):
    with tf.io.gfile.GFile(os.path.join(directory, file_path), "r") as f:
      data["sentence"].append(f.read())
      data["sentiment"].append(re.match("\d+_(\d+)\.txt", file_path).group(1))
  return pd.DataFrame.from_dict(data)

# Merge positive and negative examples, add a polarity column and shuffle.
def load_dataset(directory):
    pos_df = load_directory_data(os.path.join(directory, "pos"))
    neg_df = load_directory_data(os.path.join(directory, "neg"))
    pos_df["polarity"] = 1
    neg_df["polarity"] = 0
    return pd.concat([pos_df, neg_df]).sample(frac=1).reset_index(drop=True)

# Download and process the dataset files.
def download_and_load_datasets(force_download=False):
    dataset = tf.keras.utils.get_file(
        fname="aclImdb.tar.gz", 
        origin="http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz", 
        extract=True)

    train_df = load_dataset(os.path.join(os.path.dirname(dataset), 
                           "aclImdb", "train"))
    test_df = load_dataset(os.path.join(os.path.dirname(dataset), 
                          "aclImdb", "test"))

    return train_df, test_df

Let's use the MovieReviewData class below, to prepare/encode the data for feeding into our BERT model, by:

  • tokenizing the text
  • trim or pad it to a max_seq_len length
  • append the special tokens [CLS] and [SEP]
  • convert the string tokens to numerical IDs using the original model's token encoding from vocab.txt
In [5]:
import bert
from bert import BertModelLayer
from bert.loader import StockBertConfig, map_stock_config_to_params, load_stock_weights


class MovieReviewData:
    DATA_COLUMN = "sentence"
    LABEL_COLUMN = "polarity"

    def __init__(self, tokenizer: FullTokenizer, sample_size=None, max_seq_len=1024):
        self.tokenizer = tokenizer
        self.sample_size = sample_size
        self.max_seq_len = 0
        train, test = download_and_load_datasets()
        
        train, test = map(lambda df: df.reindex(df[MovieReviewData.DATA_COLUMN].str.len().sort_values().index), 
                          [train, test])
                
        if sample_size is not None:
            train, test = train.head(sample_size), test.head(sample_size)
            # train, test = map(lambda df: df.sample(sample_size), [train, test])
        
        ((self.train_x, self.train_y),
         (self.test_x, self.test_y)) = map(self._prepare, [train, test])

        print("max seq_len", self.max_seq_len)
        self.max_seq_len = min(self.max_seq_len, max_seq_len)
        ((self.train_x, self.train_x_token_types),
         (self.test_x, self.test_x_token_types)) = map(self._pad, 
                                                       [self.train_x, self.test_x])

    def _prepare(self, df):
        x, y = [], []
        with tqdm(total=df.shape[0], unit_scale=True) as pbar:
            for ndx, row in df.iterrows():
                text, label = row[MovieReviewData.DATA_COLUMN], row[MovieReviewData.LABEL_COLUMN]
                tokens = self.tokenizer.tokenize(text)
                tokens = ["[CLS]"] + tokens + ["[SEP]"]
                token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
                self.max_seq_len = max(self.max_seq_len, len(token_ids))
                x.append(token_ids)
                y.append(int(label))
                pbar.update()
        return np.array(x), np.array(y)

    def _pad(self, ids):
        x, t = [], []
        token_type_ids = [0] * self.max_seq_len
        for input_ids in ids:
            input_ids = input_ids[:min(len(input_ids), self.max_seq_len - 2)]
            input_ids = input_ids + [0] * (self.max_seq_len - len(input_ids))
            x.append(np.array(input_ids))
            t.append(token_type_ids)
        return np.array(x), np.array(t)

A tweak

Because of a tf.train.load_checkpoint limitation requiring list permissions on the google storage bucket, we need to copy the pre-trained BERT weights locally.

In [6]:
asset_path = 'asset'
bert_model_name = "uncased_L-12_H-768_A-12"
bert_ckpt_dir    = os.path.join(asset_path, bert_model_name)
bert_ckpt_file   = os.path.join(bert_ckpt_dir, "bert_model.ckpt")
bert_config_file = os.path.join(bert_ckpt_dir, "bert_config.json")
In [7]:
%%bash

if [ ! -f asset/uncased_L-12_H-768_A-12.zip ]; then
    curl -o asset/uncased_L-12_H-768_A-12.zip --create-dirs https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
fi
if [ ! -d asset/uncased_L-12_H-768_A-12 ]; then
    unzip asset/uncased_L-12_H-768_A-12.zip -d asset/
fi

Preparing the Data

Now let's fetch and prepare the data by taking the first max_seq_len tokenens after tokenizing with the BERT tokenizer, und use sample_size examples for both training and testing.

To keep training fast, we'll take a sample of about 2500 train and test examples, respectively, and use the first 128 tokens only (transformers memory and computation requirements scale quadraticly with the sequence length - so with a TPU you might use max_seq_len=512, but on a GPU this would be too slow, and you will have to use a very small batch_sizes to fit the model into the GPU memory).

In [8]:
%%time

tokenizer = FullTokenizer(vocab_file=os.path.join(bert_ckpt_dir, "vocab.txt"))
data = MovieReviewData(tokenizer, 
                       sample_size=10*128*2, #10*128*2
                       max_seq_len=128)
pos: 100%|██████████| 12500/12500 [00:00<00:00, 19564.03it/s]
neg: 100%|██████████| 12500/12500 [00:00<00:00, 19615.21it/s]
pos: 100%|██████████| 12500/12500 [00:00<00:00, 19864.62it/s]
neg: 100%|██████████| 12500/12500 [00:00<00:00, 19896.49it/s]
100%|██████████| 2.56k/2.56k [00:02<00:00, 897it/s]  
100%|██████████| 2.56k/2.56k [00:02<00:00, 922it/s]  
max seq_len 178
CPU times: user 19 s, sys: 4.6 s, total: 23.6 s
Wall time: 23.6 s
In [9]:
print("            train_x", data.train_x.shape)
print("train_x_token_types", data.train_x_token_types.shape)
print("            train_y", data.train_y.shape)

print("             test_x", data.test_x.shape)

print("        max_seq_len", data.max_seq_len)
            train_x (2560, 128)
train_x_token_types (2560, 128)
            train_y (2560,)
             test_x (2560, 128)
        max_seq_len 128

Adapter BERT

If we decide to use adapter-BERT we need some helpers for freezing the original BERT layers.

In [10]:
def flatten_layers(root_layer):
    if isinstance(root_layer, keras.layers.Layer):
        yield root_layer
    for layer in root_layer._layers:
        for sub_layer in flatten_layers(layer):
            yield sub_layer


def freeze_bert_layers(l_bert):
    """
    Freezes all but LayerNorm and adapter layers - see arXiv:1902.00751.
    """
    for layer in flatten_layers(l_bert):
        if layer.name in ["LayerNorm", "adapter-down", "adapter-up"]:
            layer.trainable = True
        elif len(layer._layers) == 0:
            layer.trainable = False
        l_bert.embeddings_layer.trainable = False


def create_learning_rate_scheduler(max_learn_rate=5e-5,
                                   end_learn_rate=1e-7,
                                   warmup_epoch_count=10,
                                   total_epoch_count=90):

    def lr_scheduler(epoch):
        if epoch < warmup_epoch_count:
            res = (max_learn_rate/warmup_epoch_count) * (epoch + 1)
        else:
            res = max_learn_rate*math.exp(
                math.log(end_learn_rate/max_learn_rate)*(epoch-warmup_epoch_count+1)/(total_epoch_count-warmup_epoch_count+1))
        return float(res)
    learning_rate_scheduler = tf.keras.callbacks.LearningRateScheduler(lr_scheduler, verbose=1)

    return learning_rate_scheduler

Creating a model

Now let's create a classification model using adapter-BERT, which is clever way of reducing the trainable parameter count, by freezing the original BERT weights, and adapting them with two FFN bottlenecks (i.e. adapter_size bellow) in every BERT layer.

N.B. The commented out code below show how to feed a token_type_ids/segment_ids sequence (which is not needed in our case).

In [11]:
def create_model(max_seq_len, adapter_size=64):
  """Creates a classification model."""

  #adapter_size = 64  # see - arXiv:1902.00751

  # create the bert layer
  with tf.io.gfile.GFile(bert_config_file, "r") as reader:
      bc = StockBertConfig.from_json_string(reader.read())
      bert_params = map_stock_config_to_params(bc)
      bert_params.adapter_size = adapter_size
      bert = BertModelLayer.from_params(bert_params, name="bert")
        
  input_ids      = keras.layers.Input(shape=(max_seq_len,), dtype='int32', name="input_ids")
  # token_type_ids = keras.layers.Input(shape=(max_seq_len,), dtype='int32', name="token_type_ids")
  # output         = bert([input_ids, token_type_ids])
  output         = bert(input_ids)

  print("bert shape", output.shape)
  cls_out = keras.layers.Lambda(lambda seq: seq[:, 0, :])(output)
  cls_out = keras.layers.Dropout(0.5)(cls_out)
  logits = keras.layers.Dense(units=768, activation="tanh")(cls_out)
  logits = keras.layers.Dropout(0.5)(logits)
  logits = keras.layers.Dense(units=2, activation="softmax")(logits)

  # model = keras.Model(inputs=[input_ids, token_type_ids], outputs=logits)
  # model.build(input_shape=[(None, max_seq_len), (None, max_seq_len)])
  model = keras.Model(inputs=input_ids, outputs=logits)
  model.build(input_shape=(None, max_seq_len))

  # load the pre-trained model weights
  load_stock_weights(bert, bert_ckpt_file)

  # freeze weights if adapter-BERT is used
  if adapter_size is not None:
      freeze_bert_layers(bert)

  model.compile(optimizer=keras.optimizers.Adam(),
                loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                metrics=[keras.metrics.SparseCategoricalAccuracy(name="acc")])

  model.summary()
        
  return model

Train

In [12]:
adapter_size = None # use None to fine-tune all of BERT
model = create_model(data.max_seq_len, adapter_size=adapter_size)
bert shape (None, 128, 768)
Done loading 196 BERT weights from: asset/uncased_L-12_H-768_A-12/bert_model.ckpt into <bert.model.BertModelLayer object at 0x7f44990ff9e8> (prefix:bert). Count of weights not found in the checkpoint was: [0]. Count of weights with mismatched shape: [0]
Unused weights from checkpoint: 
	bert/embeddings/token_type_embeddings
	bert/pooler/dense/bias
	bert/pooler/dense/kernel
	cls/predictions/output_bias
	cls/predictions/transform/LayerNorm/beta
	cls/predictions/transform/LayerNorm/gamma
	cls/predictions/transform/dense/bias
	cls/predictions/transform/dense/kernel
	cls/seq_relationship/output_bias
	cls/seq_relationship/output_weights
Model: "model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_ids (InputLayer)       [(None, 128)]             0         
_________________________________________________________________
bert (BertModelLayer)        (None, 128, 768)          108890112 
_________________________________________________________________
lambda (Lambda)              (None, 768)               0         
_________________________________________________________________
dropout (Dropout)            (None, 768)               0         
_________________________________________________________________
dense (Dense)                (None, 768)               590592    
_________________________________________________________________
dropout_1 (Dropout)          (None, 768)               0         
_________________________________________________________________
dense_1 (Dense)              (None, 2)                 1538      
=================================================================
Total params: 109,482,242
Trainable params: 109,482,242
Non-trainable params: 0
_________________________________________________________________
In [13]:
%%time

log_dir = ".log/movie_reviews/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%s")
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=log_dir)

total_epoch_count = 20
# model.fit(x=(data.train_x, data.train_x_token_types), y=data.train_y,
model.fit(x=data.train_x, y=data.train_y,
          validation_split=0.1,
          batch_size=12,
          shuffle=True,
          epochs=total_epoch_count,
          callbacks=[create_learning_rate_scheduler(max_learn_rate=1e-5,
                                                    end_learn_rate=1e-7,
                                                    warmup_epoch_count=20,
                                                    total_epoch_count=total_epoch_count),
                     keras.callbacks.EarlyStopping(patience=20, restore_best_weights=True),
                     tensorboard_callback])
Train on 2304 samples, validate on 256 samples

Epoch 00001: LearningRateScheduler reducing learning rate to 5.000000000000001e-07.
Epoch 1/20
2304/2304 [==============================] - 130s 56ms/sample - loss: 0.7038 - acc: 0.5556 - val_loss: 0.6826 - val_acc: 0.5273

Epoch 00002: LearningRateScheduler reducing learning rate to 1.0000000000000002e-06.
Epoch 2/20
2304/2304 [==============================] - 124s 54ms/sample - loss: 0.6928 - acc: 0.5499 - val_loss: 0.6581 - val_acc: 0.5781

Epoch 00003: LearningRateScheduler reducing learning rate to 1.5000000000000002e-06.
Epoch 3/20
2304/2304 [==============================] - 124s 54ms/sample - loss: 0.6723 - acc: 0.5864 - val_loss: 0.6119 - val_acc: 0.7344

Epoch 00004: LearningRateScheduler reducing learning rate to 2.0000000000000003e-06.
Epoch 4/20
2304/2304 [==============================] - 122s 53ms/sample - loss: 0.5676 - acc: 0.7296 - val_loss: 0.4608 - val_acc: 0.8555

Epoch 00005: LearningRateScheduler reducing learning rate to 2.5000000000000006e-06.
Epoch 5/20
2304/2304 [==============================] - 124s 54ms/sample - loss: 0.4319 - acc: 0.8837 - val_loss: 0.4156 - val_acc: 0.9062

Epoch 00006: LearningRateScheduler reducing learning rate to 3.0000000000000005e-06.
Epoch 6/20
2304/2304 [==============================] - 127s 55ms/sample - loss: 0.3982 - acc: 0.9158 - val_loss: 0.4173 - val_acc: 0.8945

Epoch 00007: LearningRateScheduler reducing learning rate to 3.5000000000000004e-06.
Epoch 7/20
2304/2304 [==============================] - 127s 55ms/sample - loss: 0.3786 - acc: 0.9358 - val_loss: 0.4217 - val_acc: 0.8867

Epoch 00008: LearningRateScheduler reducing learning rate to 4.000000000000001e-06.
Epoch 8/20
2304/2304 [==============================] - 128s 55ms/sample - loss: 0.3640 - acc: 0.9488 - val_loss: 0.4215 - val_acc: 0.8906

Epoch 00009: LearningRateScheduler reducing learning rate to 4.500000000000001e-06.
Epoch 9/20
2304/2304 [==============================] - 128s 55ms/sample - loss: 0.3548 - acc: 0.9583 - val_loss: 0.4161 - val_acc: 0.8945

Epoch 00010: LearningRateScheduler reducing learning rate to 5.000000000000001e-06.
Epoch 10/20
2304/2304 [==============================] - 127s 55ms/sample - loss: 0.3493 - acc: 0.9653 - val_loss: 0.4306 - val_acc: 0.8789

Epoch 00011: LearningRateScheduler reducing learning rate to 5.500000000000001e-06.
Epoch 11/20
2304/2304 [==============================] - 127s 55ms/sample - loss: 0.3485 - acc: 0.9631 - val_loss: 0.4167 - val_acc: 0.8945

Epoch 00012: LearningRateScheduler reducing learning rate to 6.000000000000001e-06.
Epoch 12/20
2304/2304 [==============================] - 126s 55ms/sample - loss: 0.3458 - acc: 0.9674 - val_loss: 0.4385 - val_acc: 0.8672

Epoch 00013: LearningRateScheduler reducing learning rate to 6.500000000000001e-06.
Epoch 13/20
2304/2304 [==============================] - 126s 55ms/sample - loss: 0.3438 - acc: 0.9701 - val_loss: 0.4144 - val_acc: 0.8906

Epoch 00014: LearningRateScheduler reducing learning rate to 7.000000000000001e-06.
Epoch 14/20
2304/2304 [==============================] - 126s 55ms/sample - loss: 0.3440 - acc: 0.9683 - val_loss: 0.4520 - val_acc: 0.8594

Epoch 00015: LearningRateScheduler reducing learning rate to 7.500000000000001e-06.
Epoch 15/20
2304/2304 [==============================] - 128s 56ms/sample - loss: 0.3446 - acc: 0.9688 - val_loss: 0.4258 - val_acc: 0.8867

Epoch 00016: LearningRateScheduler reducing learning rate to 8.000000000000001e-06.
Epoch 16/20
2304/2304 [==============================] - 130s 56ms/sample - loss: 0.3451 - acc: 0.9679 - val_loss: 0.4414 - val_acc: 0.8711

Epoch 00017: LearningRateScheduler reducing learning rate to 8.500000000000002e-06.
Epoch 17/20
2304/2304 [==============================] - 128s 56ms/sample - loss: 0.3388 - acc: 0.9748 - val_loss: 0.4401 - val_acc: 0.8711

Epoch 00018: LearningRateScheduler reducing learning rate to 9.000000000000002e-06.
Epoch 18/20
2304/2304 [==============================] - 129s 56ms/sample - loss: 0.3370 - acc: 0.9761 - val_loss: 0.4362 - val_acc: 0.8711

Epoch 00019: LearningRateScheduler reducing learning rate to 9.500000000000002e-06.
Epoch 19/20
2304/2304 [==============================] - 128s 56ms/sample - loss: 0.3394 - acc: 0.9735 - val_loss: 0.4440 - val_acc: 0.8672

Epoch 00020: LearningRateScheduler reducing learning rate to 1.0000000000000003e-05.
Epoch 20/20
2304/2304 [==============================] - 127s 55ms/sample - loss: 0.3408 - acc: 0.9727 - val_loss: 0.4424 - val_acc: 0.8711
CPU times: user 41min 17s, sys: 14min 34s, total: 55min 52s
Wall time: 42min 15s
Out[13]:
<tensorflow.python.keras.callbacks.History at 0x7f43e31a13c8>
In [22]:
model.save_weights('./movie_reviews.h5', overwrite=True)
In [15]:
%%time

_, train_acc = model.evaluate(data.train_x, data.train_y)
_, test_acc = model.evaluate(data.test_x, data.test_y)

print("train acc", train_acc)
print(" test acc", test_acc)
2560/2560 [==============================] - 34s 13ms/sample - loss: 0.3419 - acc: 0.9711
2560/2560 [==============================] - 33s 13ms/sample - loss: 0.3903 - acc: 0.9230
train acc 0.9710938
 test acc 0.9230469
CPU times: user 1min 6s, sys: 194 ms, total: 1min 7s
Wall time: 1min 6s

Evaluation

To evaluate the trained model, let's load the saved weights in a new model instance, and evaluate.

In [12]:
model = create_model(data.max_seq_len, adapter_size=None)
model.load_weights("./movie_reviews.h5")
bert shape (None, 128, 768)
Done loading 196 BERT weights from: asset/uncased_L-12_H-768_A-12/bert_model.ckpt into <bert.model.BertModelLayer object at 0x7ff6b45c4160> (prefix:bert). Count of weights not found in the checkpoint was: [0]. Count of weights with mismatched shape: [0]
Unused weights from checkpoint: 
	bert/embeddings/token_type_embeddings
	bert/pooler/dense/bias
	bert/pooler/dense/kernel
	cls/predictions/output_bias
	cls/predictions/transform/LayerNorm/beta
	cls/predictions/transform/LayerNorm/gamma
	cls/predictions/transform/dense/bias
	cls/predictions/transform/dense/kernel
	cls/seq_relationship/output_bias
	cls/seq_relationship/output_weights
Model: "model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_ids (InputLayer)       [(None, 128)]             0         
_________________________________________________________________
bert (BertModelLayer)        (None, 128, 768)          108890112 
_________________________________________________________________
lambda (Lambda)              (None, 768)               0         
_________________________________________________________________
dropout (Dropout)            (None, 768)               0         
_________________________________________________________________
dense (Dense)                (None, 768)               590592    
_________________________________________________________________
dropout_1 (Dropout)          (None, 768)               0         
_________________________________________________________________
dense_1 (Dense)              (None, 2)                 1538      
=================================================================
Total params: 109,482,242
Trainable params: 109,482,242
Non-trainable params: 0
_________________________________________________________________
In [17]:
%%time 

# _, train_acc = model.evaluate(data.train_x, data.train_y)
_, test_acc = model.evaluate(data.test_x, data.test_y)

# print("train acc", train_acc)
print(" test acc", test_acc)
2560/2560 [==============================] - 35s 14ms/sample - loss: 0.3903 - acc: 0.9230
 test acc 0.9230469
CPU times: user 34.6 s, sys: 113 ms, total: 34.7 s
Wall time: 34.6 s

Prediction

For prediction, we need to prepare the input text the same way as we did for training - tokenize, adding the special [CLS] and [SEP] token at begin and end of the token sequence, and pad to match the model input shape.

In [13]:
%%time
CLASSES = ["negative","positive"]
max_seq_len = 128
pred_sentences = [
  "That movie was absolutely awful",
  "The acting was a bit lacking",
  "The film was creative and surprising",
  "Absolutely fantastic!",
]

inputs = pd.DataFrame(pred_sentences)

pred_tokens    = map(tokenizer.tokenize, inputs.to_numpy()[:, 0].tolist())
pred_tokens    = map(lambda tok: ["[CLS]"] + tok + ["[SEP]"], pred_tokens)
pred_token_ids = list(map(tokenizer.convert_tokens_to_ids, pred_tokens))
pred_token_ids = map(lambda tids: tids + [0] * (max_seq_len-len(tids)), pred_token_ids)
pred_token_ids = np.array(list(pred_token_ids))

res = model(pred_token_ids).numpy().argmax(axis=-1)
[CLASSES[i] for i in res]
CPU times: user 322 ms, sys: 36 ms, total: 358 ms
Wall time: 360 ms
Out[13]:
['negative', 'negative', 'positive', 'positive']

Build & Save bentoml service

In [15]:
%%writefile bentoml_service.py

import bentoml
import tensorflow as tf
import numpy as np
import pandas as pd

from bentoml.artifact import (
    TensorflowSavedModelArtifact, PickleArtifact,
)
from bentoml.handlers import DataframeHandler, ClipperStringsHandler


CLASSES = ["negative","positive"]
max_seq_len = 128

try:
    tf.config.set_visible_devices([], 'GPU')  # disable GPU, required when served in docker
except:
    pass


@bentoml.env(pip_dependencies=['tensorflow', 'bert-for-tf2'])
@bentoml.artifacts([TensorflowSavedModelArtifact('model'), PickleArtifact('tokenizer')])
class Service(bentoml.BentoService):

    def tokenize(self, inputs: pd.DataFrame):
        tokenizer = self.artifacts.tokenizer
        if isinstance(inputs, pd.DataFrame):
            inputs = inputs.to_numpy()[:, 0].tolist()
        else:
            inputs = inputs.tolist()  # for predict_clipper
        pred_tokens = map(tokenizer.tokenize, inputs)
        pred_tokens = map(lambda tok: ["[CLS]"] + tok + ["[SEP]"], pred_tokens)
        pred_token_ids = list(map(tokenizer.convert_tokens_to_ids, pred_tokens))
        pred_token_ids = map(lambda tids: tids + [0] * (max_seq_len - len(tids)), pred_token_ids)
        pred_token_ids = tf.constant(list(pred_token_ids), dtype=tf.int32)
        return pred_token_ids

    @bentoml.api(DataframeHandler, mb_max_latency=300, mb_max_batch_size=20)
    def predict(self, inputs):
        model = self.artifacts.model
        pred_token_ids = self.tokenize(inputs)
        res = model(pred_token_ids).numpy().argmax(axis=-1)
        return [CLASSES[i] for i in res]

    @bentoml.api(ClipperStringsHandler)
    def predict_clipper(self, strings):
        model = self.artifacts.model
        pred_token_ids = self.tokenize(strings)
        res = model(pred_token_ids).numpy().argmax(axis=-1)
        return [CLASSES[i] for i in res]
Overwriting bentoml_service.py
In [16]:
from bentoml_service import Service

bento_svc = Service()
bento_svc.pack("model", model)
bento_svc.pack("tokenizer", tokenizer)
saved_path = bento_svc.save()
[2020-04-23 23:37:54,466] WARNING - BentoML local changes detected - Local BentoML repository including all code changes will be bundled together with the BentoService bundle. When used with docker, the base docker image will be default to same version as last PyPI release at version: 0.5.2. You can also force bentoml to use a specific version for deploying your BentoService bundle, by setting the config 'core/bentoml_deploy_version' to a pinned version or your custom BentoML on github, e.g.:'bentoml_deploy_version = git+https://github.com/{username}/[email protected]{branch}'
[2020-04-23 23:37:54,814] WARNING - BentoML local changes detected - Local BentoML repository including all code changes will be bundled together with the BentoService bundle. When used with docker, the base docker image will be default to same version as last PyPI release at version: 0.5.2. You can also force bentoml to use a specific version for deploying your BentoService bundle, by setting the config 'core/bentoml_deploy_version' to a pinned version or your custom BentoML on github, e.g.:'bentoml_deploy_version = git+https://github.com/{username}/[email protected]{branch}'
WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.embeddings.Embedding object at 0x7ff62d9c9ef0>, because it is not built.
WARNING:tensorflow:From /opt/anaconda3/envs/bentoml-dev-py36/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1786: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
INFO:tensorflow:Assets written to: /tmp/bentoml-temp-whqt7e5s/Service/artifacts/model_saved_model/assets
[2020-04-23 23:39:01,739] WARNING - BentoML local changes detected - Local BentoML repository including all code changes will be bundled together with the BentoService bundle. When used with docker, the base docker image will be default to same version as last PyPI release at version: 0.5.2. You can also force bentoml to use a specific version for deploying your BentoService bundle, by setting the config 'core/bentoml_deploy_version' to a pinned version or your custom BentoML on github, e.g.:'bentoml_deploy_version = git+https://github.com/{username}/[email protected]{branch}'
running sdist
running egg_info
writing BentoML.egg-info/PKG-INFO
writing dependency_links to BentoML.egg-info/dependency_links.txt
writing entry points to BentoML.egg-info/entry_points.txt
writing requirements to BentoML.egg-info/requires.txt
writing top-level names to BentoML.egg-info/top_level.txt
reading manifest file 'BentoML.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
warning: no previously-included files matching '*~' found anywhere in distribution
warning: no previously-included files matching '*.pyo' found anywhere in distribution
warning: no previously-included files matching '.git' found anywhere in distribution
warning: no previously-included files matching '.ipynb_checkpoints' found anywhere in distribution
warning: no previously-included files matching '__pycache__' found anywhere in distribution
warning: no directories found matching 'bentoml/yatai/web/dist'
no previously-included directories found matching 'e2e_tests'
no previously-included directories found matching 'tests'
no previously-included directories found matching 'benchmark'
writing manifest file 'BentoML.egg-info/SOURCES.txt'
running check
creating BentoML-0.5.2+173.ga4278d4.dirty
creating BentoML-0.5.2+173.ga4278d4.dirty/BentoML.egg-info
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/bundler
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/clipper
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration/__pycache__
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/aws_lambda
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/sagemaker
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/marshal
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations/__pycache__
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations/versions
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations/versions/__pycache__
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/proto
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/repository
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server/static
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils/validator
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai
creating BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai/client
copying files to BentoML-0.5.2+173.ga4278d4.dirty...
copying LICENSE -> BentoML-0.5.2+173.ga4278d4.dirty
copying MANIFEST.in -> BentoML-0.5.2+173.ga4278d4.dirty
copying README.md -> BentoML-0.5.2+173.ga4278d4.dirty
copying pyproject.toml -> BentoML-0.5.2+173.ga4278d4.dirty
copying setup.cfg -> BentoML-0.5.2+173.ga4278d4.dirty
copying setup.py -> BentoML-0.5.2+173.ga4278d4.dirty
copying versioneer.py -> BentoML-0.5.2+173.ga4278d4.dirty
copying BentoML.egg-info/PKG-INFO -> BentoML-0.5.2+173.ga4278d4.dirty/BentoML.egg-info
copying BentoML.egg-info/SOURCES.txt -> BentoML-0.5.2+173.ga4278d4.dirty/BentoML.egg-info
copying BentoML.egg-info/dependency_links.txt -> BentoML-0.5.2+173.ga4278d4.dirty/BentoML.egg-info
copying BentoML.egg-info/entry_points.txt -> BentoML-0.5.2+173.ga4278d4.dirty/BentoML.egg-info
copying BentoML.egg-info/requires.txt -> BentoML-0.5.2+173.ga4278d4.dirty/BentoML.egg-info
copying BentoML.egg-info/top_level.txt -> BentoML-0.5.2+173.ga4278d4.dirty/BentoML.egg-info
copying bentoml/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml
copying bentoml/_version.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml
copying bentoml/alembic.ini -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml
copying bentoml/db.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml
copying bentoml/exceptions.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml
copying bentoml/service.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml
copying bentoml/service_env.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml
copying bentoml/artifact/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/fastai_model_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/fasttext_model_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/h2o_model_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/keras_model_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/lightgbm_model_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/pickle_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/pytorch_model_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/sklearn_model_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/text_file_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/tf_savedmodel_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/artifact/xgboost_model_artifact.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/artifact
copying bentoml/bundler/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/bundler
copying bentoml/bundler/bundler.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/bundler
copying bentoml/bundler/config.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/bundler
copying bentoml/bundler/loader.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/bundler
copying bentoml/bundler/py_module_utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/bundler
copying bentoml/bundler/templates.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/bundler
copying bentoml/bundler/utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/bundler
copying bentoml/cli/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
copying bentoml/cli/aws_lambda.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
copying bentoml/cli/aws_sagemaker.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
copying bentoml/cli/bento.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
copying bentoml/cli/click_utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
copying bentoml/cli/config.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
copying bentoml/cli/deployment.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
copying bentoml/cli/utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
copying bentoml/cli/yatai_service.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/cli
copying bentoml/clipper/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/clipper
copying bentoml/configuration/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration
copying bentoml/configuration/configparser.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration
copying bentoml/configuration/default_bentoml.cfg -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration
copying bentoml/configuration/__pycache__/__init__.cpython-36.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration/__pycache__
copying bentoml/configuration/__pycache__/__init__.cpython-37.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration/__pycache__
copying bentoml/configuration/__pycache__/__init__.cpython-38.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration/__pycache__
copying bentoml/configuration/__pycache__/configparser.cpython-36.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration/__pycache__
copying bentoml/configuration/__pycache__/configparser.cpython-37.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration/__pycache__
copying bentoml/configuration/__pycache__/configparser.cpython-38.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/configuration/__pycache__
copying bentoml/deployment/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment
copying bentoml/deployment/operator.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment
copying bentoml/deployment/store.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment
copying bentoml/deployment/utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment
copying bentoml/deployment/aws_lambda/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/aws_lambda
copying bentoml/deployment/aws_lambda/download_extra_resources.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/aws_lambda
copying bentoml/deployment/aws_lambda/lambda_app.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/aws_lambda
copying bentoml/deployment/aws_lambda/utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/aws_lambda
copying bentoml/deployment/sagemaker/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/sagemaker
copying bentoml/deployment/sagemaker/sagemaker_nginx.conf -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/sagemaker
copying bentoml/deployment/sagemaker/sagemaker_serve.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/sagemaker
copying bentoml/deployment/sagemaker/sagemaker_wsgi.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/deployment/sagemaker
copying bentoml/handlers/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/handlers/base_handlers.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/handlers/clipper_handler.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/handlers/dataframe_handler.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/handlers/fastai_image_handler.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/handlers/image_handler.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/handlers/json_handler.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/handlers/pytorch_tensor_handler.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/handlers/tensorflow_tensor_handler.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/handlers/utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/handlers
copying bentoml/marshal/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/marshal
copying bentoml/marshal/dispatcher.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/marshal
copying bentoml/marshal/marshal.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/marshal
copying bentoml/marshal/utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/marshal
copying bentoml/migrations/README -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations
copying bentoml/migrations/env.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations
copying bentoml/migrations/script.py.mako -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations
copying bentoml/migrations/__pycache__/env.cpython-36.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations/__pycache__
copying bentoml/migrations/__pycache__/env.cpython-37.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations/__pycache__
copying bentoml/migrations/versions/a6b00ae45279_add_last_updated_at_for_deployments.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations/versions
copying bentoml/migrations/versions/__pycache__/a6b00ae45279_add_last_updated_at_for_deployments.cpython-36.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations/versions/__pycache__
copying bentoml/migrations/versions/__pycache__/a6b00ae45279_add_last_updated_at_for_deployments.cpython-37.pyc -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/migrations/versions/__pycache__
copying bentoml/proto/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/proto
copying bentoml/proto/deployment_pb2.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/proto
copying bentoml/proto/repository_pb2.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/proto
copying bentoml/proto/status_pb2.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/proto
copying bentoml/proto/yatai_service_pb2.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/proto
copying bentoml/proto/yatai_service_pb2_grpc.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/proto
copying bentoml/repository/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/repository
copying bentoml/repository/metadata_store.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/repository
copying bentoml/server/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server
copying bentoml/server/bento_api_server.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server
copying bentoml/server/bento_sagemaker_server.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server
copying bentoml/server/gunicorn_config.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server
copying bentoml/server/gunicorn_server.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server
copying bentoml/server/marshal_server.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server
copying bentoml/server/middlewares.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server
copying bentoml/server/utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server
copying bentoml/server/static/swagger-ui-bundle.js -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server/static
copying bentoml/server/static/swagger-ui.css -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/server/static
copying bentoml/utils/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/benchmark.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/cloudpickle.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/hybridmethod.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/log.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/pip_pkg.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/s3.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/tempdir.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/trace.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/usage_stats.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils
copying bentoml/utils/validator/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/utils/validator
copying bentoml/yatai/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai
copying bentoml/yatai/deployment_utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai
copying bentoml/yatai/status.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai
copying bentoml/yatai/utils.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai
copying bentoml/yatai/yatai_service_impl.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai
copying bentoml/yatai/client/__init__.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai/client
copying bentoml/yatai/client/bento_repository_api.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai/client
copying bentoml/yatai/client/deployment_api.py -> BentoML-0.5.2+173.ga4278d4.dirty/bentoml/yatai/client
Writing BentoML-0.5.2+173.ga4278d4.dirty/setup.cfg
UPDATING BentoML-0.5.2+173.ga4278d4.dirty/bentoml/_version.py
set BentoML-0.5.2+173.ga4278d4.dirty/bentoml/_version.py to '0.5.2+173.ga4278d4.dirty'
Creating tar archive
removing 'BentoML-0.5.2+173.ga4278d4.dirty' (and everything under it)
[2020-04-23 23:39:03,215] WARNING - BentoML local changes detected - Local BentoML repository including all code changes will be bundled together with the BentoService bundle. When used with docker, the base docker image will be default to same version as last PyPI release at version: 0.5.2. You can also force bentoml to use a specific version for deploying your BentoService bundle, by setting the config 'core/bentoml_deploy_version' to a pinned version or your custom BentoML on github, e.g.:'bentoml_deploy_version = git+https://github.com/{username}/[email protected]{branch}'
[2020-04-23 23:39:03,230] WARNING - Saved BentoService bundle version mismatch: loading BentoServie bundle create with BentoML version 0.5.2,  but loading from BentoML version 0.5.2+173.ga4278d4.dirty
[2020-04-23 23:39:03,559] INFO - BentoService bundle 'Service:20200423233754_0DF217' saved to: /home/bentoml/bentoml/repository/Service/20200423233754_0DF217
In [17]:
print(saved_path)
/home/bentoml/bentoml/repository/Service/20200423233754_0DF217

Serving

Since BERT is a large model, if you met OOM bellow,
you may need to restart this kernel to release the RAM/GRAM used by training model.

In [18]:
bentoml_bundle_path = '/home/bentoml/bentoml/repository/Service/20200423233754_0DF217' # saved_path
In [19]:
from bentoml.utils import detect_free_port
PORT = detect_free_port()
print(PORT)
56717
In [5]:
# Option 1: serve directly
print(f"bentoml serve-gunicorn {bentoml_bundle_path} --port {PORT} --enable-microbatch --workers 1")

!bentoml serve-gunicorn {bentoml_bundle_path} --port {PORT} --enable-microbatch --workers 1
bentoml serve-gunicorn /home/bentoml/bentoml/repository/Service/20200423231933_91D0AB --port 40159 --enable-microbatch --workers 1
In [21]:
# Option 2: serve in docker
!cd {bentoml_bundle_path}
IMG_NAME = bentoml_bundle_path.split('/')[-1].lower()
!docker build -t {IMG_NAME} {bentoml_bundle_path}
# launch docker instances
!docker run -itd -p {PORT}:5000 -e FLAGS="--workers 1 --enable-microbatch" {IMG_NAME}:latest
Sending build context to Docker daemon  457.3MB
Step 1/15 : FROM continuumio/miniconda3:4.7.12
 ---> 406f2b43ea59
Step 2/15 : ENTRYPOINT [ "/bin/bash", "-c" ]
 ---> Using cache
 ---> 72ac38cf396d
Step 3/15 : EXPOSE 5000
 ---> Using cache
 ---> 8475dc08cadd
Step 4/15 : RUN set -x      && apt-get update      && apt-get install --no-install-recommends --no-install-suggests -y libpq-dev build-essential      && rm -rf /var/lib/apt/lists/*
 ---> Using cache
 ---> 0fc16fc9a6ea
Step 5/15 : RUN conda install pip numpy scipy       && pip install gunicorn
 ---> Running in a7a9e19f6122
Collecting package metadata (current_repodata.json): ...working... done
Solving environment: ...working... done

## Package Plan ##

  environment location: /opt/conda

  added / updated specs:
    - numpy
    - pip
    - scipy


The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    blas-1.0                   |              mkl           6 KB
    ca-certificates-2020.1.1   |                0         125 KB
    certifi-2020.4.5.1         |           py37_0         155 KB
    conda-4.8.3                |           py37_0         2.8 MB
    intel-openmp-2020.0        |              166         756 KB
    libgfortran-ng-7.3.0       |       hdf63c60_0        1006 KB
    mkl-2020.0                 |              166       128.9 MB
    mkl-service-2.3.0          |   py37he904b0f_0         218 KB
    mkl_fft-1.0.15             |   py37ha843d7b_0         154 KB
    mkl_random-1.1.0           |   py37hd6b4f25_0         321 KB
    numpy-1.18.1               |   py37h4f9e942_0           5 KB
    numpy-base-1.18.1          |   py37hde5b4d6_1         4.2 MB
    openssl-1.1.1g             |       h7b6447c_0         2.5 MB
    pip-20.0.2                 |           py37_1         1.7 MB
    scipy-1.4.1                |   py37h0b6359f_0        14.5 MB
    ------------------------------------------------------------
                                           Total:       157.3 MB

The following NEW packages will be INSTALLED:

  blas               pkgs/main/linux-64::blas-1.0-mkl
  intel-openmp       pkgs/main/linux-64::intel-openmp-2020.0-166
  libgfortran-ng     pkgs/main/linux-64::libgfortran-ng-7.3.0-hdf63c60_0
  mkl                pkgs/main/linux-64::mkl-2020.0-166
  mkl-service        pkgs/main/linux-64::mkl-service-2.3.0-py37he904b0f_0
  mkl_fft            pkgs/main/linux-64::mkl_fft-1.0.15-py37ha843d7b_0
  mkl_random         pkgs/main/linux-64::mkl_random-1.1.0-py37hd6b4f25_0
  numpy              pkgs/main/linux-64::numpy-1.18.1-py37h4f9e942_0
  numpy-base         pkgs/main/linux-64::numpy-base-1.18.1-py37hde5b4d6_1
  scipy              pkgs/main/linux-64::scipy-1.4.1-py37h0b6359f_0

The following packages will be UPDATED:

  ca-certificates                               2019.8.28-0 --> 2020.1.1-0
  certifi                                  2019.9.11-py37_0 --> 2020.4.5.1-py37_0
  conda                                       4.7.12-py37_0 --> 4.8.3-py37_0
  openssl                                 1.1.1d-h7b6447c_2 --> 1.1.1g-h7b6447c_0
  pip                                         19.2.3-py37_0 --> 20.0.2-py37_1


Proceed ([y]/n)? 

Downloading and Extracting Packages
pip-20.0.2           | 1.7 MB    | ########## | 100% 
openssl-1.1.1g       | 2.5 MB    | ########## | 100% 
mkl-service-2.3.0    | 218 KB    | ########## | 100% 
intel-openmp-2020.0  | 756 KB    | ########## | 100% 
mkl-2020.0           | 128.9 MB  | ########## | 100% 
libgfortran-ng-7.3.0 | 1006 KB   | ########## | 100% 
numpy-base-1.18.1    | 4.2 MB    | ########## | 100% 
numpy-1.18.1         | 5 KB      | ########## | 100% 
mkl_fft-1.0.15       | 154 KB    | ########## | 100% 
mkl_random-1.1.0     | 321 KB    | ########## | 100% 
ca-certificates-2020 | 125 KB    | ########## | 100% 
certifi-2020.4.5.1   | 155 KB    | ########## | 100% 
scipy-1.4.1          | 14.5 MB   | ########## | 100% 
conda-4.8.3          | 2.8 MB    | ########## | 100% 
blas-1.0             | 6 KB      | ########## | 100% 
Preparing transaction: ...working... done
Verifying transaction: ...working... done
Executing transaction: ...working... done
Collecting gunicorn
  Downloading gunicorn-20.0.4-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools>=3.0 in /opt/conda/lib/python3.7/site-packages (from gunicorn) (41.4.0)
Installing collected packages: gunicorn
Successfully installed gunicorn-20.0.4
Removing intermediate container a7a9e19f6122
 ---> b2dc0adaeadb
Step 6/15 : COPY . /bento
 ---> d7b975c97702
Step 7/15 : WORKDIR /bento
 ---> Running in 2c4d019b96d4
Removing intermediate container 2c4d019b96d4
 ---> 226cff48a45e
Step 8/15 : RUN if [ -f /bento/setup.sh ]; then /bin/bash -c /bento/setup.sh; fi
 ---> Running in de65a1259575
Removing intermediate container de65a1259575
 ---> 2131a1a56473
Step 9/15 : RUN conda env update -n base -f /bento/environment.yml
 ---> Running in ac705abc8f5e
Collecting package metadata (repodata.json): ...working... done
Solving environment: ...working... done

Downloading and Extracting Packages
pyopenssl-19.1.0     | 87 KB     | ########## | 100% 
wheel-0.34.2         | 51 KB     | ########## | 100% 
idna-2.9             | 49 KB     | ########## | 100% 
tqdm-4.36.1          | 50 KB     | ########## | 100% 
pysocks-1.7.1        | 30 KB     | ########## | 100% 
pycosat-0.6.3        | 82 KB     | ########## | 100% 
pycparser-2.20       | 92 KB     | ########## | 100% 
ld_impl_linux-64-2.3 | 568 KB    | ########## | 100% 
chardet-3.0.4        | 180 KB    | ########## | 100% 
cffi-1.14.0          | 223 KB    | ########## | 100% 
asn1crypto-1.3.0     | 164 KB    | ########## | 100% 
conda-4.8.3          | 2.8 MB    | ########## | 100% 
cryptography-2.8     | 552 KB    | ########## | 100% 
sqlite-3.31.1        | 1.1 MB    | ########## | 100% 
setuptools-46.1.3    | 513 KB    | ########## | 100% 
ruamel_yaml-0.15.87  | 257 KB    | ########## | 100% 
certifi-2020.4.5.1   | 155 KB    | ########## | 100% 
urllib3-1.25.8       | 169 KB    | ########## | 100% 
scipy-1.2.1          | 13.7 MB   | ########## | 100% 
six-1.14.0           | 27 KB     | ########## | 100% 
requests-2.23.0      | 91 KB     | ########## | 100% 
numpy-1.14.2         | 3.2 MB    | ########## | 100% 
conda-package-handli | 795 KB    | ########## | 100% 
pip-20.0.2           | 1.7 MB    | ########## | 100% 
python-3.6.10        | 29.7 MB   | ########## | 100% 
Preparing transaction: ...working... done
Verifying transaction: ...working... done
Executing transaction: ...working... done
#
# To activate this environment, use
#
#     $ conda activate base
#
# To deactivate an active environment, use
#
#     $ conda deactivate

Removing intermediate container ac705abc8f5e
 ---> d013d1cc37d3
Step 10/15 : ARG PIP_TRUSTED_HOST
 ---> Running in 8d1c29741c56
Removing intermediate container 8d1c29741c56
 ---> aec02ee008bd
Step 11/15 : ARG PIP_INDEX_URL
 ---> Running in d07477dc3ed6
Removing intermediate container d07477dc3ed6
 ---> 1e3ecadeccd5
Step 12/15 : RUN pip install -r /bento/requirements.txt
 ---> Running in 45ef9a05351b
Collecting bentoml==0.5.2
  Downloading BentoML-0.5.2-py3-none-any.whl (521 kB)
Collecting tensorflow
  Downloading tensorflow-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl (421.8 MB)
Collecting bert-for-tf2
  Downloading bert-for-tf2-0.14.4.tar.gz (40 kB)
Collecting cerberus
  Downloading Cerberus-1.3.2.tar.gz (52 kB)
Collecting configparser
  Downloading configparser-5.0.0-py3-none-any.whl (22 kB)
Collecting sqlalchemy>=1.3.0
  Downloading SQLAlchemy-1.3.16-cp36-cp36m-manylinux2010_x86_64.whl (1.2 MB)
Collecting pandas
  Downloading pandas-1.0.3-cp36-cp36m-manylinux1_x86_64.whl (10.0 MB)
Collecting packaging
  Downloading packaging-20.3-py2.py3-none-any.whl (37 kB)
Collecting humanfriendly
  Downloading humanfriendly-8.2-py2.py3-none-any.whl (86 kB)
Collecting protobuf>=3.6.0
  Downloading protobuf-3.11.3-cp36-cp36m-manylinux1_x86_64.whl (1.3 MB)
Collecting python-json-logger
  Downloading python-json-logger-0.1.11.tar.gz (6.0 kB)
Collecting alembic
  Downloading alembic-1.4.2.tar.gz (1.1 MB)
  Installing build dependencies: started
  Installing build dependencies: finished with status 'done'
  Getting requirements to build wheel: started
  Getting requirements to build wheel: finished with status 'done'
    Preparing wheel metadata: started
    Preparing wheel metadata: finished with status 'done'
Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from bentoml==0.5.2->-r /bento/requirements.txt (line 1)) (1.14.0)
Collecting ruamel.yaml>=0.15.0
  Downloading ruamel.yaml-0.16.10-py2.py3-none-any.whl (111 kB)
Collecting gunicorn
  Using cached gunicorn-20.0.4-py2.py3-none-any.whl (77 kB)
Collecting tabulate
  Downloading tabulate-0.8.7-py3-none-any.whl (24 kB)
Collecting grpcio
  Downloading grpcio-1.28.1-cp36-cp36m-manylinux2010_x86_64.whl (2.8 MB)
Collecting click>=7.0
  Downloading click-7.1.1-py2.py3-none-any.whl (82 kB)
Collecting flask
  Downloading Flask-1.1.2-py2.py3-none-any.whl (94 kB)
Collecting prometheus-client
  Downloading prometheus_client-0.7.1.tar.gz (38 kB)
Collecting pathlib2
  Downloading pathlib2-2.3.5-py2.py3-none-any.whl (18 kB)
Collecting boto3
  Downloading boto3-1.12.44-py2.py3-none-any.whl (128 kB)
Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from bentoml==0.5.2->-r /bento/requirements.txt (line 1)) (2.23.0)
Collecting docker
  Downloading docker-4.2.0-py2.py3-none-any.whl (143 kB)
Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from bentoml==0.5.2->-r /bento/requirements.txt (line 1)) (1.14.2)
Collecting termcolor>=1.1.0
  Downloading termcolor-1.1.0.tar.gz (3.9 kB)
Collecting tensorflow-estimator<2.2.0,>=2.1.0rc0
  Downloading tensorflow_estimator-2.1.0-py2.py3-none-any.whl (448 kB)
Collecting absl-py>=0.7.0
  Downloading absl-py-0.9.0.tar.gz (104 kB)
Collecting keras-applications>=1.0.8
  Downloading Keras_Applications-1.0.8-py3-none-any.whl (50 kB)
Collecting google-pasta>=0.1.6
  Downloading google_pasta-0.2.0-py3-none-any.whl (57 kB)
Collecting opt-einsum>=2.3.2
  Downloading opt_einsum-3.2.1-py3-none-any.whl (63 kB)
Collecting astor>=0.6.0
  Downloading astor-0.8.1-py2.py3-none-any.whl (27 kB)
Collecting tensorboard<2.2.0,>=2.1.0
  Downloading tensorboard-2.1.1-py3-none-any.whl (3.8 MB)
Collecting gast==0.2.2
  Downloading gast-0.2.2.tar.gz (10 kB)
Collecting keras-preprocessing>=1.1.0
  Downloading Keras_Preprocessing-1.1.0-py2.py3-none-any.whl (41 kB)
Requirement already satisfied: wheel>=0.26; python_version >= "3" in /opt/conda/lib/python3.6/site-packages (from tensorflow->-r /bento/requirements.txt (line 2)) (0.34.2)
Collecting wrapt>=1.11.1
  Downloading wrapt-1.12.1.tar.gz (27 kB)
Collecting scipy==1.4.1; python_version >= "3"
  Downloading scipy-1.4.1-cp36-cp36m-manylinux1_x86_64.whl (26.1 MB)
Collecting py-params>=0.9.6
  Downloading py-params-0.9.7.tar.gz (6.8 kB)
Collecting params-flow>=0.8.0
  Downloading params-flow-0.8.1.tar.gz (19 kB)
Requirement already satisfied: setuptools in /opt/conda/lib/python3.6/site-packages (from cerberus->bentoml==0.5.2->-r /bento/requirements.txt (line 1)) (46.1.3.post20200330)
Collecting python-dateutil>=2.6.1
  Downloading python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2017.2
  Downloading pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting pyparsing>=2.0.2
  Downloading pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting python-editor>=0.3
  Downloading python_editor-1.0.4-py3-none-any.whl (4.9 kB)
Collecting Mako
  Downloading Mako-1.1.2-py2.py3-none-any.whl (75 kB)
Collecting ruamel.yaml.clib>=0.1.2; platform_python_implementation == "CPython" and python_version < "3.9"
  Downloading ruamel.yaml.clib-0.2.0-cp36-cp36m-manylinux1_x86_64.whl (548 kB)
Collecting itsdangerous>=0.24
  Downloading itsdangerous-1.1.0-py2.py3-none-any.whl (16 kB)
Collecting Jinja2>=2.10.1
  Downloading Jinja2-2.11.2-py2.py3-none-any.whl (125 kB)
Collecting Werkzeug>=0.15
  Downloading Werkzeug-1.0.1-py2.py3-none-any.whl (298 kB)
Collecting s3transfer<0.4.0,>=0.3.0
  Downloading s3transfer-0.3.3-py2.py3-none-any.whl (69 kB)
Collecting jmespath<1.0.0,>=0.7.1
  Downloading jmespath-0.9.5-py2.py3-none-any.whl (24 kB)
Collecting botocore<1.16.0,>=1.15.44
  Downloading botocore-1.15.44-py2.py3-none-any.whl (6.1 MB)
Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->bentoml==0.5.2->-r /bento/requirements.txt (line 1)) (2020.4.5.1)
Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->bentoml==0.5.2->-r /bento/requirements.txt (line 1)) (3.0.4)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->bentoml==0.5.2->-r /bento/requirements.txt (line 1)) (1.25.8)
Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->bentoml==0.5.2->-r /bento/requirements.txt (line 1)) (2.9)
Collecting websocket-client>=0.32.0
  Downloading websocket_client-0.57.0-py2.py3-none-any.whl (200 kB)
Collecting h5py
  Downloading h5py-2.10.0-cp36-cp36m-manylinux1_x86_64.whl (2.9 MB)
Collecting google-auth-oauthlib<0.5,>=0.4.1
  Downloading google_auth_oauthlib-0.4.1-py2.py3-none-any.whl (18 kB)
Collecting markdown>=2.6.8
  Downloading Markdown-3.2.1-py2.py3-none-any.whl (88 kB)
Collecting google-auth<2,>=1.6.3
  Downloading google_auth-1.14.1-py2.py3-none-any.whl (89 kB)
Requirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from params-flow>=0.8.0->bert-for-tf2->-r /bento/requirements.txt (line 3)) (4.36.1)
Collecting MarkupSafe>=0.9.2
  Downloading MarkupSafe-1.1.1-cp36-cp36m-manylinux1_x86_64.whl (27 kB)
Collecting docutils<0.16,>=0.10
  Downloading docutils-0.15.2-py3-none-any.whl (547 kB)
Collecting requests-oauthlib>=0.7.0
  Downloading requests_oauthlib-1.3.0-py2.py3-none-any.whl (23 kB)
Collecting rsa<4.1,>=3.1.4
  Downloading rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.2.1
  Downloading pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting cachetools<5.0,>=2.0.0
  Downloading cachetools-4.1.0-py3-none-any.whl (10 kB)
Collecting oauthlib>=3.0.0
  Downloading oauthlib-3.1.0-py2.py3-none-any.whl (147 kB)
Collecting pyasn1>=0.1.3
  Downloading pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Building wheels for collected packages: bert-for-tf2, cerberus, python-json-logger, alembic, prometheus-client, termcolor, absl-py, gast, wrapt, py-params, params-flow
  Building wheel for bert-for-tf2 (setup.py): started
  Building wheel for bert-for-tf2 (setup.py): finished with status 'done'
  Created wheel for bert-for-tf2: filename=bert_for_tf2-0.14.4-py3-none-any.whl size=30114 sha256=dc03ab4d0523c4fe6fd705d049c29c8d32f43d76e7dcb04b7b5d90bf30de45b6
  Stored in directory: /root/.cache/pip/wheels/7c/b5/ea/3e768502fa1032a6bfefcd0e7dd95e2110e026bf3c584940cd
  Building wheel for cerberus (setup.py): started
  Building wheel for cerberus (setup.py): finished with status 'done'
  Created wheel for cerberus: filename=Cerberus-1.3.2-py3-none-any.whl size=54335 sha256=02b4989a972ade5651d73b1829523f72b18acbc067807c830988a548676f87c1
  Stored in directory: /root/.cache/pip/wheels/57/73/88/653dda2eaa03ee38e7ab10f48ddaadd192fadda426bb5c0bdf
  Building wheel for python-json-logger (setup.py): started
  Building wheel for python-json-logger (setup.py): finished with status 'done'
  Created wheel for python-json-logger: filename=python_json_logger-0.1.11-py2.py3-none-any.whl size=5075 sha256=68f49ece191d9b329114ebd73409ac94a4dc87d761714945870ea1f60b02ac8c
  Stored in directory: /root/.cache/pip/wheels/99/ad/da/22b0dbe95e769350f2641f2f18c25d3ba3a0ce21dd1f655cf1
  Building wheel for alembic (PEP 517): started
  Building wheel for alembic (PEP 517): finished with status 'done'
  Created wheel for alembic: filename=alembic-1.4.2-py2.py3-none-any.whl size=159543 sha256=10b75c5c9a5b956bdd25a8266bb440ed53a3802ed3e1237319f50421e22dde4f
  Stored in directory: /root/.cache/pip/wheels/f2/50/61/5cc491b0ca39be60dfb4dce940b389ff91b847d62e0eb2d680
  Building wheel for prometheus-client (setup.py): started
  Building wheel for prometheus-client (setup.py): finished with status 'done'
  Created wheel for prometheus-client: filename=prometheus_client-0.7.1-py3-none-any.whl size=41404 sha256=38114b16991bc0f88fffb229964893171d0b34f7dfc1e9640d3d2fa11d9e7841
  Stored in directory: /root/.cache/pip/wheels/1d/4a/79/a3ad3f74b3495b4555359375ca33ad7b64e77f8b7a53c8894f
  Building wheel for termcolor (setup.py): started
  Building wheel for termcolor (setup.py): finished with status 'done'
  Created wheel for termcolor: filename=termcolor-1.1.0-py3-none-any.whl size=4830 sha256=269c12e7350d6782ce935e2cb9d040d33095765fab41350f37252eca12c36d55
  Stored in directory: /root/.cache/pip/wheels/93/2a/eb/e58dbcbc963549ee4f065ff80a59f274cc7210b6eab962acdc
  Building wheel for absl-py (setup.py): started
  Building wheel for absl-py (setup.py): finished with status 'done'
  Created wheel for absl-py: filename=absl_py-0.9.0-py3-none-any.whl size=121931 sha256=462508fc23e26d38a985b2a955ff83dea24859ffc371ae2d640ea2b565df0847
  Stored in directory: /root/.cache/pip/wheels/c3/af/84/3962a6af7b4ab336e951b7877dcfb758cf94548bb1771e0679
  Building wheel for gast (setup.py): started
  Building wheel for gast (setup.py): finished with status 'done'
  Created wheel for gast: filename=gast-0.2.2-py3-none-any.whl size=7539 sha256=bc6fb1480664fa5588adfd03da22efbbd0bde3c7b51a07a1b2207b4dc611cb4d
  Stored in directory: /root/.cache/pip/wheels/19/a7/b9/0740c7a3a7d1d348f04823339274b90de25fbcd217b2ee1fbe
  Building wheel for wrapt (setup.py): started
  Building wheel for wrapt (setup.py): finished with status 'done'
  Created wheel for wrapt: filename=wrapt-1.12.1-cp36-cp36m-linux_x86_64.whl size=75208 sha256=9ee1a5fa6c8e3fe3471f5d4c194588f4d9681643f97b46aa18455011fbc23083
  Stored in directory: /root/.cache/pip/wheels/32/42/7f/23cae9ff6ef66798d00dc5d659088e57dbba01566f6c60db63
  Building wheel for py-params (setup.py): started
  Building wheel for py-params (setup.py): finished with status 'done'
  Created wheel for py-params: filename=py_params-0.9.7-py3-none-any.whl size=7302 sha256=ab548fc73ca0a33b7d43024f0c13caced381d57d2262066098cf2c3df6acf3b3
  Stored in directory: /root/.cache/pip/wheels/18/e2/80/244168a554fc8469897bb86b3ea7183b56ecc92f484e67b375
  Building wheel for params-flow (setup.py): started
  Building wheel for params-flow (setup.py): finished with status 'done'
  Created wheel for params-flow: filename=params_flow-0.8.1-py3-none-any.whl size=16121 sha256=0e444e7ee4c9cb0467404383187929aa838cbdb63b896d151ae672c3277f5254
  Stored in directory: /root/.cache/pip/wheels/cc/70/6f/98e4cd9239940e8160866872af2be0ca038cdfbdc8113db2ad
Successfully built bert-for-tf2 cerberus python-json-logger alembic prometheus-client termcolor absl-py gast wrapt py-params params-flow
ERROR: tensorflow 2.1.0 has requirement numpy<2.0,>=1.16.0, but you'll have numpy 1.14.2 which is incompatible.
Installing collected packages: cerberus, configparser, sqlalchemy, python-dateutil, pytz, pandas, pyparsing, packaging, humanfriendly, protobuf, python-json-logger, python-editor, MarkupSafe, Mako, alembic, ruamel.yaml.clib, ruamel.yaml, gunicorn, tabulate, grpcio, click, itsdangerous, Jinja2, Werkzeug, flask, prometheus-client, pathlib2, docutils, jmespath, botocore, s3transfer, boto3, websocket-client, docker, bentoml, termcolor, tensorflow-estimator, absl-py, h5py, keras-applications, google-pasta, opt-einsum, astor, pyasn1, rsa, pyasn1-modules, cachetools, google-auth, oauthlib, requests-oauthlib, google-auth-oauthlib, markdown, tensorboard, gast, keras-preprocessing, wrapt, scipy, tensorflow, py-params, params-flow, bert-for-tf2
  Attempting uninstall: scipy
    Found existing installation: scipy 1.2.1
    Uninstalling scipy-1.2.1:
      Successfully uninstalled scipy-1.2.1
Successfully installed Jinja2-2.11.2 Mako-1.1.2 MarkupSafe-1.1.1 Werkzeug-1.0.1 absl-py-0.9.0 alembic-1.4.2 astor-0.8.1 bentoml-0.5.2 bert-for-tf2-0.14.4 boto3-1.12.44 botocore-1.15.44 cachetools-4.1.0 cerberus-1.3.2 click-7.1.1 configparser-5.0.0 docker-4.2.0 docutils-0.15.2 flask-1.1.2 gast-0.2.2 google-auth-1.14.1 google-auth-oauthlib-0.4.1 google-pasta-0.2.0 grpcio-1.28.1 gunicorn-20.0.4 h5py-2.10.0 humanfriendly-8.2 itsdangerous-1.1.0 jmespath-0.9.5 keras-applications-1.0.8 keras-preprocessing-1.1.0 markdown-3.2.1 oauthlib-3.1.0 opt-einsum-3.2.1 packaging-20.3 pandas-1.0.3 params-flow-0.8.1 pathlib2-2.3.5 prometheus-client-0.7.1 protobuf-3.11.3 py-params-0.9.7 pyasn1-0.4.8 pyasn1-modules-0.2.8 pyparsing-2.4.7 python-dateutil-2.8.1 python-editor-1.0.4 python-json-logger-0.1.11 pytz-2019.3 requests-oauthlib-1.3.0 rsa-4.0 ruamel.yaml-0.16.10 ruamel.yaml.clib-0.2.0 s3transfer-0.3.3 scipy-1.4.1 sqlalchemy-1.3.16 tabulate-0.8.7 tensorboard-2.1.1 tensorflow-2.1.0 tensorflow-estimator-2.1.0 termcolor-1.1.0 websocket-client-0.57.0 wrapt-1.12.1
Removing intermediate container 45ef9a05351b
 ---> 4c095636c56f
Step 13/15 : RUN if [ -f /bento/bentoml_init.sh ]; then /bin/bash -c /bento/bentoml_init.sh; fi
 ---> Running in b1448dd3093d
Processing ./bundled_pip_dependencies/BentoML-0.5.2+173.ga4278d4.dirty.tar.gz
  Installing build dependencies: started
  Installing build dependencies: finished with status 'done'
  Getting requirements to build wheel: started
  Getting requirements to build wheel: finished with status 'done'
    Preparing wheel metadata: started
    Preparing wheel metadata: finished with status 'done'
Requirement already satisfied, skipping upgrade: protobuf>=3.6.0 in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (3.11.3)
Collecting contextvars; python_version < "3.7"
  Downloading contextvars-2.4.tar.gz (9.6 kB)
Requirement already satisfied, skipping upgrade: numpy in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (1.14.2)
Requirement already satisfied, skipping upgrade: humanfriendly in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (8.2)
Requirement already satisfied, skipping upgrade: boto3 in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (1.12.44)
Requirement already satisfied, skipping upgrade: sqlalchemy>=1.3.0 in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (1.3.16)
Collecting sqlalchemy-utils
  Downloading SQLAlchemy-Utils-0.36.3.tar.gz (128 kB)
Requirement already satisfied, skipping upgrade: docker in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (4.2.0)
Collecting grpcio<=1.27.2
  Downloading grpcio-1.27.2-cp36-cp36m-manylinux2010_x86_64.whl (2.7 MB)
Requirement already satisfied, skipping upgrade: python-json-logger in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (0.1.11)
Collecting aiohttp
  Downloading aiohttp-3.6.2-cp36-cp36m-manylinux1_x86_64.whl (1.2 MB)
Requirement already satisfied, skipping upgrade: gunicorn in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (20.0.4)
Requirement already satisfied, skipping upgrade: tabulate in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (0.8.7)
Requirement already satisfied, skipping upgrade: alembic in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (1.4.2)
Collecting py-zipkin
  Downloading py_zipkin-0.20.0-py2.py3-none-any.whl (52 kB)
Collecting python-dateutil<2.8.1,>=2.1
  Downloading python_dateutil-2.8.0-py2.py3-none-any.whl (226 kB)
Requirement already satisfied, skipping upgrade: click>=7.0 in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (7.1.1)
Requirement already satisfied, skipping upgrade: prometheus-client in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (0.7.1)
Requirement already satisfied, skipping upgrade: packaging in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (20.3)
Requirement already satisfied, skipping upgrade: requests in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (2.23.0)
Requirement already satisfied, skipping upgrade: configparser in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (5.0.0)
Requirement already satisfied, skipping upgrade: flask in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (1.1.2)
Requirement already satisfied, skipping upgrade: pandas in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (1.0.3)
Requirement already satisfied, skipping upgrade: ruamel.yaml>=0.15.0 in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (0.16.10)
Requirement already satisfied, skipping upgrade: cerberus in /opt/conda/lib/python3.6/site-packages (from BentoML==0.5.2+173.ga4278d4.dirty) (1.3.2)
Requirement already satisfied, skipping upgrade: six>=1.9 in /opt/conda/lib/python3.6/site-packages (from protobuf>=3.6.0->BentoML==0.5.2+173.ga4278d4.dirty) (1.14.0)
Requirement already satisfied, skipping upgrade: setuptools in /opt/conda/lib/python3.6/site-packages (from protobuf>=3.6.0->BentoML==0.5.2+173.ga4278d4.dirty) (46.1.3.post20200330)
Collecting immutables>=0.9
  Downloading immutables-0.12-cp36-cp36m-manylinux1_x86_64.whl (97 kB)
Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /opt/conda/lib/python3.6/site-packages (from boto3->BentoML==0.5.2+173.ga4278d4.dirty) (0.9.5)
Requirement already satisfied, skipping upgrade: botocore<1.16.0,>=1.15.44 in /opt/conda/lib/python3.6/site-packages (from boto3->BentoML==0.5.2+173.ga4278d4.dirty) (1.15.44)
Requirement already satisfied, skipping upgrade: s3transfer<0.4.0,>=0.3.0 in /opt/conda/lib/python3.6/site-packages (from boto3->BentoML==0.5.2+173.ga4278d4.dirty) (0.3.3)
Requirement already satisfied, skipping upgrade: websocket-client>=0.32.0 in /opt/conda/lib/python3.6/site-packages (from docker->BentoML==0.5.2+173.ga4278d4.dirty) (0.57.0)
Collecting async-timeout<4.0,>=3.0
  Downloading async_timeout-3.0.1-py3-none-any.whl (8.2 kB)
Collecting idna-ssl>=1.0; python_version < "3.7"
  Downloading idna-ssl-1.1.0.tar.gz (3.4 kB)
Requirement already satisfied, skipping upgrade: chardet<4.0,>=2.0 in /opt/conda/lib/python3.6/site-packages (from aiohttp->BentoML==0.5.2+173.ga4278d4.dirty) (3.0.4)
Collecting attrs>=17.3.0
  Downloading attrs-19.3.0-py2.py3-none-any.whl (39 kB)
Collecting typing-extensions>=3.6.5; python_version < "3.7"
  Downloading typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Collecting yarl<2.0,>=1.0
  Downloading yarl-1.4.2-cp36-cp36m-manylinux1_x86_64.whl (252 kB)
Collecting multidict<5.0,>=4.5
  Downloading multidict-4.7.5-cp36-cp36m-manylinux1_x86_64.whl (148 kB)
Requirement already satisfied, skipping upgrade: Mako in /opt/conda/lib/python3.6/site-packages (from alembic->BentoML==0.5.2+173.ga4278d4.dirty) (1.1.2)
Requirement already satisfied, skipping upgrade: python-editor>=0.3 in /opt/conda/lib/python3.6/site-packages (from alembic->BentoML==0.5.2+173.ga4278d4.dirty) (1.0.4)
Collecting thriftpy2>=0.4.0
  Downloading thriftpy2-0.4.11.tar.gz (498 kB)
Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /opt/conda/lib/python3.6/site-packages (from packaging->BentoML==0.5.2+173.ga4278d4.dirty) (2.4.7)
Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->BentoML==0.5.2+173.ga4278d4.dirty) (2.9)
Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->BentoML==0.5.2+173.ga4278d4.dirty) (2020.4.5.1)
Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->BentoML==0.5.2+173.ga4278d4.dirty) (1.25.8)
Requirement already satisfied, skipping upgrade: itsdangerous>=0.24 in /opt/conda/lib/python3.6/site-packages (from flask->BentoML==0.5.2+173.ga4278d4.dirty) (1.1.0)
Requirement already satisfied, skipping upgrade: Werkzeug>=0.15 in /opt/conda/lib/python3.6/site-packages (from flask->BentoML==0.5.2+173.ga4278d4.dirty) (1.0.1)
Requirement already satisfied, skipping upgrade: Jinja2>=2.10.1 in /opt/conda/lib/python3.6/site-packages (from flask->BentoML==0.5.2+173.ga4278d4.dirty) (2.11.2)
Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /opt/conda/lib/python3.6/site-packages (from pandas->BentoML==0.5.2+173.ga4278d4.dirty) (2019.3)
Requirement already satisfied, skipping upgrade: ruamel.yaml.clib>=0.1.2; platform_python_implementation == "CPython" and python_version < "3.9" in /opt/conda/lib/python3.6/site-packages (from ruamel.yaml>=0.15.0->BentoML==0.5.2+173.ga4278d4.dirty) (0.2.0)
Requirement already satisfied, skipping upgrade: docutils<0.16,>=0.10 in /opt/conda/lib/python3.6/site-packages (from botocore<1.16.0,>=1.15.44->boto3->BentoML==0.5.2+173.ga4278d4.dirty) (0.15.2)
Requirement already satisfied, skipping upgrade: MarkupSafe>=0.9.2 in /opt/conda/lib/python3.6/site-packages (from Mako->alembic->BentoML==0.5.2+173.ga4278d4.dirty) (1.1.1)
Collecting ply<4.0,>=3.4
  Downloading ply-3.11-py2.py3-none-any.whl (49 kB)
Building wheels for collected packages: BentoML, contextvars, sqlalchemy-utils, idna-ssl, thriftpy2
  Building wheel for BentoML (PEP 517): started
  Building wheel for BentoML (PEP 517): finished with status 'done'
  Created wheel for BentoML: filename=BentoML-0.5.2+173.ga4278d4.dirty-py3-none-any.whl size=553995 sha256=f98b2b46504c4955466088ddb2e000ac0486d403cbaf50ad3784b293895b5f4c
  Stored in directory: /root/.cache/pip/wheels/94/a6/ab/42dcd1c63a9024fdeb1825d3f6345b71c927dff77c2f106814
  Building wheel for contextvars (setup.py): started
  Building wheel for contextvars (setup.py): finished with status 'done'
  Created wheel for contextvars: filename=contextvars-2.4-py3-none-any.whl size=7664 sha256=0be4f58266e970c71a94bd89439691f95ad0b79016b7410f86844f1abc9c7a19
  Stored in directory: /root/.cache/pip/wheels/41/11/53/911724983aa48deb94792432e14e518447212dd6c5477d49d3
  Building wheel for sqlalchemy-utils (setup.py): started
  Building wheel for sqlalchemy-utils (setup.py): finished with status 'done'
  Created wheel for sqlalchemy-utils: filename=SQLAlchemy_Utils-0.36.3-py2.py3-none-any.whl size=89208 sha256=8165ac0a571cf1589269320cb65421ec7f41a9c4ad38c543b35785b094c2bdf1
  Stored in directory: /root/.cache/pip/wheels/e2/72/f1/5c9586f19dcdf1b034fc9a3184c689ae234c7fbda5b971a3e1
  Building wheel for idna-ssl (setup.py): started
  Building wheel for idna-ssl (setup.py): finished with status 'done'
  Created wheel for idna-ssl: filename=idna_ssl-1.1.0-py3-none-any.whl size=3161 sha256=0f95965c567674975d646fe55f22fc0bfaa6c35a4f95040aa199cb311371ea06
  Stored in directory: /root/.cache/pip/wheels/6a/f5/9c/f8331a854f7a8739cf0e74c13854e4dd7b1af11b04fe1dde13
  Building wheel for thriftpy2 (setup.py): started
  Building wheel for thriftpy2 (setup.py): finished with status 'done'
  Created wheel for thriftpy2: filename=thriftpy2-0.4.11-cp36-cp36m-linux_x86_64.whl size=1070008 sha256=e08353b91e469b33a4d4912de88de2e5ffd8488d89e5fa932cb5717f4c0ed9ed
  Stored in directory: /root/.cache/pip/wheels/64/c6/25/269aea238ed60341cdca72c4a4efc161345553f50cd96fa6ee
Successfully built BentoML contextvars sqlalchemy-utils idna-ssl thriftpy2
ERROR: tensorflow 2.1.0 has requirement numpy<2.0,>=1.16.0, but you'll have numpy 1.14.2 which is incompatible.
Installing collected packages: immutables, contextvars, sqlalchemy-utils, grpcio, async-timeout, idna-ssl, attrs, typing-extensions, multidict, yarl, aiohttp, ply, thriftpy2, py-zipkin, python-dateutil, BentoML
  Attempting uninstall: grpcio
    Found existing installation: grpcio 1.28.1
    Uninstalling grpcio-1.28.1:
      Successfully uninstalled grpcio-1.28.1
  Attempting uninstall: python-dateutil
    Found existing installation: python-dateutil 2.8.1
    Uninstalling python-dateutil-2.8.1:
      Successfully uninstalled python-dateutil-2.8.1
  Attempting uninstall: BentoML
    Found existing installation: BentoML 0.5.2
    Uninstalling BentoML-0.5.2:
      Successfully uninstalled BentoML-0.5.2
Successfully installed BentoML-0.5.2+173.ga4278d4.dirty aiohttp-3.6.2 async-timeout-3.0.1 attrs-19.3.0 contextvars-2.4 grpcio-1.27.2 idna-ssl-1.1.0 immutables-0.12 multidict-4.7.5 ply-3.11 py-zipkin-0.20.0 python-dateutil-2.8.0 sqlalchemy-utils-0.36.3 thriftpy2-0.4.11 typing-extensions-3.7.4.2 yarl-1.4.2
Removing intermediate container b1448dd3093d
 ---> c153392c14eb
Step 14/15 : ENV FLAGS=""
 ---> Running in 8c3b0d4a59ac
Removing intermediate container 8c3b0d4a59ac
 ---> 17b4b7e8749e
Step 15/15 : CMD ["bentoml serve-gunicorn /bento $FLAGS"]
 ---> Running in fc56a9768af2
Removing intermediate container fc56a9768af2
 ---> 0f14aba82b37
Successfully built 0f14aba82b37
Successfully tagged 20200423233754_0df217:latest
08d8ad1431f904928c96f281a3d12c6701aa060763b7fe418a8192afac97aebe

Test the API with requests

In [24]:
%%time
import requests
import pandas as pd

server_url = f"http://127.0.0.1:{PORT}/predict"
method = "POST"
headers = {"content-type": "application/json"}
pred_sentences =  ["The acting was a bit lacking."]
data = pd.DataFrame(pred_sentences).to_json()

r = requests.request(method, server_url, headers=headers, data=data)
print(r.content)
b'["negative"]'
CPU times: user 4.14 ms, sys: 3.43 ms, total: 7.57 ms
Wall time: 214 ms
In [ ]: