In [ ]:
# !pip install annoy==1.17.0
# !pip install mtcnn==0.1.0
# !pip install tensorflow==2.3.1
# !pip install keras==2.4.3
# !pip install keras-vggface==0.6
# !pip install keras_applications==1.0.8
# !pip install imutils==0.5.3
# !pip install pip install voila==0.2.3
# !pip install ipywidgets==7.5.1
# !pip install opencv-python==4.4.0.44
# !pip install matplotlib==3.3.2
# !pip install numpy==1.18.5
In [ ]:
import os
import requests
import matplotlib.pyplot as plt
import json
import math
from annoy import AnnoyIndex
from mtcnn.mtcnn import MTCNN
from keras_vggface.vggface import VGGFace
from keras_vggface.utils import preprocess_input
from PIL import Image
from numpy import asarray
import numpy as np
import cv2
import ipywidgets as widgets
from IPython.display import display
import io
import matplotlib.pyplot as plt
import imutils
import pickle
In [ ]:
path0 = ".keras"
os.makedirs(path0, exist_ok=True)
path1 = ".keras/models"
os.makedirs(path1, exist_ok=True)
path2 = ".keras/models/vggface"
os.makedirs(path2, exist_ok=True)

vggface_model_path = ".keras/models/vggface/rcmalli_vggface_tf_notop_resnet50.h5"
if not os.path.exists(vggface_model_path):
    os.system("wget -q https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_notop_resnet50.h5 -O {}".format(vggface_model_path))
In [ ]:
def download_file_from_google_drive(id, destination):
    URL = "https://docs.google.com/uc?export=download"

    session = requests.Session()

    response = session.get(URL, params = { 'id' : id }, stream = True)
    token = get_confirm_token(response)

    if token:
        params = { 'id' : id, 'confirm' : token }
        response = session.get(URL, params = params, stream = True)

    save_response_content(response, destination)    

def get_confirm_token(response):
    for key, value in response.cookies.items():
        if key.startswith('download_warning'):
            return value

    return None

def save_response_content(response, destination):
    CHUNK_SIZE = 32768

    with open(destination, "wb") as f:
        for chunk in response.iter_content(CHUNK_SIZE):
            if chunk: # filter out keep-alive new chunks
                f.write(chunk)

os.makedirs('celeb_images', exist_ok=True)

files_to_download_dict =  {'1wDaaSQ6NjxLkxpzYyTRknefizZUKnKDj': 'celeb_mapping.json', '1-3Wb7fiINbrk9FSagTxjLdSjp7KzrMp7': 'celeb_index_60.ann', '1_c2sOyRmk96WI5eQ_0MWRmgXIrtVRuEe': 'celeb_images/srk.jpg', '1OakbrrVYoCVs1NTpeKcmIg_wtsgh317D': 'celeb_images/sk.jpg', '16o9pwqFOai1oJ6WgeoRj01INGz7HTbcz': 'celeb_images/pc.jpg', '1w5N0XGa1w3kMWobcS7rfKJki01kssMdG': 'celeb_images/obama.jpg', '1p9PJqMN45rlh8GJnghAy_LlROHo6qgAG': 'celeb_images/dp.jpg', '1IXDqYsIIywRX0dq6jprncUBnmPQ3mTXS': 'celeb_images/dp_output.png', '1CYk7iTGBb-5MnOQGK8A-wdNbPXVVjCib': 'celeb_images/pc_output.png', '1eo2RQulBhSsGVmc3btUZwucaH_z0cfgN': 'celeb_images/obama_output.png'}

for gdrive_id, path in files_to_download_dict.items():
    if not os.path.exists(path):
        download_file_from_google_drive(gdrive_id, path)
In [ ]:
with open('celeb_mapping.json') as json_file:
    celeb_mapping_1_temp = json.load(json_file)
celeb_mapping_1 = {}
for key, value_list in celeb_mapping_1_temp.items():
    for each_id in value_list:
        celeb_mapping_1[each_id] = str(key)
In [ ]:
def get_celeb_name_from_id(result_list, dist_threshold=0.9):
    id_list = result_list[0]
    dist_list = result_list[1]
    counts = dict()
    for each_id, each_dist in zip(id_list, dist_list):
        if each_dist < dist_threshold:
            output = celeb_mapping_1.get(each_id)
            counts[output] = counts.get(output, 0) + 1
    return counts
In [ ]:
def face_distance_to_conf(face_distance, face_match_threshold=0.34):
	if face_distance > face_match_threshold:
		range = (1.0 - face_match_threshold)
		linear_val = (1.0 - face_distance) / (range * 2.0)
		return linear_val
	else:
		range = face_match_threshold
		linear_val = 1.0 - (face_distance / (range * 2.0))
		return linear_val + ((1.0 - linear_val) * math.pow((linear_val - 0.5) * 2, 0.2))
In [ ]:
face_detector = MTCNN()
encoder_model = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3), pooling='avg')
ann_index = AnnoyIndex(2048, 'angular')
_ = ann_index.load("celeb_index_60.ann")
In [ ]:
def get_encoding_new(img):
	results = face_detector.detect_faces(img)
	if len(results)>0:
		encodings = []
		bbox = []
		for result in results:
			x1, y1, width, height = result['box']
			if x1 <0:
				x1 = 0
			if y1 <0:
				y1 = 0
			x2, y2 = x1 + width, y1 + height
			face = img[y1:y2, x1:x2]
			image = Image.fromarray(face)
			image = image.resize((224,224))
			face_array = asarray(image)

			samples = asarray(face_array, 'float32')
			samples = preprocess_input(samples, version=2)
			samples = np.expand_dims(samples, axis=0)
			encoding = encoder_model.predict(samples)
			encodings.append(encoding)
			bbox.append((x1, y1, width, height))
		return encodings, bbox
	else:
		return None, None
In [ ]:
def get_celeb_prediction_new_1(img):
  encs, bbox = get_encoding_new(img)
  data = []
  for index, enc in enumerate(encs):
    cv2.rectangle(img, bbox[index], (255,0,0), 2)
    temp_data = {}
    temp_data["bbox"] = bbox[index]
    results = ann_index.get_nns_by_vector(enc[0], 10, search_k=-1, include_distances=True)
    dist_threshold = 0.9
    celeb_count_dict = get_celeb_name_from_id(results, dist_threshold)
    distance = results[1][0]
    if len(celeb_count_dict)!=0 and max(celeb_count_dict.values()) > 3:
        celeb_name = max(celeb_count_dict, key=celeb_count_dict.get)
        cv2.putText(img, celeb_name.upper(), (bbox[index][0]-5, bbox[index][1] - 5), cv2.FONT_HERSHEY_DUPLEX, 1, (0,0,255), 1)
        temp_data["celeb_name"] = celeb_name
        temp_data["confidence"] = face_distance_to_conf(distance)
    else:
        temp_data["celeb_name"] = "unknown"
        temp_data["confidence"] = 0.0
    data.append(temp_data)
  img = imutils.resize(img, width=400)
  return data, img

Celebrity recognition

A model to recognize celebrities in images. Works using a MTCNN face detector to detect faces, and VGGFace for creating encodings. These encodings are matched using a libray called annoy (from spotify). Model is created using a dataset of around 6K images for 60 celebrities (mostly Indian).

Please upload an image to find the closest celebrity matches.

In [ ]:
im = cv2.imread('celeb_images/dp_output.png')
im = imutils.resize(im, width=400)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
im = Image.fromarray(im)
display(im)
In [ ]:
btn_run = widgets.Button(description='Identify celeb')
lbl_pred = widgets.Label()
out_pl = widgets.Output()
In [ ]:
def on_click_classify(change):
    img = cv2.imdecode(np.frombuffer(btn_upload.data[-1], np.uint8), -1)
    pred, img = get_celeb_prediction_new_1(img)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    im_pil = Image.fromarray(img)
    out_pl.clear_output()
    with out_pl: display(im_pil)
    celebs = [c["celeb_name"] for c in pred]
    lbl_pred.value = 'Prediction: {}'.format(celebs)

btn_run.on_click(on_click_classify)
In [ ]:
btn_upload = widgets.FileUpload(multiple=False)
widgets.VBox([btn_upload, btn_run, out_pl, lbl_pred])