Cosine Similarity (Version B)¶

# Imports

import numpy
print("numpy:   " + numpy.version.version)

import sklearn
import sklearn.metrics
print("sklearn: " + sklearn.__version__)

# Class instance to access data (wp texts, pre-computed embeddings)
import data_access
data_accessor = data_access.DataAccess(source_texts_directory, embeddings_directory)

# Load embeddings
embeddings_british_a = data_accessor.load_embeddings(id_british_a)
embeddings_british_b = data_accessor.load_embeddings(id_british_b)
print()

# Compute means
def get_mean(embeddings, note = "", printinfo = True):
    mean = numpy.mean(embeddings, axis=0)
    if printinfo:
        print(str(type(mean)) + " " + str(mean.shape) + " " +  note)
    return mean

mean_british_a  = get_mean(embeddings_british_a, "BritishA")
mean_british_b  = get_mean(embeddings_british_b, "BritishB")

# Differences of arrays as one value
def differenceValue(a, b):
    x = 0
    for i in range(len(a)):
        x += abs(a[i] - b[i])
    return x;
# Test .1 + .2 + .3 + .4 = 1
if True:
    print(differenceValue(numpy.array([1,2,3,0]), numpy.array([1.1,2.2,3.3,-0.4])))


# Compute difference values between embeddings of single texts and mean-embeddings
# Array:
#  [0] index (to look up source texts)
#  [1] difference to mean t1 (== A)
#  [2] difference to mean t2 (== B)
differences = []
for i in range(len(mean_british_a)):
    differences.append((i,   differenceValue(mean_british_a, embeddings_british_a[i]),   differenceValue(mean_british_b, embeddings_british_b[i])))

# Sort by largest difference
largest_differences_a = sorted(differences, key=lambda tup: tup[1], reverse=True)
largest_differences_b = sorted(differences, key=lambda tup: tup[2], reverse=True)


# Explore embeddings of two points of time directly
differences_direct = []
for i in range(len(mean_british_a)):
    differences_direct.append((i, differenceValue(embeddings_british_a[i], embeddings_british_b[i])))

# Sort
largest_differences_direct = sorted(differences_direct, key=lambda tup: tup[1], reverse=True)


# Print source texts
def print_source_text(directory, category_id, index):
    print()
    print("Category: " + category_id)
    print("Index:    " + str(index))
    file = data_accessor.get_embeddings_dict_filename(category_id, index);
    print("File:     ")
    print(data_accessor.read_source_text(directory, file))
    print()

print("Smallest cosine similarity to mean of A")
print(smallest_similarities_a[0])
print(smallest_similarities_a[1])
print(smallest_similarities_a[2])
print("...")
print(smallest_similarities_a[len(smallest_similarities_a)-2])
print(smallest_similarities_a[len(smallest_similarities_a)-1])
print()

print("Largest difference values to mean of A")
print(largest_differences_a[0])
print(largest_differences_a[1])
print(largest_differences_a[2])
print("...")
print(largest_differences_a[len(largest_differences_a)-2])
print(largest_differences_a[len(largest_differences_a)-1])
print("\n")

print("Smallest cosine similarity to mean of B")
print(smallest_similarities_b[0])
print(smallest_similarities_b[1])
print(smallest_similarities_b[2])
print("...")
print(smallest_similarities_b[len(smallest_similarities_b)-2])
print(smallest_similarities_b[len(smallest_similarities_b)-1])
print()

print("Largest difference values to mean of B")
print(largest_differences_b[0])
print(largest_differences_b[1])
print(largest_differences_b[2])
print("...")
print(largest_differences_b[len(largest_differences_b)-2])
print(largest_differences_b[len(largest_differences_b)-1])
print("\n")

print("Smallest cosine similarity (direct)")
print(smallest_similarities_direct[0])
print(smallest_similarities_direct[1])
print(smallest_similarities_direct[2])
print("...")
print(smallest_similarities_direct[len(smallest_similarities_direct)-2])
print(smallest_similarities_direct[len(smallest_similarities_direct)-1])
print()

print("Largest difference values (direct)")
print(largest_differences_direct[0])
print(largest_differences_direct[1])
print(largest_differences_direct[2])
print("...")
print(largest_differences_direct[len(largest_differences_direct)-2])
print(largest_differences_direct[len(largest_differences_direct)-1])
print()

# Explore underlying texts (Largest difference values to mean of A)
if False:
    print("721: As expected")
    print_source_text(id_british_a, id_british, largest_differences_a[0][0])
    print_source_text(id_british_b, id_british, largest_differences_a[0][0])
if True:
    print("333: Very similar, even if difference to mean of A is large")
    print_source_text(id_british_a, id_british, largest_differences_a[1][0])
    print_source_text(id_british_b, id_british, largest_differences_a[1][0])

# Explore underlying texts (Largest difference values to mean of B)
if True:
    print("179: Very similar, even if difference to mean of B is large")
    print_source_text(id_british_a, id_british, largest_differences_b[0][0])
    print_source_text(id_british_b, id_british, largest_differences_b[0][0])
if False:
    print("680: Very similar, even if difference to mean of B is large")
    print_source_text(id_british_a, id_british, largest_differences_b[1][0])
    print_source_text(id_british_b, id_british, largest_differences_b[1][0])