# EM for IBM Model 2¶

Following the pseudo code proposed by Mike Collins. Use at your own risk, please report bug.

Author: Yoav Artzi (http://yoavartzi.com).

In [6]:
from collections import defaultdict
import random

# Our (big) data.
data = [(['le', 'chien'], ['the', 'dog']), (['le', 'chat'], ['the', 'cat'])]
f_len = 2
e_len = 2
e_vocab = ['the', 'dog', 'cat']
f_vocab = ['le', 'chien', 'chat']

# Dictionaries to maintain probabilities.
q = defaultdict(float)
t = defaultdict(float)

# Init randomly (cutting corners: init is not normalized!)
for e_word in e_vocab:
for f_word in f_vocab:
t[(f_word, e_word)] = random.random()
for j in range(e_len):
for i in range(f_len):
q[(j,i,e_len,f_len)] = random.random()

# EM loop.
for s in range(1,100):
print '#### Epoch %d' % (s)
# Collect empirical counts.
counts = defaultdict(float)
for f, e in data:
m = len(f)
l = len(e)
for i in range(len(e)):
for j in range(len(f)):
delta_sum = sum([q[(delta_sum_j,i,l,m)] * t[(f[i], e[delta_sum_j])] for delta_sum_j in range(l)])
delta = (q[(j,i,l,m)] * t[(f[i], e[j])]) / delta_sum
counts[(e[i], f[j])] += delta
counts[e[i]] += delta
counts[(j,i,l,m)] += delta
counts[(i,l,m)] += delta
# Recompute probabilities and print.
changed = False
for e_word in e_vocab:
for f_word in f_vocab:
old_value = t[(f_word, e_word)]
t[(f_word, e_word)] = counts[(e_word, f_word)] / counts[e_word]
changed = old_value != t[(f_word, e_word)]
print 't(%s | %s) = %f' % (f_word, e_word, t[(f_word, e_word)])
for j in range(e_len):
for i in range(f_len):
old_value = q[(j,i, e_len, f_len)]
q[(j,i, e_len, f_len)] = counts[(j,i,e_len,f_len)] / counts[(i,e_len, f_len)]
changed = old_value != q[(j,i, e_len, f_len)]
print 'q(%d | %d, %d, %d) = %f' % (j,i,e_len, f_len, q[(j,i, e_len, f_len)])
if not changed:
break

#### Epoch 1
t(le | the) = 0.238651
t(chien | the) = 0.308175
t(chat | the) = 0.453174
t(le | dog) = 0.425076
t(chien | dog) = 0.574924
t(chat | dog) = 0.000000
t(le | cat) = 0.451927
t(chien | cat) = 0.000000
t(chat | cat) = 0.548073
q(0 | 0, 2, 2) = 0.238651
q(0 | 1, 2, 2) = 0.438502
q(1 | 0, 2, 2) = 0.761349
q(1 | 1, 2, 2) = 0.561498
#### Epoch 2
t(le | the) = 0.145835
t(chien | the) = 0.425175
t(chat | the) = 0.428990
t(le | dog) = 0.295085
t(chien | dog) = 0.704915
t(chat | dog) = 0.000000
t(le | cat) = 0.392366
t(chien | cat) = 0.000000
t(chat | cat) = 0.607634
q(0 | 0, 2, 2) = 0.145835
q(0 | 1, 2, 2) = 0.343726
q(1 | 0, 2, 2) = 0.854165
q(1 | 1, 2, 2) = 0.656274
#### Epoch 3
t(le | the) = 0.068743
t(chien | the) = 0.461093
t(chat | the) = 0.470164
t(le | dog) = 0.240067
t(chien | dog) = 0.759933
t(chat | dog) = 0.000000
t(le | cat) = 0.269950
t(chien | cat) = 0.000000
t(chat | cat) = 0.730050
q(0 | 0, 2, 2) = 0.068743
q(0 | 1, 2, 2) = 0.255009
q(1 | 0, 2, 2) = 0.931257
q(1 | 1, 2, 2) = 0.744991
#### Epoch 4
t(le | the) = 0.019575
t(chien | the) = 0.489650
t(chat | the) = 0.490775
t(le | dog) = 0.171973
t(chien | dog) = 0.828027
t(chat | dog) = 0.000000
t(le | cat) = 0.180627
t(chien | cat) = 0.000000
t(chat | cat) = 0.819373
q(0 | 0, 2, 2) = 0.019575
q(0 | 1, 2, 2) = 0.176300
q(1 | 0, 2, 2) = 0.980425
q(1 | 1, 2, 2) = 0.823700
#### Epoch 5
t(le | the) = 0.002213
t(chien | the) = 0.498866
t(chat | the) = 0.498920
t(le | dog) = 0.112349
t(chien | dog) = 0.887651
t(chat | dog) = 0.000000
t(le | cat) = 0.113631
t(chien | cat) = 0.000000
t(chat | cat) = 0.886369
q(0 | 0, 2, 2) = 0.002213
q(0 | 1, 2, 2) = 0.112990
q(1 | 0, 2, 2) = 0.997787
q(1 | 1, 2, 2) = 0.887010
#### Epoch 6
t(le | the) = 0.000043
t(chien | the) = 0.499978
t(chat | the) = 0.499978
t(le | dog) = 0.066807
t(chien | dog) = 0.933193
t(chat | dog) = 0.000000
t(le | cat) = 0.066904
t(chien | cat) = 0.000000
t(chat | cat) = 0.933096
q(0 | 0, 2, 2) = 0.000043
q(0 | 1, 2, 2) = 0.066856
q(1 | 0, 2, 2) = 0.999957
q(1 | 1, 2, 2) = 0.933144
#### Epoch 7
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.036967
t(chien | dog) = 0.963033
t(chat | dog) = 0.000000
t(le | cat) = 0.036971
t(chien | cat) = 0.000000
t(chat | cat) = 0.963029
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.036969
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.963031
#### Epoch 8
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.019541
t(chien | dog) = 0.980459
t(chat | dog) = 0.000000
t(le | cat) = 0.019541
t(chien | cat) = 0.000000
t(chat | cat) = 0.980459
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.019541
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.980459
#### Epoch 9
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.010062
t(chien | dog) = 0.989938
t(chat | dog) = 0.000000
t(le | cat) = 0.010062
t(chien | cat) = 0.000000
t(chat | cat) = 0.989938
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.010062
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.989938
#### Epoch 10
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.005107
t(chien | dog) = 0.994893
t(chat | dog) = 0.000000
t(le | cat) = 0.005107
t(chien | cat) = 0.000000
t(chat | cat) = 0.994893
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.005107
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.994893
#### Epoch 11
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.002573
t(chien | dog) = 0.997427
t(chat | dog) = 0.000000
t(le | cat) = 0.002573
t(chien | cat) = 0.000000
t(chat | cat) = 0.997427
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.002573
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.997427
#### Epoch 12
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.001292
t(chien | dog) = 0.998708
t(chat | dog) = 0.000000
t(le | cat) = 0.001292
t(chien | cat) = 0.000000
t(chat | cat) = 0.998708
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.001292
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.998708
#### Epoch 13
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000647
t(chien | dog) = 0.999353
t(chat | dog) = 0.000000
t(le | cat) = 0.000647
t(chien | cat) = 0.000000
t(chat | cat) = 0.999353
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000647
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999353
#### Epoch 14
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000324
t(chien | dog) = 0.999676
t(chat | dog) = 0.000000
t(le | cat) = 0.000324
t(chien | cat) = 0.000000
t(chat | cat) = 0.999676
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000324
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999676
#### Epoch 15
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000162
t(chien | dog) = 0.999838
t(chat | dog) = 0.000000
t(le | cat) = 0.000162
t(chien | cat) = 0.000000
t(chat | cat) = 0.999838
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000162
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999838
#### Epoch 16
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000081
t(chien | dog) = 0.999919
t(chat | dog) = 0.000000
t(le | cat) = 0.000081
t(chien | cat) = 0.000000
t(chat | cat) = 0.999919
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000081
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999919
#### Epoch 17
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000041
t(chien | dog) = 0.999959
t(chat | dog) = 0.000000
t(le | cat) = 0.000041
t(chien | cat) = 0.000000
t(chat | cat) = 0.999959
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000041
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999959
#### Epoch 18
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000020
t(chien | dog) = 0.999980
t(chat | dog) = 0.000000
t(le | cat) = 0.000020
t(chien | cat) = 0.000000
t(chat | cat) = 0.999980
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000020
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999980
#### Epoch 19
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000010
t(chien | dog) = 0.999990
t(chat | dog) = 0.000000
t(le | cat) = 0.000010
t(chien | cat) = 0.000000
t(chat | cat) = 0.999990
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000010
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999990
#### Epoch 20
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000005
t(chien | dog) = 0.999995
t(chat | dog) = 0.000000
t(le | cat) = 0.000005
t(chien | cat) = 0.000000
t(chat | cat) = 0.999995
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000005
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999995
#### Epoch 21
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000003
t(chien | dog) = 0.999997
t(chat | dog) = 0.000000
t(le | cat) = 0.000003
t(chien | cat) = 0.000000
t(chat | cat) = 0.999997
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000003
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999997
#### Epoch 22
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000001
t(chien | dog) = 0.999999
t(chat | dog) = 0.000000
t(le | cat) = 0.000001
t(chien | cat) = 0.000000
t(chat | cat) = 0.999999
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000001
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999999
#### Epoch 23
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000001
t(chien | dog) = 0.999999
t(chat | dog) = 0.000000
t(le | cat) = 0.000001
t(chien | cat) = 0.000000
t(chat | cat) = 0.999999
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000001
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 0.999999
#### Epoch 24
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 25
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 26
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 27
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 28
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 29
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 30
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 31
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 32
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 33
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 34
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 35
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 36
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 37
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 38
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 39
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 40
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 41
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 42
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 43
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 44
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 45
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 46
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 47
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 48
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 49
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 50
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 51
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 52
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 53
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 54
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 55
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000
#### Epoch 56
t(le | the) = 0.000000
t(chien | the) = 0.500000
t(chat | the) = 0.500000
t(le | dog) = 0.000000
t(chien | dog) = 1.000000
t(chat | dog) = 0.000000
t(le | cat) = 0.000000
t(chien | cat) = 0.000000
t(chat | cat) = 1.000000
q(0 | 0, 2, 2) = 0.000000
q(0 | 1, 2, 2) = 0.000000
q(1 | 0, 2, 2) = 1.000000
q(1 | 1, 2, 2) = 1.000000