Chapter 9. Big Entropy and the Generalized Linear Model

In [1]:
import seaborn as sns
import torch

import pyro

import rethinking

Code 9.1

In [2]:
p = {}
p["A"] = torch.tensor([0., 0, 10, 0, 0])
p["B"] = torch.tensor([0., 1, 8, 1, 0])
p["C"] = torch.tensor([0., 2, 6, 2, 0])
p["D"] = torch.tensor([1., 2, 4, 2, 1])
p["E"] = torch.tensor([2., 2, 2, 2, 2])

Code 9.2

In [3]:
p_norm = {key: (q / q.sum()) for key, q in p.items()}

Code 9.3

In [4]:
H = {key: -torch.where(q == 0, torch.tensor(0.), q * q.log()).sum()
     for key, q in p_norm.items()}
H
Out[4]:
{'A': tensor(-0.),
 'B': tensor(0.6390),
 'C': tensor(0.9503),
 'D': tensor(1.4708),
 'E': tensor(1.6094)}

Code 9.4

In [5]:
ways = torch.tensor([1., 90, 1260, 37800, 113400])
logwayspp = ways.log() / 10

Code 9.5

In [6]:
# build tensor of the candidate distributions
p = torch.empty(4, 4)
p[0] = torch.tensor([1/4, 1/4, 1/4, 1/4])
p[1] = torch.tensor([2/6, 1/6, 1/6, 2/6])
p[2] = torch.tensor([1/6, 2/6, 2/6, 1/6])
p[3] = torch.tensor([1/8, 4/8, 2/8, 1/8])

# compute expected value of each
(p * torch.tensor([0., 1, 1, 2])).sum(1)
Out[6]:
tensor([1., 1., 1., 1.])

Code 9.6

In [7]:
# compute entropy of each distribution
-(p * p.log()).sum(1)
Out[7]:
tensor([1.3863, 1.3297, 1.3297, 1.2130])

Code 9.7

In [8]:
p = 0.7
A = torch.tensor([(1 - p) ** 2, p * (1 - p), (1 - p) * p, p ** 2])
A
Out[8]:
tensor([0.0900, 0.2100, 0.2100, 0.4900])

Code 9.8

In [9]:
-(A * A.log()).sum()
Out[9]:
tensor(1.2217)

Code 9.9

In [10]:
def sim_p(G=1.4):
    x123 = torch.rand(3)
    x4 = (G * x123.sum() - x123[1] - x123[2]) / (2 - G)
    z = torch.cat([x123, x4.unsqueeze(0)]).sum()
    p = torch.cat([x123, x4.unsqueeze(0)]) / z
    return {"H": -(p * p.log()).sum(), "p": p}

Code 9.10

In [11]:
H = [torch.stack(x) for x in zip(*[sim_p(1.4).values() for _ in range(int(1e5))])]
ax = sns.distplot(H[0])
ax.set(xlabel="Entropy", ylabel="Density");

Code 9.11

In [12]:
entropies = H[0]
distributions = H[1]

Code 9.12

In [13]:
entropies.max()
Out[13]:
tensor(1.2217)

Code 9.13

In [14]:
distributions[entropies.argmax()]
Out[14]:
tensor([0.0902, 0.2095, 0.2100, 0.4902])