Title: Multilayer Architectures Author: Thomas Breuel Institution: UniKL
import scipy,pylab,numpy
from numpy import *
from scipy import *
from pylab import *
from urllib import urlopen
from gzip import GzipFile
from scipy.spatial import distance
from scipy.spatial.distance import cdist
def sigmoid(x):
return 1/(1+exp(-x))
N = 20
ni = 3
no = 2
seq = array(rand(N)*ni,'i')%3
output = 1*(roll(seq,1)==0)*(seq==1)
target = zeros((N,no))
target[output] = 1
plot(seq[:20])
plot(output[:20])
[<matplotlib.lines.Line2D at 0x3c30650>]
xs = zeros((N,ni))
for i,n in enumerate(seq): xs[i,n] = 1.0
imshow(xs[:20,:].T,interpolation='nearest')
<matplotlib.image.AxesImage at 0x3db1a50>
nh = 3
ys = zeros((N,nh))
zs = zeros((N,no))
A = randn(nh,ni)
a = randn(nh)
B = randn(no,nh)
b = randn(no)
def sigmoid(x): return 1/(1+exp(-x))
def forward():
for t in range(N-1):
ys[t] = sigmoid(dot(A,xs[t])+a)
zs[t] = sigmoid(dot(B,ys[t])+b)
forward()
dzs = zeros((N,no))
dys = zeros((N,nh))
def backward():
for t in range(N-1)[::-1]:
dzs[t] = (zs[t]-target[t])*zs[t]*(1-zs[t])
dys[t] = dot(dzs[t],B)*ys[t]*(1-ys[t])
backward()
def update(eta=0.3):
global A,a,B,b
for t in range(N-1):
B -= eta*outer(dzs[t],ys[t])
b -= eta*dzs[t]
A -= eta*outer(dys[t],xs[t])
a -= eta*dys[t]
update()
for i in range(1000):
forward()
backward()
update()
if i%100==0: print i,sum((zs[:N-1]-target[:N-1])**2)**.5
0 2.08162539801 100 1.8904233524 200 1.87767422863 300 1.86765662972 400 1.86310296207 500 1.86075458681 600 1.85941584069 700 1.8585984244 800 1.85806426965 900 1.85769429465
A = randn(nh,ni)
a = randn(nh)
B = randn(no,nh)
b = randn(no)
ns = 3
states = zeros((N,ns))
S = randn(ns,nh+ns)
s = randn(ns)
def forward():
for t in range(N-1):
ys[t] = sigmoid(dot(A,xs[t])+a)
states[t] = sigmoid(dot(S,concatenate([ys[t],states[t-1]]))+s)
zs[t] = sigmoid(dot(B,ys[t])+b)
forward()
dstates = zeros((N,ns))
def backward():
for t in range(N-1)[::-1]:
dzs[t] = (zs[t]-target[t])*zs[t]*(1-zs[t])
dys[t] = dot(dzs[t],B)*ys[t]*(1-ys[t])
v = dot(states[t],S)
dys[t] += v[:nh]
dstates[t-1] = v[nh:]
backward()
def update(eta=1.0):
global A,a,B,b,S,s
for t in range(N-1):
B -= eta*outer(dzs[t],ys[t])
b -= eta*dzs[t]
S -= eta*outer(dstates[t],concatenate([ys[t],states[t-1]]))
s -= eta*dstates[t]
A -= eta*outer(dys[t],xs[t])
a -= eta*dys[t]
update()
for i in range(1000):
forward()
backward()
update()
if i%100==0: print i,sum((zs[:N-1]-target[:N-1])**2)**.5
0 1.85758042421 100 1.85705193087 200 1.8567707023 300 1.85659832279 400 1.85648265595 500 1.85640002593 600 1.85633822734 700 1.85629036387 800 1.85625225887 900 1.85622124147
print A
[[ -5.88549888 -9.06246784 -5.03989056] [ -1.36378817 -0.28744102 3.41206429] [ -6.06789449 -10.10759542 -4.22821784]]