Vahid Moosavi

11 April 2017

# Introduction to Representation Learning: Convolutional Neural Networks¶

### To be discussed¶

• Review of Fourier and its limits
• Feature Enginering and Feature (Representation) Learning
• Convolution
• Convolutional Neural Networks
• Examples in Urban Studies
• Extensions and applications
In [1]:
import warnings
warnings.filterwarnings("ignore")
import datetime
import pandas as pd
# import pandas.io.data
import numpy as np
import matplotlib
from matplotlib import pyplot as plt
import sys
import sompylib.sompy as SOM# from pandas import Series, DataFrame

from ipywidgets import interact, HTML, FloatSlider

%matplotlib inline


# Signal Processing is a main part of Data Driven Modeling cases¶

• Usually observations are homogeneous
• Pixels of image
• Sequence of values in sound signal or any other time series
• A Graphical representation of an artifact: City, building,...
• Usually, we are looking for something on top of the observations (Categories)
• Message of the sentence
• A certain pattern in an image such as a face in the picture
• All the methods are developed in a way to capture "invariances" within categories of interest
• translation invariance
• rotation invariance
• scale invariance
• Deformation
• Hierarchical Representation and Compositionality
• Text
• Video
• A Building
• Cities

# Fourier Series as Idealized Basis Functions¶

## The Fourier series is linear algebra in infinite dimensions, where vectors are functions¶

In [2]:
#Base Filters 1D
def base_filter_1D(N):
dd = -2*np.pi/N
w = np.exp(dd*1j)
#     w = np.exp(-2*np.pi*1j/N)

W = np.ones((N,N),dtype=complex)

for i in range(N):
for j in range(N):
W[i,j] = np.power(w,i*j)

#     W = W/np.sqrt(N)
return W

def DFT_1D_basis_vis(i):
fig = plt.figure(figsize=(10,5))
plt.subplot(1,2,1)
#Real Part of the vector
plt.plot(np.real(W)[i],'r')
#Imaginary Part of the vector
plt.title('real part')
plt.axis('off')
plt.subplot(1,2,2)
plt.title('imaginary part')
plt.plot(np.imag(W)[i],'b')
plt.axis('off')
N = 256
W = base_filter_1D(N)
interact(DFT_1D_basis_vis,i=(0,N-1,1));


## Now from Time domain to Frequency domain¶

In [3]:
# Two signals have the same frequencies but with a shift in time
N = 256
t = np.arange(N)
x1 = .4*np.sin(1*t+.1) + .6*np.cos(-15*t+.1) + .3*np.random.rand(N)
x2 = .4*np.sin(1*(t-2)+.1) + .6*np.cos(-15*(t-16)+.1) + .1*np.random.rand(N)
plt.plot(x1)
plt.plot(x2)

Out[3]:
[<matplotlib.lines.Line2D at 0x110be19d0>]
In [4]:
W = base_filter_1D(N)
X1 = W.dot(x1)
X2 = W.dot(x2)

In [5]:
plt.plot(np.abs(np.absolute(X1)),'b');
plt.plot(np.abs(np.absolute(X2)),'g');


# 2D Fourier¶

http://www.robots.ox.ac.uk/~az/lectures/ia/lect2.pdf

In [6]:
N = 16
W1D = base_filter_1D(N)
def base_filter2d_vis(u=1,v=1):

r = W1D[u][np.newaxis,:].T
c = W1D[v][np.newaxis,:]
W2 = r.dot(c)
fig = plt.figure(figsize=(15,7))
plt.subplot(1,2,1)
plt.title('Real Part(CoSine Wave)')
plt.imshow(np.real(W2),cmap=plt.cm.gray)
plt.axis('off')
plt.subplot(1,2,2)
plt.title('Imaginary Part (Sine Wave)')
plt.axis('off')
plt.imshow(np.imag(W2),cmap=plt.cm.gray)

In [7]:
interact(base_filter2d_vis,u=(0,N-1,1),v=(0,N-1,1));

In [8]:
#Base Filters 2D
def base_filter_2D(N):
W1D = base_filter_1D(N)*np.sqrt(N)
W2D = np.ones((N,N,N,N),dtype=complex)

for u in range(0,N):
for v in range(0,N):
r = W1D[u][np.newaxis,:].T
c = W1D[v][np.newaxis,:]
W2D[u,v,:,:] = r.dot(c)
W2D = W2D/(np.sqrt(N*N))
return W2D


## Base Filters Dictionary¶

• Note that these base filters are defined independent of the data!
In [47]:
N = 8
W2D = base_filter_2D(N)
print W2D.shape
fig = plt.figure(figsize=(7,7))
k =1
for u in range(0,N):
for v in range(0,N):
W2 = W2D[u,v,:,:]
plt.subplot(N,N,k)
plt.imshow(np.real(W2),cmap=plt.cm.gray)
#         plt.imshow(np.imag(W2),cmap=plt.cm.gray)
k = k +1
plt.axis('off')

(8, 8, 8, 8)

In [50]:
# Example 1
# With shift stil the patters are similar in freq domain
N = 128
x = 5
y = 5
img = np.zeros((128,128))

for i in range(20):
indx = np.random.randint(x,N-x)
indy = np.random.randint(y,N-y)
img[indx:indx+x,indy:indy+y] = 1
plt.subplot(2,2,1);
plt.imshow(img,cmap=plt.cm.gray);
plt.axis('off');

plt.subplot(2,2,2);
F_img = np.fft.fft2(img)

#To shift low pass (lower freq) to the center
F_img = np.fft.fftshift(F_img)
plt.imshow(np.log(np.absolute(F_img)));
plt.axis('off');

img = np.zeros((128,128))

x = 10
y = 10
for i in range(20):
indx = np.random.randint(x,N-x)
indy = np.random.randint(y,N-y)
img[indx:indx+x,indy:indy+y] = 1
plt.subplot(2,2,3);
plt.imshow(img,cmap=plt.cm.gray);
plt.axis('off');

plt.subplot(2,2,4);
F_img = np.fft.fft2(img)

#To shift low pass (lower freq) to the center
F_img = np.fft.fftshift(F_img)
plt.imshow(np.log(np.absolute(F_img)));
plt.axis('off');


# Representation Learning¶

## Considering these factors?¶

• translation invariance
• rotation invariance
• scale invariance
• Deformation

• Compositionality and Hierarchical Representation ## Convolutional Neural Networks (CNN) are seemingly a very good answer

# Convolution¶

## Discrete case¶

### 1D Convolution¶

In [8]:
N = 100
t = np.linspace(0,4,num=N)
t =np.unique(np.concatenate((t,-t)))
r = t[-1]-t[0]
N = t.shape[0]
f = 1*np.sin(2.5*t) + 1*np.cos(1.5*t+.1)  + .63*np.random.rand(N)
f[f<0]=0
f = np.exp(-t)
f[t<0]=0

# f = np.ones(t.shape)*1
# c = 0
# f[t>=(c+1)]=0
# f[t<=(c-1)]=0

# sigma = 2
# f = np.exp(-(t-c)**2/2*sigma)

def vis_con(k=2):

plt.subplot(2,1,1)
plt.plot(t, f,'-b');

#     c = 0
fgs = []
sigma = 1

for c in t:
g = np.ones(t.shape)*1
g[t>=(c+.5)]=0
g[t<=(c-.5)]=0

#         g = np.exp(-((t-c)**2)/(2*(sigma**2)))
#         g = g/(np.sqrt(2*np.pi)*sigma)
fgs.append(np.dot(f,g))

g = np.ones(t.shape)*1
g[t>=(k+.5)]=0
g[t<=(k-.5)]=0

#Guassian
#     g = np.exp(-((t-k)**2)/(2*(sigma**2)))
#     g = g/(np.sqrt(2*np.pi)*sigma)
#     g = 1/np.sqrt(2*np.pi*sigma**2)*np.exp(-((t-k)**2)/(2*sigma**2))

fg = np.dot(f,g)
plt.plot(t,g,'-r',linewidth=3)
plt.ylabel('f')

bottom1 = np.minimum(g, 0)
bottom2 = np.minimum(0,f)
bottom= np.minimum(bottom1,bottom2)
top = np.minimum(g,f)
plt.fill_between(t,top, bottom,facecolor='red',edgecolor="None", interpolate=True);
plt.subplot(2,1,2);
plt.plot(t,fgs)
#     plt.plot(t,np.convolve(f,g,mode='same'),'g');
plt.plot(k,fg,'or');
plt.xlim(t[0],t[-1])
plt.ylabel('conv f*g')
plt.xlabel('t')
interact(vis_con,k=(-4,4,.1));


## Guassian Kernel¶

In [9]:
N = 100
t = np.linspace(0,4,num=N)
t =np.unique(np.concatenate((t,-t)))
r = t[-1]-t[0]
N = t.shape[0]
f = 1*np.sin(2.5*t) + 1*np.cos(1.5*t+.1)  + .63*np.random.rand(N)
f[f<0]=0
# f = np.exp(-t)
# f[t<0]=0

# f = np.ones(t.shape)*1
# c = 0
# f[t>=(c+1)]=0
# f[t<=(c-1)]=0

# sigma = 2
# f = np.exp(-(t-c)**2/2*sigma)

def vis_con(k=2,sigma=1):

plt.subplot(2,1,1)
plt.plot(t, f,'-b');

#     c = 0
fgs = []
#     sigma = 1

for c in t:
g = np.ones(t.shape)*1
g[t>=(c+.5)]=0
g[t<=(c-.5)]=0

g = np.exp(-((t-c)**2)/(2*(sigma**2)))
g = g/(np.sqrt(2*np.pi)*sigma)
fgs.append(np.dot(f,g))

g = np.ones(t.shape)*1
g[t>=(k+.5)]=0
g[t<=(k-.5)]=0

#Guassian
g = np.exp(-((t-k)**2)/(2*(sigma**2)))
g = g/(np.sqrt(2*np.pi)*sigma)
#     g = 1/np.sqrt(2*np.pi*sigma**2)*np.exp(-((t-k)**2)/(2*sigma**2))

fg = np.dot(f,g)
plt.plot(t,g,'-r',linewidth=3)
plt.ylabel('f')

bottom1 = np.minimum(np.abs(g), 0)
bottom2 = np.minimum(0,np.abs(f))
bottom= np.minimum(bottom1,bottom2)
top = np.minimum(np.abs(g),np.abs(f))
plt.fill_between(t,top, bottom,facecolor='red',edgecolor="None", interpolate=True);
plt.subplot(2,1,2);
plt.plot(t,fgs)
#     plt.plot(t,np.convolve(f,g,mode='same'),'g');
plt.plot(k,fg,'or');
plt.xlim(t[0],t[-1])
plt.ylabel('conv f*g')
plt.xlabel('t')
interact(vis_con,k=(-4,4,.1),sigma=(.01,10,.1));


## It will be simply an average over f¶

In [54]:
N = 100
t = np.linspace(0,4,num=N)
t =np.unique(np.concatenate((t,-t)))
r = t[-1]-t[0]
N = t.shape[0]
f = 1*np.sin(2.5*t) + 1*np.cos(1.5*t+.1)  + .63*np.random.rand(N)
# f[f<0]=0
# f = np.exp(-t)
# f[t<0]=0

# f = np.ones(t.shape)*1
# c = 0
# f[t>=(c+1)]=0
# f[t<=(c-1)]=0

sigma = 2
# f = np.exp(-(t-c)**2/2*sigma)

def vis_con(k=2,sigma=1):

plt.subplot(2,1,1)
plt.plot(t, f,'-b');

#     c = 0
fgs = []
#     sigma = 1

for c in t:
g = np.ones(t.shape)*1
g[t>=(c+sigma)]=0
g[t<=(c-sigma)]=0

#         g = np.exp(-((t-c)**2)/(2*(sigma**2)))
#         g = g/(np.sqrt(2*np.pi)*sigma)
fgs.append(np.dot(f,g))

g = np.ones(t.shape)*1
g[t>=(k+sigma)]=0
g[t<=(k-sigma)]=0

#Guassian
#     g = np.exp(-((t-k)**2)/(2*(sigma**2)))
#     g = g/(np.sqrt(2*np.pi)*sigma)
#     g = 1/np.sqrt(2*np.pi*sigma**2)*np.exp(-((t-k)**2)/(2*sigma**2))

fg = np.dot(f,g)
plt.plot(t,g,'-r',linewidth=3)
plt.ylabel('f')

bottom1 = np.minimum(np.abs(g), 0)
bottom2 = np.minimum(0,np.abs(f))
bottom= np.minimum(bottom1,bottom2)
top = np.minimum(np.abs(g),np.abs(f))
plt.fill_between(t,top, bottom,facecolor='red',edgecolor="None", interpolate=True);
plt.subplot(2,1,2);
plt.plot(t,fgs)
#     plt.plot(t,np.convolve(f,g,mode='same'),'g');
plt.plot(k,fg,'or');
plt.xlim(t[0],t[-1])
plt.ylabel('conv f*g')
plt.xlabel('t')
interact(vis_con,k=(-4,4,.1),sigma=(.01,10,.1));


# Fourier Transform and Convolution¶

### Therefore, convolution of each base is invariant and it is just one value for each base filter¶

In [55]:
interact(DFT_1D_basis_vis,i=(0,N-1,1));

In [56]:
N = 100
t = np.linspace(0,4,num=N)
t =np.unique(np.concatenate((t,-t)))
r = t[-1]-t[0]
N = t.shape[0]
f = 1*np.sin(2.5*t) + 1*np.cos(1.5*t+.1)  + .63*np.random.rand(N)
f[f<0]=0
# f = np.exp(-t)
# f[t<0]=0

# f = np.ones(t.shape)*1
# c = 0
# f[t>=(c+1)]=0
# f[t<=(c-1)]=0

# sigma = 2
# f = np.exp(-(t-c)**2/2*sigma)

def vis_con_f(base_filter=1,k=2):

n = base_filter
plt.subplot(2,1,1)
plt.plot(t, f,'-b');

#     c = 0
fgs = []
#     sigma = 1

for c in t:
g = np.real(W[n])
fgs.append(np.dot(f,g))

g = np.real(W[n])

#Guassian
#     g = np.exp(-((t-k)**2)/(2*(sigma**2)))
#     g = g/(np.sqrt(2*np.pi)*sigma)
#     g = 1/np.sqrt(2*np.pi*sigma**2)*np.exp(-((t-k)**2)/(2*sigma**2))

fg = np.dot(f,g)
plt.plot(t,g,'-r',linewidth=3)
plt.ylabel('f')

bottom1 = np.minimum(np.abs(g), 0)
bottom2 = np.minimum(0,np.abs(f))
bottom= np.minimum(bottom1,bottom2)
top = np.minimum(np.abs(g),np.abs(f))
plt.fill_between(t,top, bottom,facecolor='red',edgecolor="None", interpolate=True);
plt.subplot(2,1,2);
plt.plot(t,fgs)
#     plt.plot(t,np.convolve(f,g,mode='same'),'g');
plt.plot(k,fg,'or');
plt.xlim(t[0],t[-1])
plt.ylabel('conv f*g')
plt.xlabel('t')

N = t.shape[0]
W = base_filter_1D(N)
interact(vis_con_f,base_filter=(0,W.shape[0],1),k=(-4,4,.1));


## 2D Convolution¶

### Arbitrary Convolutional Kernels¶

In [58]:
#2D
from skimage import data
from skimage import data, io, filters
from skimage.color import rgb2gray
import scipy.signal as sg

fig = plt.figure(figsize=(10,5));
image = data.coins()
plt.subplot(1,2,1);
plt.imshow(image,plt.cm.gray);

#Bluring Kernel
kernel = [[0,1,0],[1,-4,1],[0,1,0]]

#Edge
kernel = [[1,-1],[-1,1]]
plt.axis('off');
fm = sg.convolve2d(image,kernel)
plt.subplot(1,2,2);
plt.imshow(fm,plt.cm.gray);
plt.axis('off');