In [1]:
%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -d

Sebastian Raschka 06/21/2015

CPython 3.4.3
IPython 3.1.0


# Activation Functions Cheatsheet¶

Linear: $$g(z) = z$$

Unit step: $$g(z) = \begin{cases} 0 & z < 0, \\ 1 & z \ge 0 \end{cases}$$

logistic (sigmoid): $$g(z) = \frac{1}{1 + e^{-z}}$$

hyperbolic tangent (sigmoid): $$g(z) = tanh(z) = \frac{e^{z}-e^{-z}}{e^{z}+e^{-z}}$$

Piecewise linear: $$g(z) = \begin{cases} 1 & \mbox{for } z \geq \frac{1}{2} \\ z + \frac{1}{2} & \mbox{for } -\frac{1}{2} < z < \frac{1}{2} \\ 0 & \mbox{for } z \leq -\frac{1}{2} \end{cases}$$

In [2]:
import numpy as np
def activation(z, kind):

implemented = ('linear', 'unit step', 'logistic', 'piecewise linear', 'tanh')
if kind not in implemented:
raise AttributeError('%s not in %s' % (kind, implemented))

if kind == 'unit step':
return np.where(z >= 0.0, 1, 0)

elif kind == 'logistic':
return 1.0 / (1.0 + np.exp(-z))

elif kind == 'tanh':
e_p = np.exp(z)
e_m = np.exp(-z)
return (e_p - e_m) / (e_p + e_m)

elif kind == 'piecewise linear':
if z >= 0.5:
return 1
elif z <= -0.5:
return 0
else:
return z + 0.5

else:
return z

In [3]:
import matplotlib.pyplot as plt
%matplotlib inline

In [4]:
x = np.arange(-5, 5, 0.005)
for a in ('linear', 'unit step', 'logistic', 'piecewise linear', 'tanh'):
y = [activation(z, kind=a) for z in x]
plt.plot(x, y)
plt.title(a)
plt.ylim([-1.5, 1.5])
plt.xlabel('z')
plt.grid()
plt.show()