#!/usr/bin/env python
# coding: utf-8

# In[1]:


get_ipython().run_line_magic('load_ext', 'watermark')
get_ipython().run_line_magic('watermark', "-a 'Sebastian Raschka' -v -d")


# #Activation Functions Cheatsheet

# Linear:  $$g(z) = z$$ 
# 
# Unit step: $$g(z) = \begin{cases} 0 & z < 0, \\ 1 & z \ge 0 \end{cases}$$
# 
# logistic (sigmoid): $$g(z) = \frac{1}{1 + e^{-z}}$$
# 
# hyperbolic tangent (sigmoid): $$g(z) = tanh(z) = \frac{e^{z}-e^{-z}}{e^{z}+e^{-z}}$$
# 
# 
# Piecewise linear: $$g(z) = \begin{cases}
#  1 & \mbox{for } z \geq \frac{1}{2} \\
#  z + \frac{1}{2} & \mbox{for } -\frac{1}{2} < z < \frac{1}{2} \\
#  0 & \mbox{for } z \leq -\frac{1}{2}
#  \end{cases}$$

# In[2]:


import numpy as np
def activation(z, kind):
    
    implemented = ('linear', 'unit step', 'logistic', 'piecewise linear', 'tanh')
    if kind not in implemented:
        raise AttributeError('%s not in %s' % (kind, implemented))
    
    if kind == 'unit step':
        return np.where(z >= 0.0, 1, 0)
    
    elif kind == 'logistic':
        return 1.0 / (1.0 + np.exp(-z))
    
    elif kind == 'tanh':
        e_p = np.exp(z) 
        e_m = np.exp(-z)
        return (e_p - e_m) / (e_p + e_m)  
    
    elif kind == 'piecewise linear':
        if z >= 0.5:
            return 1
        elif z <= -0.5:
            return 0
        else:
            return z + 0.5
        
    else:
        return z


# In[3]:


import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')


# In[4]:


x = np.arange(-5, 5, 0.005)
for a in ('linear', 'unit step', 'logistic', 'piecewise linear', 'tanh'):
    y = [activation(z, kind=a) for z in x]
    plt.plot(x, y)
    plt.title(a)
    plt.ylim([-1.5, 1.5])
    plt.xlabel('z')
    plt.grid()
    plt.show()