#!/usr/bin/env python
# coding: utf-8

# # GATE Neural Network with Linear Two Neurons - Backpropagation

# ## 1. Layers with Forward and Backward

# In[1]:


import numpy as np
import random
import math


# In[2]:


class AffineWithTwoInputs:
    def __init__(self):
        self.w = np.array([random.random(), random.random()])   # weight of one input
        self.b = np.array([random.random()])  # bias
        self.x = None
        self.dw = None
        self.db = None
        
    def forward(self, x):
        self.x = x
        out = np.dot(self.w, self.x) + self.b
        return out

    def backward(self, din):
        if isinstance(din, np.ndarray) and din.size == 1:
            din = np.asscalar(din)
        dx = np.dot(din, self.w.T)
        self.dw = np.dot(self.x.T, din)
        self.db = din
        return dx

class AffineWithOneInput:
    def __init__(self):
        self.w = np.array([random.random()])   # weight of one input
        self.b = np.array([random.random()])   # bias
        self.x = None
        self.dw = None
        self.db = None

    def forward(self, x):
        self.x = x
        out = np.dot(self.w, self.x) + self.b
        return out

    def backward(self, din):
        dx = np.dot(din, self.w.T)
        self.dw = np.dot(self.x.T, din)
        self.db = din
        return dx
    
class Relu:
    def __init__(self):
        self.x = None

    def forward(self, x):
        self.x = x
        mask = (self.x <= 0)
        out = self.x.copy()
        out[mask] = 0
        return out

    def backward(self, din):
        if isinstance(din, np.ndarray):
            mask = (self.x <= 0)
            din[mask] = 0
            dx = din
        else:
            if self.x <= 0:
                dx = 0
            else:
                dx = din
        return dx
    
class SquaredError:
    def __init__(self):
        self.z = None
        self.z_target = None
    
    def forward(self, z, z_target):
        self.z = z
        self.z_target = z_target
        loss = 1.0 / 2.0 * math.pow(self.z - self.z_target, 2)
        return loss

    def backward(self, din):
        dx = (self.z - self.z_target) * din
        return dx


# ## 2. Neural Network Model of Linear Two Neurons

# In[3]:


class LinearTwoNeurons:
    def __init__(self):
        self.n1 = AffineWithTwoInputs()
        self.relu1 = Relu()
        self.n2 = AffineWithOneInput()
        self.relu2 = Relu()
        self.loss = SquaredError()
        print("Neuron n1 - Initial w: {0}, b: {1}".format(self.n1.w, self.n1.b))
        print("Neuron n2 - Initial w: {0}, b: {1}".format(self.n2.w, self.n2.b))


    def predict(self, x):
        u1 = self.n1.forward(x)
        z1 = self.relu1.forward(u1)
        u2 = self.n2.forward(z1)
        z2 = self.relu2.forward(u2)
        return z2
    
    def backpropagation_gradient(self, x, z_target):
        # forward
        z2 = self.predict(x)
        self.loss.forward(z2, z_target)

        # backward
        din = 1
        din = self.loss.backward(din)
        din = self.relu2.backward(din)
        din = self.n2.backward(din)
        din = self.relu1.backward(din)
        self.n1.backward(din)

    def learning(self, alpha, x, z_target):
        self.backpropagation_gradient(x, z_target)

        self.n1.w = self.n1.w - alpha * self.n1.dw
        self.n1.b = self.n1.b - alpha * self.n1.db
        self.n2.w = self.n2.w - alpha * self.n2.dw
        self.n2.b = self.n2.b - alpha * self.n2.db


# ## 3. OR gate with Two Linear Neurons - Learing and Testing

# In[4]:


class Data:
    def __init__(self):
        self.training_input_value = np.array([(0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0)])
        self.training_z_target = np.array([0.0, 1.0, 1.0, 1.0])
        self.numTrainData = len(self.training_input_value)

if __name__ == '__main__':
    ltn = LinearTwoNeurons()
    d = Data()
    for idx in range(d.numTrainData):
        x = d.training_input_value[idx]
        z2 = ltn.predict(x)
        z_target = d.training_z_target[idx]
        error = ltn.loss.forward(z2, z_target)
        print("x: {0:s}, z2: {1:s}, z_target: {2:s}, error: {3:7.5f}".format(str(x), str(z2), str(z_target), error))

    max_epoch = 1000
    print_epoch_period = 100
    for i in range(max_epoch + 1):
        for idx in range(d.numTrainData):
            x = d.training_input_value[idx]
            z_target = d.training_z_target[idx]
            ltn.learning(0.01, x, z_target)

        if i % print_epoch_period == 0:
            sum = 0.0
            for idx in range(d.numTrainData):
                x = d.training_input_value[idx]
                z2 = ltn.predict(x)
                z_target = d.training_z_target[idx]
                sum = sum + ltn.loss.forward(z2, z_target)

            print("Epoch{0:4d}-Error:{1:7.5f}, Neuron n1[w11: {2:7.5f}, w12: {3:7.5f}, b1: {4:7.5f}], Neuron n2[w2: {5:7.5f}, b2: {6:7.5f}]".format(
                i, 
                sum / d.numTrainData,
                ltn.n1.w[0],
                ltn.n1.w[1],
                ltn.n1.b[0],
                ltn.n2.w[0],
                ltn.n2.b[0])
            )
            
    for idx in range(d.numTrainData):
        x = d.training_input_value[idx]
        z2 = ltn.predict(x)
        z_target = d.training_z_target[idx]
        error = ltn.loss.forward(z2, z_target)
        print("x: {0:s}, z2: {1:s}, z_target: {2:s}, error: {3:7.5f}".format(str(x), str(z2), str(z_target), error))


# ## 4. AND gate with Two Linear Neurons - Learing and Testing

# In[5]:


class Data:
    def __init__(self):
        self.training_input_value = np.array([(0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0)])
        self.training_z_target = np.array([0.0, 0.0, 0.0, 1.0])
        self.numTrainData = len(self.training_input_value)

if __name__ == '__main__':
    ltn = LinearTwoNeurons()
    d = Data()
    for idx in range(d.numTrainData):
        x = d.training_input_value[idx]
        z2 = ltn.predict(x)
        z_target = d.training_z_target[idx]
        error = ltn.loss.forward(z2, z_target)
        print("x: {0:s}, z2: {1:s}, z_target: {2:s}, error: {3:7.5f}".format(str(x), str(z2), str(z_target), error))

    max_epoch = 1000
    print_epoch_period = 100
    for i in range(max_epoch + 1):
        for idx in range(d.numTrainData):
            x = d.training_input_value[idx]
            z_target = d.training_z_target[idx]
            ltn.learning(0.01, x, z_target)

        if i % print_epoch_period == 0:
            sum = 0.0
            for idx in range(d.numTrainData):
                x = d.training_input_value[idx]
                z2 = ltn.predict(x)
                z_target = d.training_z_target[idx]
                sum = sum + ltn.loss.forward(z2, z_target)

            print("Epoch{0:4d}-Error:{1:7.5f}, Neuron n1[w11: {2:7.5f}, w12: {3:7.5f}, b1: {4:7.5f}], Neuron n2[w2: {5:7.5f}, b2: {6:7.5f}]".format(
                i, 
                sum / d.numTrainData,
                ltn.n1.w[0],
                ltn.n1.w[1],
                ltn.n1.b[0],
                ltn.n2.w[0],
                ltn.n2.b[0])
            )
            
    for idx in range(d.numTrainData):
        x = d.training_input_value[idx]
        z2 = ltn.predict(x)
        z_target = d.training_z_target[idx]
        error = ltn.loss.forward(z2, z_target)
        print("x: {0:s}, z2: {1:s}, z_target: {2:s}, error: {3:7.5f}".format(str(x), str(z2), str(z_target), error))


# ## 5. XOR gate with Two Linear Neurons - Learing and Testing

# In[6]:


class Data:
    def __init__(self):
        self.training_input_value = np.array([(0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0)])
        self.training_z_target = np.array([0.0, 1.0, 1.0, 0.0])
        self.numTrainData = len(self.training_input_value)

if __name__ == '__main__':
    ltn = LinearTwoNeurons()
    d = Data()
    for idx in range(d.numTrainData):
        x = d.training_input_value[idx]
        z2 = ltn.predict(x)
        z_target = d.training_z_target[idx]
        error = ltn.loss.forward(z2, z_target)
        print("x: {0:s}, z2: {1:s}, z_target: {2:s}, error: {3:7.5f}".format(str(x), str(z2), str(z_target), error))

    max_epoch = 1000
    print_epoch_period = 100
    for i in range(max_epoch + 1):
        for idx in range(d.numTrainData):
            x = d.training_input_value[idx]
            z_target = d.training_z_target[idx]
            ltn.learning(0.01, x, z_target)

        if i % print_epoch_period == 0:
            sum = 0.0
            for idx in range(d.numTrainData):
                x = d.training_input_value[idx]
                z2 = ltn.predict(x)
                z_target = d.training_z_target[idx]
                sum = sum + ltn.loss.forward(z2, z_target)

            print("Epoch{0:4d}-Error:{1:7.5f}, Neuron n1[w11: {2:7.5f}, w12: {3:7.5f}, b1: {4:7.5f}], Neuron n2[w2: {5:7.5f}, b2: {6:7.5f}]".format(
                i, 
                sum / d.numTrainData,
                ltn.n1.w[0],
                ltn.n1.w[1],
                ltn.n1.b[0],
                ltn.n2.w[0],
                ltn.n2.b[0])
            )
            
    for idx in range(d.numTrainData):
        x = d.training_input_value[idx]
        z2 = ltn.predict(x)
        z_target = d.training_z_target[idx]
        error = ltn.loss.forward(z2, z_target)
        print("x: {0:s}, z2: {1:s}, z_target: {2:s}, error: {3:7.5f}".format(str(x), str(z2), str(z_target), error))