In [3]:
import torch
import numpy as np
In [4]:
torch.__version__
Out[4]:
'1.4.0'
In [6]:
torch.nn.Linear?
In [8]:
a = torch.nn.ModuleList()
a.add_module?
In [73]:
class NeuralNetwork_Pytorch(torch.nn.Module):
    
    def __init__(self, n_inputs, n_hiddens_list, n_outputs, device='cpu'):
        
        super().__init__()

        self.n_inputs = n_inputs
        self.n_hiddens_list = n_hiddens_list
        self.n_outputs = n_outputs
        self.device = device
        
        self.model = torch.nn.ModuleList()
        ni = n_inputs
        for nh in n_hiddens_list + [n_outputs]:
            self.model.append(torch.nn.Linear(ni, nh))
            ni = nh
        # self.model.append(torch.nn.Linear(ni,n_outputs))
        self.stand_params = None
        self.error_trace = []
    
    
    def __repr__(self):
        return f'NeuralNetwork_Pytorch({self.n_inputs}, {self.n_hiddens_list}, {self.n_outputs}, device=\'{self.device}\')'

    def forward(self, Xst):
        Ys = [Xst]
        for layer in self.model[:-1]:
            Ys.append(torch.tanh(layer(Ys[-1])))
        last_layer = self.model[-1]
        Ys.append(last_layer(Ys[-1]))
        return Ys[1:]
    
    # def backward(self.......):
    
    def use(self, X, return_hidden_layer_outputs=False):
        X = torch.from_numpy(X.astype(np.float32))
        Xst = self.standardize_X(X)
        Ys = self.forward(Xst)
        Y = Ys[-1]
        Y = self.unstandardize_T(Y)
        Zs = Ys[:-1]
        return (Y, Zs) if return_hidden_layer_outputs else Y
    
    def train(self, X, T, n_epochs, learning_rate=0.01, method='adam', verbose=True):
        
        if isinstance(X, np.ndarray):
            X = torch.from_numpy(X.astype(np.float32))
        if isinstance(T, np.ndarray):
            T = torch.from_numpy(T.astype(np.float32))

        self.stand_params = self.calc_standardize_parameters(X, T)
        
        Xst = self.standardize_X(X)
        Tst = self.standardize_T(T)

        if method == 'sgd':
            optimizer = torch.optim.SGD(self.parameters(), lr=learning_rate)
        elif method == 'adam':
            optimizer = torch.optim.Adam(self.parameters(), lr=learning_rate)
        else:
            print('train: method must be \'sgd\', \'adam\'.')
  
        error_f = torch.nn.MSELoss()
    
        for epoch in range(n_epochs):
            
            Yst = self.forward(Xst)[-1]  # output layer output
            mse = error_f(Yst, Tst)
            
            optimizer.zero_grad()
            
            mse.backward()
            optimizer.step()
            
            self.error_trace.append(mse)
  
        return self


    def calc_standardize_parameters(self, X, T):
        Xmeans = X.mean(axis=0)
        Xstds = X.std(axis=0)
        Xstds[Xstds == 0] = Xstds[Xstds > 0].mean()
        Tmeans = T.mean(axis=0)
        Tstds = T.std(axis=0)
        return {'Xmeans': Xmeans, 'Xstds': Xstds, 'Tmeans': Tmeans, 'Tstds': Tstds}

    def standardize_X(self, X):
        return (X - self.stand_params['Xmeans']) / self.stand_params['Xstds']

    def unstandardize_X(self, Xst):
        return Xst * self.stand_params['Xstds'] + self.stand_params['Xmeans']

    def standardize_T(self, T):
        return (T - self.stand_params['Tmeans']) / self.stand_params['Tstds']

    def unstandardize_T(self, Tst):
        return Tst * self.stand_params['Tstds'] + self.stand_params['Tmeans']
In [74]:
list(nnet.parameters())
Out[74]:
[Parameter containing:
 tensor([[-0.5649],
         [ 0.5244],
         [ 0.9894],
         [-0.2179],
         [-0.8250],
         [-0.9461],
         [ 0.3119],
         [-0.4054],
         [ 0.1036],
         [-0.9440]], requires_grad=True),
 Parameter containing:
 tensor([ 0.9089,  0.0030,  0.1964,  0.3385, -0.3227, -0.2038, -0.7111,  0.8669,
         -0.8024, -0.7803], requires_grad=True),
 Parameter containing:
 tensor([[ 0.2544,  0.1741, -0.0011, -0.1637, -0.2629, -0.2105, -0.0434, -0.2701,
          -0.3099, -0.2607]], requires_grad=True),
 Parameter containing:
 tensor([0.1280], requires_grad=True)]
In [75]:
nnet.stand_params['Xmeans']
Out[75]:
tensor([4.5000])
In [76]:
nnet = NeuralNetwork_Pytorch(1, [10], 1)
In [77]:
X = np.arange(10).reshape(-1, 1)
X
Out[77]:
array([[0],
       [1],
       [2],
       [3],
       [4],
       [5],
       [6],
       [7],
       [8],
       [9]])
In [78]:
T = X * 0.1

T
Out[78]:
array([[0. ],
       [0.1],
       [0.2],
       [0.3],
       [0.4],
       [0.5],
       [0.6],
       [0.7],
       [0.8],
       [0.9]])
In [79]:
nnet.stand_params = nnet.calc_standardize_parameters(X, T)
In [80]:
nnet.use(X)
Out[80]:
tensor([[4.8894],
        [5.0679],
        [5.2758],
        [5.4902],
        [5.6720],
        [5.7839],
        [5.8134],
        [5.7771],
        [5.7059],
        [5.6276]], grad_fn=<AddBackward0>)
In [81]:
nnet
Out[81]:
NeuralNetwork_Pytorch(1, [10], 1, device='cpu')
In [85]:
nnet.train(X, T, 100)
Out[85]:
NeuralNetwork_Pytorch(1, [10], 1, device='cpu')
In [86]:
nnet.use(X)
Out[86]:
tensor([[0.2627],
        [0.9561],
        [1.8146],
        [2.8249],
        [3.9388],
        [5.0795],
        [6.1702],
        [7.1620],
        [8.0309],
        [8.7616]], grad_fn=<AddBackward0>)
In [84]:
T
Out[84]:
array([[0. ],
       [0.1],
       [0.2],
       [0.3],
       [0.4],
       [0.5],
       [0.6],
       [0.7],
       [0.8],
       [0.9]])
In [87]:
import matplotlib.pyplot as plt

plt.plot(nnet.error_trace)
Out[87]:
[<matplotlib.lines.Line2D at 0x7f2710322d00>]
In [ ]: