SINGA Model Classes

Layer

Typically, the life cycle of a layer instance includes:

  1. construct layer without input_sample_shapes, goto 2; or,

    construct layer with input_sample_shapes, goto 3;

  2. call setup to create the parameters and setup other meta fields;
  3. initialize the parameters of the layer
  4. call forward or access layer members
  5. call backward and get parameters for update
In [1]:
from singa import tensor, device, layer

#help(layer.Layer)
layer.engine='singacpp'

Common layers

In [2]:
from singa.layer import Dense, Conv2D, MaxPooling2D, Activation, BatchNormalization, Softmax

Dense Layer

In [3]:
dense = Dense('dense', 3, input_sample_shape=(2,))
#dense.param_names()
w, b = dense.param_values()
print(w.shape, b.shape)
(2, 3) (3,)
In [4]:
w.gaussian(0, 0.1)
b.set_value(0)
In [5]:
x  = tensor.Tensor((2,2))
x.uniform(-1, 1)
y = dense.forward(True, x)
tensor.to_numpy(y)
Out[5]:
array([[ 0.02440065, -0.03396009,  0.01396658],
       [ 0.00771775,  0.07841966, -0.05931654]], dtype=float32)
In [6]:
gx, [gw, gb] = dense.backward(True, y)
print(gx.shape, gw.shape, gb.shape)
(2, 2) (2, 3) (3,)

Convolution Layer

In [7]:
conv = Conv2D('conv', 4, 3, 1, input_sample_shape=(3, 6, 6))
print(conv.get_output_sample_shape())
(4, 6, 6)

Pooling Layer

In [8]:
pool = MaxPooling2D('pool', 3, 2, input_sample_shape=(4, 6, 6))
print(pool.get_output_sample_shape())
(4, 3, 3)

Branch layers

In [9]:
from singa.layer import Split, Merge, Slice, Concat
In [10]:
split = Split('split', 2, input_sample_shape=(4, 6, 6))
print(split.get_output_sample_shape())
[(4, 6, 6), (4, 6, 6)]
In [11]:
merge = Merge('merge', input_sample_shape=(4, 6, 6))
print(merge.get_output_sample_shape())
(4, 6, 6)
In [12]:
sli = Slice('slice', 1, [2], input_sample_shape=(4, 6, 6))
print(sli.get_output_sample_shape())
[(2, 6, 6), (2, 6, 6)]
In [13]:
concat = Concat('concat', 1, input_sample_shapes=[(3, 6, 6), (1, 6, 6)])
print(concat.get_output_sample_shape())
(4, 6, 6)

Metric and Loss

In [14]:
from singa import metric
import numpy as np

x = tensor.Tensor((3, 5))
x.uniform(0, 1)  # randomly genearte the prediction activation
x = tensor.softmax(x)  # normalize the prediction into probabilities
print(tensor.to_numpy(x))
y = tensor.from_numpy(np.array([0, 1, 3], dtype=np.int))  # set the truth

f = metric.Accuracy()
acc = f.evaluate(x, y)  # averaged accuracy over all 3 samples in x
print(acc)
[[ 0.13973515  0.21827343  0.21068712  0.21905626  0.21224809]
 [ 0.2354937   0.1047527   0.18490241  0.23617713  0.23867407]
 [ 0.2659435   0.11397494  0.1659178   0.22683497  0.22732878]]
0.0
In [15]:
from singa import loss

x = tensor.Tensor((3, 5))
x.uniform(0, 1)  # randomly genearte the prediction activation
y = tensor.from_numpy(np.array([0, 1, 3], dtype=np.int))  # set the truth

f = loss.SoftmaxCrossEntropy()
l = f.forward(True, x, y)  # l is tensor with 3 loss values
g = f.backward()  # g is a tensor containing all gradients of x w.r.t l
print(l.l1())
print(tensor.to_numpy(g))
1.8030937910079956
[[-0.78104687  0.18748793  0.16346708  0.24803984  0.18205206]
 [ 0.21501946 -0.83683592  0.19003348  0.20714596  0.22463693]
 [ 0.20000091  0.23285127  0.26842937 -0.87474263  0.17346108]]

Optimizer

In [16]:
from singa import optimizer

sgd = optimizer.SGD(lr=0.01, momentum=0.9, weight_decay=1e-4)
p = tensor.Tensor((3,5))
p.uniform(-1, 1)
g = tensor.Tensor((3,5))
g.gaussian(0, 0.01)

sgd.apply(1, g, p, 'param')  # use the global lr=0.1 for epoch 1
sgd.apply_with_lr(2, 0.03, g, p, 'param')  # use lr=0.03 for epoch 2
Out[16]:
<singa.tensor.Tensor at 0x7f539260f710>

FeedForwardNet

In [17]:
from singa import net as ffnet
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(layer.Conv2D('conv1', 32, 5, 1, input_sample_shape=(3,32,32,)))
net.add(layer.Activation('relu1'))
net.add(layer.MaxPooling2D('pool1', 3, 2))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('dense', 10))

# init parameters
for p in net.param_values():
    if len(p.shape) == 0:
        p.set_value(0)
    else:
        p.gaussian(0, 0.01)
print(net.param_names())
('conv1', (32, 32, 32))
('relu1', (32, 32, 32))
('pool1', (32, 16, 16))
('flat', (8192,))
('dense', (10,))
['conv1/weight', 'conv1/bias', 'dense/weight', 'dense/bias']
In [18]:
layer.engine = 'cudnn'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(layer.Conv2D('conv1', 32, 5, 1, input_sample_shape=(3,32,32,)))
net.add(layer.Activation('relu1'))
net.add(layer.MaxPooling2D('pool1', 3, 2))
net.add(layer.Flatten('flat'))
net.add(layer.Dense('dense', 10))

# init parameters
for p in net.param_values():
    if len(p.shape) == 0:
        p.set_value(0)
    else:
        p.gaussian(0, 0.01)
        
# move net onto gpu
dev = device.create_cuda_gpu()
net.to_device(dev)
('conv1', (32, 32, 32))
('relu1', (32, 32, 32))
('pool1', (32, 16, 16))
('flat', (8192,))
('dense', (10,))