Image('./day5/rnn_structure.png')
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
x = [float(i)*0.01 for i in range(-1000, 1000)]
tanh = np.tanh(x)
plt.plot(tanh)
plt.show()
Image('./day5/rnn3d.png')
Image('./day5/rnn_mechanism.gif.png')
Image('./day5/batching.gif.png')
Image('./day5/input_batch.gif.png')
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
tf.reset_default_graph()
t = np.array([float(i)*0.01 for i in range(10000+1)])
sin = np.sin(t[:-1])
sin_next = np.sin(t[1:])
print_summaries = 3
print(sin[:print_summaries], sin[-print_summaries:])
print(sin_next[:print_summaries], sin_next[-print_summaries:])
plt.figure(1)
plt.plot(sin)
plt.figure(2)
plt.plot(sin_next, color='g')
plt.show()
[ 0. 0.00999983 0.01999867] [-0.53200348 -0.5235096 -0.51496337] [ 0.00999983 0.01999867 0.0299955 ] [-0.5235096 -0.51496337 -0.50636564]
time_step = 100
reshaped_sin = np.reshape(sin, [-1, time_step, 1])
reshaped_sin_next = np.reshape(sin_next, [-1, 1])
signal = tf.placeholder(tf.float32, [None, time_step, 1])
signal_next = tf.placeholder(tf.float32, [None, 1])
inputs = tf.unstack(signal, axis=1)
for i, _input in enumerate(inputs):
print('unstacked input {} shape {}'.format(i, _input.shape))
unstacked input 0 shape (?, 1) unstacked input 1 shape (?, 1) unstacked input 2 shape (?, 1) unstacked input 3 shape (?, 1) unstacked input 4 shape (?, 1) unstacked input 5 shape (?, 1) unstacked input 6 shape (?, 1) unstacked input 7 shape (?, 1) unstacked input 8 shape (?, 1) unstacked input 9 shape (?, 1) unstacked input 10 shape (?, 1) unstacked input 11 shape (?, 1) unstacked input 12 shape (?, 1) unstacked input 13 shape (?, 1) unstacked input 14 shape (?, 1) unstacked input 15 shape (?, 1) unstacked input 16 shape (?, 1) unstacked input 17 shape (?, 1) unstacked input 18 shape (?, 1) unstacked input 19 shape (?, 1) unstacked input 20 shape (?, 1) unstacked input 21 shape (?, 1) unstacked input 22 shape (?, 1) unstacked input 23 shape (?, 1) unstacked input 24 shape (?, 1) unstacked input 25 shape (?, 1) unstacked input 26 shape (?, 1) unstacked input 27 shape (?, 1) unstacked input 28 shape (?, 1) unstacked input 29 shape (?, 1) unstacked input 30 shape (?, 1) unstacked input 31 shape (?, 1) unstacked input 32 shape (?, 1) unstacked input 33 shape (?, 1) unstacked input 34 shape (?, 1) unstacked input 35 shape (?, 1) unstacked input 36 shape (?, 1) unstacked input 37 shape (?, 1) unstacked input 38 shape (?, 1) unstacked input 39 shape (?, 1) unstacked input 40 shape (?, 1) unstacked input 41 shape (?, 1) unstacked input 42 shape (?, 1) unstacked input 43 shape (?, 1) unstacked input 44 shape (?, 1) unstacked input 45 shape (?, 1) unstacked input 46 shape (?, 1) unstacked input 47 shape (?, 1) unstacked input 48 shape (?, 1) unstacked input 49 shape (?, 1) unstacked input 50 shape (?, 1) unstacked input 51 shape (?, 1) unstacked input 52 shape (?, 1) unstacked input 53 shape (?, 1) unstacked input 54 shape (?, 1) unstacked input 55 shape (?, 1) unstacked input 56 shape (?, 1) unstacked input 57 shape (?, 1) unstacked input 58 shape (?, 1) unstacked input 59 shape (?, 1) unstacked input 60 shape (?, 1) unstacked input 61 shape (?, 1) unstacked input 62 shape (?, 1) unstacked input 63 shape (?, 1) unstacked input 64 shape (?, 1) unstacked input 65 shape (?, 1) unstacked input 66 shape (?, 1) unstacked input 67 shape (?, 1) unstacked input 68 shape (?, 1) unstacked input 69 shape (?, 1) unstacked input 70 shape (?, 1) unstacked input 71 shape (?, 1) unstacked input 72 shape (?, 1) unstacked input 73 shape (?, 1) unstacked input 74 shape (?, 1) unstacked input 75 shape (?, 1) unstacked input 76 shape (?, 1) unstacked input 77 shape (?, 1) unstacked input 78 shape (?, 1) unstacked input 79 shape (?, 1) unstacked input 80 shape (?, 1) unstacked input 81 shape (?, 1) unstacked input 82 shape (?, 1) unstacked input 83 shape (?, 1) unstacked input 84 shape (?, 1) unstacked input 85 shape (?, 1) unstacked input 86 shape (?, 1) unstacked input 87 shape (?, 1) unstacked input 88 shape (?, 1) unstacked input 89 shape (?, 1) unstacked input 90 shape (?, 1) unstacked input 91 shape (?, 1) unstacked input 92 shape (?, 1) unstacked input 93 shape (?, 1) unstacked input 94 shape (?, 1) unstacked input 95 shape (?, 1) unstacked input 96 shape (?, 1) unstacked input 97 shape (?, 1) unstacked input 98 shape (?, 1) unstacked input 99 shape (?, 1)
rnn_cell = tf.nn.rnn_cell.BasicRNNCell(10)
outputs, state = tf.nn.static_rnn(rnn_cell, inputs, dtype=tf.float32)
for i, _output in enumerate(outputs):
print('rnn output {} shape {}'.format(i, _output.shape))
reshaped_outputs = tf.reshape(tf.stack(outputs, axis=1), [-1, 10])
print('stacked and reshaped rnn output {}'.format(reshaped_outputs.shape))
out = tf.layers.dense(reshaped_outputs, 1, use_bias=False)
print('output {}'.format(out.shape))
loss = tf.losses.mean_squared_error(signal_next, out)
train_op = tf.train.GradientDescentOptimizer(1e-2).minimize(loss)
rnn output 0 shape (?, 10) rnn output 1 shape (?, 10) rnn output 2 shape (?, 10) rnn output 3 shape (?, 10) rnn output 4 shape (?, 10) rnn output 5 shape (?, 10) rnn output 6 shape (?, 10) rnn output 7 shape (?, 10) rnn output 8 shape (?, 10) rnn output 9 shape (?, 10) rnn output 10 shape (?, 10) rnn output 11 shape (?, 10) rnn output 12 shape (?, 10) rnn output 13 shape (?, 10) rnn output 14 shape (?, 10) rnn output 15 shape (?, 10) rnn output 16 shape (?, 10) rnn output 17 shape (?, 10) rnn output 18 shape (?, 10) rnn output 19 shape (?, 10) rnn output 20 shape (?, 10) rnn output 21 shape (?, 10) rnn output 22 shape (?, 10) rnn output 23 shape (?, 10) rnn output 24 shape (?, 10) rnn output 25 shape (?, 10) rnn output 26 shape (?, 10) rnn output 27 shape (?, 10) rnn output 28 shape (?, 10) rnn output 29 shape (?, 10) rnn output 30 shape (?, 10) rnn output 31 shape (?, 10) rnn output 32 shape (?, 10) rnn output 33 shape (?, 10) rnn output 34 shape (?, 10) rnn output 35 shape (?, 10) rnn output 36 shape (?, 10) rnn output 37 shape (?, 10) rnn output 38 shape (?, 10) rnn output 39 shape (?, 10) rnn output 40 shape (?, 10) rnn output 41 shape (?, 10) rnn output 42 shape (?, 10) rnn output 43 shape (?, 10) rnn output 44 shape (?, 10) rnn output 45 shape (?, 10) rnn output 46 shape (?, 10) rnn output 47 shape (?, 10) rnn output 48 shape (?, 10) rnn output 49 shape (?, 10) rnn output 50 shape (?, 10) rnn output 51 shape (?, 10) rnn output 52 shape (?, 10) rnn output 53 shape (?, 10) rnn output 54 shape (?, 10) rnn output 55 shape (?, 10) rnn output 56 shape (?, 10) rnn output 57 shape (?, 10) rnn output 58 shape (?, 10) rnn output 59 shape (?, 10) rnn output 60 shape (?, 10) rnn output 61 shape (?, 10) rnn output 62 shape (?, 10) rnn output 63 shape (?, 10) rnn output 64 shape (?, 10) rnn output 65 shape (?, 10) rnn output 66 shape (?, 10) rnn output 67 shape (?, 10) rnn output 68 shape (?, 10) rnn output 69 shape (?, 10) rnn output 70 shape (?, 10) rnn output 71 shape (?, 10) rnn output 72 shape (?, 10) rnn output 73 shape (?, 10) rnn output 74 shape (?, 10) rnn output 75 shape (?, 10) rnn output 76 shape (?, 10) rnn output 77 shape (?, 10) rnn output 78 shape (?, 10) rnn output 79 shape (?, 10) rnn output 80 shape (?, 10) rnn output 81 shape (?, 10) rnn output 82 shape (?, 10) rnn output 83 shape (?, 10) rnn output 84 shape (?, 10) rnn output 85 shape (?, 10) rnn output 86 shape (?, 10) rnn output 87 shape (?, 10) rnn output 88 shape (?, 10) rnn output 89 shape (?, 10) rnn output 90 shape (?, 10) rnn output 91 shape (?, 10) rnn output 92 shape (?, 10) rnn output 93 shape (?, 10) rnn output 94 shape (?, 10) rnn output 95 shape (?, 10) rnn output 96 shape (?, 10) rnn output 97 shape (?, 10) rnn output 98 shape (?, 10) rnn output 99 shape (?, 10) stacked and reshaped rnn output (?, 10) output (?, 1)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(1000):
_, _loss = sess.run([train_op, loss], feed_dict={signal: reshaped_sin, signal_next: reshaped_sin_next})
if i%100 == 0:
print('step: {}, loss: {}'.format(i, _loss))
_pred = sess.run(out, feed_dict={signal: reshaped_sin})
_reshaped_pred = np.reshape(_pred, [-1])
plt.plot(_reshaped_pred)
plt.show()
step: 0, loss: 0.40227705240249634 step: 100, loss: 0.010497445240616798 step: 200, loss: 0.008502776734530926 step: 300, loss: 0.007291349116712809 step: 400, loss: 0.006422840058803558 step: 500, loss: 0.005748937837779522 step: 600, loss: 0.005201821215450764 step: 700, loss: 0.004744363483041525 step: 800, loss: 0.004353827331215143 step: 900, loss: 0.004015189129859209