CS 20 : TensorFlow for Deep Learning Research¶

Lecture 02 : Operations¶

Setup¶

In [1]:

from __future__ import absolute_import, division, print_function
import numpy as np
import tensorflow as tf

print(tf.__version__)
sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))

1.12.0

1. Fun with TensorBoard¶

In [2]:

# Nodes: operators, variables, and constants 
# Edges: tensors
a = tf.constant(2)
b = tf.constant(3)
x = tf.add(a,b)
print(a, b, x)
with tf.Session(config = sess_config) as sess:
    print(sess.run(x))

Tensor("Const:0", shape=(), dtype=int32) Tensor("Const_1:0", shape=(), dtype=int32) Tensor("Add:0", shape=(), dtype=int32)
5

In [3]:

# To visualizer above program with Tensorboard
tf.reset_default_graph()

a = tf.constant(2)
b = tf.constant(3)
x = tf.add(a,b)
print(a, b, x)

# Create the summary writer after graph definition and before running your
# session
writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/add_example',
                      graph = tf.get_default_graph())
writer.close()
with tf.Session(config = sess_config) as sess:
    print(sess.run(x))

Tensor("Const:0", shape=(), dtype=int32) Tensor("Const_1:0", shape=(), dtype=int32) Tensor("Add:0", shape=(), dtype=int32)
5

2. Constant op¶

It’s straightforward to create a constant in TensorFlow

In [4]:

# constant of 1d tensor (vector)
a = tf.constant([2, 2], name = 'vector')
print(a)
with tf.Session(config = sess_config) as sess:
    print(sess.run(a))

Tensor("vector:0", shape=(2,), dtype=int32)
[2 2]

In [5]:

# constant of 2x2 tensor (matrix)
b = tf.constant([[0,1], [2,3]], name = 'matrix')
print(b)
with tf.Session(config = sess_config) as sess:
    print(sess.run(b))

Tensor("matrix:0", shape=(2, 2), dtype=int32)
[[0 1]
 [2 3]]

In [6]:

# create a tensor of shape and all elements are zeros
print(tf.zeros(shape = [2,3], dtype = tf.int32))
with tf.Session(config = sess_config) as sess:
    print(sess.run(tf.zeros(shape = [2,3], dtype = tf.int32)))

Tensor("zeros:0", shape=(2, 3), dtype=int32)
[[0 0 0]
 [0 0 0]]

In [7]:

# create a tensor of shape and type (unless type is specified) as the input_tensor but all
# elements are zeros.
# input_tensor [[0, 1], [2, 3], [4, 5]]
input_tensor = [[0, 1], [2, 3], [4, 5]]
print(tf.zeros_like(input_tensor))
with tf.Session(config = sess_config) as sess:
    print(sess.run(tf.zeros_like(input_tensor)))

Tensor("zeros_like:0", shape=(3, 2), dtype=int32)
[[0 0]
 [0 0]
 [0 0]]

In [8]:

# create a tensor of shape and all elements are ones
print(tf.ones(shape = [2,3], dtype = tf.int32))
with tf.Session(config = sess_config) as sess:
    print(sess.run(tf.ones(shape = [2,3], dtype = tf.int32)))

Tensor("ones:0", shape=(2, 3), dtype=int32)
[[1 1 1]
 [1 1 1]]

In [9]:

# create a tensor of shape and type (unless type is specified) as the input_tensor but all
# elements are ones.
# input_tensor is [[0, 1], [2, 3], [4, 5]]
print(tf.ones_like(input_tensor))
with tf.Session(config = sess_config) as sess:
    print(sess.run(tf.ones_like(input_tensor)))

Tensor("ones_like:0", shape=(3, 2), dtype=int32)
[[1 1]
 [1 1]
 [1 1]]

In [10]:

# create a tensor filled with a scalar value.
print(tf.fill([2,3], 8))
with tf.Session(config = sess_config) as sess:
    print(sess.run(tf.fill([2,3], 8)))

Tensor("Fill:0", shape=(2, 3), dtype=int32)
[[8 8 8]
 [8 8 8]]

You can create constants that are sequences

In [11]:

'''
create a sequence of num evenly-spaced values are generated beginning at start. If num >
1, the values in the sequence increase by (stop - start) / (num - 1), so that the last one
is exactly stop. comparable to but slightly different from numpy.linspace
'''
print(tf.lin_space(10., 13., 4))
with tf.Session(config = sess_config) as sess:
    print(sess.run(tf.lin_space(10., 13., 4)))

Tensor("LinSpace:0", shape=(4,), dtype=float32)
[10. 11. 12. 13.]

In [12]:

# create a sequence of numbers that begins at start and extends by increments of delta up to
# but not including limit
print(tf.range(5)) 
with tf.Session(config = sess_config) as sess:
    print(sess.run(tf.range(5))) # TensorFlow sequences are not iterable

Tensor("range:0", shape=(5,), dtype=int32)
[0 1 2 3 4]

In [13]:

with tf.Session(config = sess_config) as sess:
    print(sess.run(tf.range(3, 18, 3)))
    print(sess.run(tf.range(3, 1, -.5)))
    print(sess.run(tf.range(5)))

[ 3  6  9 12 15]
[3.  2.5 2.  1.5]
[0 1 2 3 4]

In [14]:

for _ in tf.range(4):
    print(_)

--------------------------------------
TypeErrorTraceback (most recent call last)
<ipython-input-14-b1a040fb85ad> in <module>
----> 1 for _ in tf.range(4):
      2     print(_)

/usr/local/var/pyenv/versions/3.6.6/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in __iter__(self)
    457     if not context.executing_eagerly():
    458       raise TypeError(
--> 459           "Tensor objects are only iterable when eager execution is "
    460           "enabled. To iterate over this tensor use tf.map_fn.")
    461     shape = self._shape_tuple()

TypeError: Tensor objects are only iterable when eager execution is enabled. To iterate over this tensor use tf.map_fn.

3. Math Operations¶

recommend reading the lecture notes

4. Data Types¶

recommend reading the lecture notes

5. Variables¶

In [15]:

tf.reset_default_graph()

In [16]:

# Create variables
# Note that when we use tf.constant as an initializer,
# we don't need to provide shape
s = tf.get_variable(name = 'scalar',
                    initializer = tf.constant(2))
m = tf.get_variable(name = 'matrix',
                    initializer = tf.constant([[0, 1], [2, 3]]))
W = tf.get_variable(name = 'big_matrix', shape = [784, 10],
                    initializer = tf.zeros_initializer())

In [17]:

# Initialize variables
with tf.Session(config = sess_config) as sess:
    print(sess.run(tf.report_uninitialized_variables()))

[b'scalar' b'matrix' b'big_matrix']

In [18]:

with tf.Session(config = sess_config) as sess:
    sess.run(tf.global_variables_initializer())
    print(sess.run(tf.report_uninitialized_variables()))

[]

In [19]:

# Interesting Assign example
tf.reset_default_graph()
W = tf.Variable(10)
W.assign(100)

with tf.Session(config = sess_config) as sess:
    sess.run(W.initializer)
    print(W.eval())

In [20]:

tf.reset_default_graph()
W = tf.Variable(10)
assign_op = W.assign(100)

with tf.Session(config = sess_config) as sess:
    sess.run(assign_op)
    print(W.eval())

In [21]:

tf.reset_default_graph()
W = tf.get_variable(name = 'weights', initializer = tf.constant([[1.,1.],[2.,2.]]))
tf.get_variable_scope().reuse_variables()
w = tf.get_variable(name = 'weights')
writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/get_variables',
                      graph = tf.get_default_graph())
writer.close()

with tf.Session(config = sess_config) as sess:
    sess.run(tf.global_variables_initializer())
    print(W, sess.run(W))
    print(w, sess.run(w))

<tf.Variable 'weights:0' shape=(2, 2) dtype=float32_ref> [[1. 1.]
 [2. 2.]]
<tf.Variable 'weights:0' shape=(2, 2) dtype=float32_ref> [[1. 1.]
 [2. 2.]]

6. Interactive Session¶

recommend reading the lecture notes

7. Control Dependencies¶

recommend reading the lecture notes

8. Importing Data¶

tf.placeholder

In [22]:

tf.reset_default_graph()
a = tf.placeholder(dtype = tf.float32, shape = [3])
b = tf.constant(value = [5, 5, 5], dtype = tf.float32)
c = a + b

writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/placeholder',
                               graph = tf.get_default_graph())

with tf.Session(config = sess_config) as sess:
    print(sess.run(c, feed_dict = {a : [1.,2.,3.]}))

[6. 7. 8.]

In [23]:

# You can feed values to tensors that aren't placeholders.
# Any tensors that are feedable can be fed
tf.reset_default_graph()
a = tf.add(2,5)
b = tf.multiply(a, 3)

with tf.Session(config = sess_config) as sess:
    print(sess.run(b))
    print(sess.run(b, feed_dict = {a : 15}))

21
45

In [24]:

with tf.Session(config = sess_config) as sess:
    print(sess.graph.is_feedable(a), sess.graph.is_feedable(b))

True True

9. The trap of lazy loading¶

Lazy loading is a term that refers to a programming pattern when you defer declaring/initializing an object until it is loaded

In [25]:

# normal loading
tf.reset_default_graph()
x = tf.get_variable(name = 'x', initializer = tf.constant(10))
y = tf.get_variable(name = 'y', initializer = tf.constant(20))
z = tf.add(x,y)

with tf.Session(config = sess_config) as sess:
    sess.run(tf.global_variables_initializer())
    writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/normal_loading',
                                   graph = tf.get_default_graph())
    
    for _ in range(3):
        sess.run(z)
    else:
        print(tf.get_default_graph().get_operations())
        writer.close()

[<tf.Operation 'Const' type=Const>, <tf.Operation 'x' type=VariableV2>, <tf.Operation 'x/Assign' type=Assign>, <tf.Operation 'x/read' type=Identity>, <tf.Operation 'Const_1' type=Const>, <tf.Operation 'y' type=VariableV2>, <tf.Operation 'y/Assign' type=Assign>, <tf.Operation 'y/read' type=Identity>, <tf.Operation 'Add' type=Add>, <tf.Operation 'init' type=NoOp>]

In [26]:

# lazy loading
tf.reset_default_graph()
x = tf.get_variable(name = 'x', initializer = tf.constant(10))
y = tf.get_variable(name = 'y', initializer = tf.constant(20))

with tf.Session(config = sess_config) as sess:
    sess.run(tf.global_variables_initializer())
    writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/lazy_loading',
                                   graph = sess.graph)
    for _ in range(3):
        sess.run(tf.add(x,y))
    else:
        print(tf.get_default_graph().get_operations())
        writer.close()

[<tf.Operation 'Const' type=Const>, <tf.Operation 'x' type=VariableV2>, <tf.Operation 'x/Assign' type=Assign>, <tf.Operation 'x/read' type=Identity>, <tf.Operation 'Const_1' type=Const>, <tf.Operation 'y' type=VariableV2>, <tf.Operation 'y/Assign' type=Assign>, <tf.Operation 'y/read' type=Identity>, <tf.Operation 'init' type=NoOp>, <tf.Operation 'Add' type=Add>, <tf.Operation 'Add_1' type=Add>, <tf.Operation 'Add_2' type=Add>]