TensorFlow in Practice

TensorFlow In
Practice
Nathan Lintz
nathan@indico.io

Inputs
Parameters and
Operations
Outputs

Inputs
Parameters and
Operations
Outputs
Cost

Batter
Cake
Doneness
Doneness
Temperature
Mush
Perfect
Burnt

Batter
Cake
Doneness
Doneness
Temperature
Mush
Perfect
Burnt
𝑦 = 𝑚𝑥 + 𝑏 ?

Inputs
(x)
(placeholders)
Parameters and
Operations
(m, b)
Outputs
(y_predict)
Cost
y_target
(doneness)

Placeholders
Parameters + Operations
Cost
Optimization
Train
TensorFlow in 5 Easy Pieces

Inputs
(placeholders) import tensorflow as tf
temp = tf.placeholder(tf.float32, [10, 1])
cake_doneness = tf.placeholder(tf.float32, [10, 1])

import tensorflow as tf
temp_m = tf.get_variable(‘temp_m’, [1, 1])
temp_b = tf.get_variable(‘temp_b’, [1])
predicted_output = tf.nn.xw_plus_b(temp, temp_m ,
temp_b)
Parameters and
Operations
(m, b)
Outputs
(y)

cake_doneness = tf.placeholder(tf.float32, [10, 1]
predicted_output = tf.nn.xw_plus_b(temp, temp_m , temp_b)
cost = tf.reduce_mean((cake_doneness –
predicted_output)**2)
Cost

cost = tf.reduce_mean((cake_doneness –predicted_output)**2)
optimizer =
tf.train.GradientDescentOptimizer(learning_rate=0.01).
minimize(cost)
Optimizer

cost = tf.reduce_mean((cake_doneness –predicted_output) 2)
optimizer =
tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)
sess = tf.Session()
sess.run(tf.initialize_all_variables())
temp_train = np.linspace(0, 10, 10).reshape(-1, 1)
doneness_train = temp_observe * 5. + 1. +
np.random.randn(10, 1)
for _ in range(100): sess.run(optimizer, feed_dict={temp:
temp_train, cake_doneness: doneness_train})
predicted_doneness = sess.run(predicted_output,
feed_dict={temp: temp_train})
Train Code

Batter
Cake
Doneness
Doneness
Temperature
Mush
Perfect
Burnt
𝑦 = 𝑚𝑥 + 𝑏
m = 4.99
b = 1.21

Handwritten Digit
(28 x 28 pixels) -> 784 pixels Predicted Digit Value

X (pixels)
[784]
softmax(
mx + b)
m b
Y_true
[10]

from tensorflow.examples.tutorials.mnist import
input_data
mnist = input_data.read_data_sets('MNIST_data',
one_hot=True)
X = tf.placeholder(tf.float32, [128, 784])
Y_true = tf.placeholder(tf.float32, [128, 10])
Placeholders

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
m = tf.get_variable('m', [784, 10])
b = tf.get_variable('b', [10])
Y_pred = tf.nn.xw_plus_b(X, m, b)
Parameters
and
Operations

cost =
tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
Y_pred, Y_true))
Cost

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(Y_pred,
Y_true))
optimzer =
tf.train.GradientDescentOptimizer(learning_rate=0.5)
.minimize(cost)
Optimizer

Y_true))
optimzer =
tf.train.GradientDescentOptimizer(learning_rate=0.5) .minimize(cost)
sess = tf.Session()
for i in range(2000):
trX, trY = mnist.train.next_batch(128)
sess.run(optimzer, feed_dict={X: trX, Y_true: trY})
Train Code

Relu
𝑌 =
𝑥 𝑖𝑓 𝑥 > 0
0 𝑒𝑙𝑠𝑒

X (pixels)
[784]
softmax(
m1h + b1)
m0 b0
Y_true
[10]
m1 b1
relu(
m0x + b0)
h
hidden layer classifier layer

Placeholders

m0 = tf.get_variable('m0', [784, 256])
b0 = tf.get_variable('b0', [256],
initializer=tf.constant_initializer(0.))
h = tf.nn.relu(tf.nn.xw_plus_b(X, m0, b0))
Y_pred = tf.nn.xw_plus_b(h, m1, b1)
Parameters
and
Operations

b0 = tf.get_variable('b0', [256], initializer=tf.constant_initializer(0.))
cost =
tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
Y_pred, Y_true))
Cost

Y_true))
optimzer =
tf.train.GradientDescentOptimizer(learning_rate=0.5)
.minimize(cost)
Optimizer

Y_true))
optimzer =
tf.train.GradientDescentOptimizer(learning_rate=0.5) .minimize(cost)
sess = tf.Session()
Train Code

97% Test Accuracy!
(97% train accuracy)

m0 m1 m2b0 b1 b2
X
(pixels)
[784]
relu(
m0x + b0)
relu(
m1x + b1)
softmax(
m2h1 + b2)
Y_true
[10]
h1 h2
hidden layer 1 classifier layerhidden layer 2

def model(X):
h1 = tf.nn.relu(tf.nn.xw_plus_b(X, m0, b0))
h2 = tf.nn.relu(tf.nn.xw_plus_b(h1, m1, b1))
output = tf.nn.xw_plus_b(h2, m2, b2)
return output
Y_pred = model(X)
Parameters
and
Operations
(with 2 hidden
layers)

97% Test Accuracy!

Overfitting
Train cost
Test cost
Cost
Iterations

def model(X, p_keep):
h1 = tf.nn.dropout(h1, p_keep)
return output
Y_pred = model(X, 0.8)
Y_pred_test = model(X, 1.)
Parameters
and
Operations
(with 2 hidden
layers and
dropout)

m0 m1 m2b0 b1 b2
X
(pixels)
[784]
relu(
m0x + b0)
relu(
m1x + b1)
softmax(
m2h1 + b2)
Y_true
[10]
Dropout(h1) Dropout(h2)
hidden layer 1 classifier layerhidden layer 2

98% Test Accuracy!

Scaling Predictions
X (pixels)
[784]
m b
softmax(
mx + b)
Y_true
[10]
cost =
tf.reduce_mean(tf.nn.softmax_cross_entrop
y_with_logits(Y_pred, Y_true))
VS.
cost =
tf.reduce_mean(tf.nn.softmax_cross_entrop
y_with_logits(tf.nn.softmax(Y_pred)
, Y_true))

Parameter Sharing
return output

m0 m1 m2b0 b1 b2
X
(pixels)
[784]
relu(
m0x + b0)
relu(
m1x + b1)
softmax(
m2h1 + b2)
Y_true
[10]
m0_test m1_test m2_testb0_test b1_test b2_test
X
(pixels)
[784]
relu(
m0x + b0)
relu(
m1x + b1)
softmax(
m2h1 + b2)
Y_true
[10]
Y_pred = model(X, 1.)

m0 m1 m2b0 b1 b2
X
(pixels)
[784]
relu(
m0x + b0)
relu(
m1x + b1)
softmax(
m2h1 + b2)
Y_true
[10]
m0 m1 m2b0 b1 b2
X
(pixels)
[784]
relu(
m0x + b0)
relu(
m1x + b1)
softmax(
m2h1 + b2)
Y_true
[10]
Y_pred = model(X, 1.)

Parameter Sharing (correct)
return output
with tf.variable_scope(“model”) as scope:
scope.reuse_variables()

Collections
def model(X):
tf.add_to_collection(“activations”, h1)
tf.add_to_collection(“activations”, h2)
return output
Y_pred = model(X)

Collections
activations = tf.get_collection(‘activations’)
activations_values = session.run(activations)
parameters = tf.get_collection(‘trainable_parameters’)
parameter_values = session.run(parameters)

Placeholders

X = tf.placeholder(tf.float32, [None, 784])
Placeholders

Placeholders
model = …
cost = …
optimizer = …

Advanced Tensorflow: Building RNNs
Note – Most of the code for the generation is “pseudo-code” meant mostly
to illustrate my point. If you wish to see the actual code, feel free to email
me and I’ll send you a copy.

RNNs
“The food at the restaurant,
was very good”
[1]

RNNs
[The, food, at, the, restaurant,
was, very, good]
[1]

RNNs
[The, food, at, the, restaurant,
was, very, good]
[1]
t = 7
t = 0 t = 1

RNNs
𝑌𝑡 = 𝑡𝑎𝑛ℎ(𝑚 𝑥 𝑋𝑡 + 𝑚ℎℎ 𝑡−1 + 𝑏)
mxXt +
mhht-1+
b
Xt ht
ht-1

RNNs
X = tf.placeholder(tf.float32, [28, 128, 28])
X_split = [tf.squeeze(x) for x in tf.split(0, 28, X)]
rnn = tf.nn.rnn_cell.BasicRNNCell(256, 28)
outputs, states = tf.nn.rnn(rnn, X_split, dtype=tf.float32)

Scan
elems = [1, 2, 3, 4, 5, 6]
def step(a, x):
return a + x
sum = scan(step, elems)
>>> sum = [1, 3, 6, 10, 15, 21]

RNNs with Scan
X = tf.placeholder(tf.float32, [28, 128, 28])
m_x = tf.get_variable(‘m_x’, [28, 256])
m_h = tf.get_variable(‘m_h’, [256, 256])
def step(h_tm1, x):
return tf.tanh(tf.nn.xw_plus_b(x, m_x, b_x) +
tf.nn.xw_plus_b(h_tm1, m_h, b_h))
states = tf.scan(step, X, initializer=tf.zeros(256))
𝑌𝑡 = 𝑡𝑎𝑛ℎ(𝑚 𝑥 𝑋𝑡 + 𝑚ℎℎ 𝑡−1 + 𝑏)

MNIST Generation
t = 28
t = 0
t = 28
t = 0

X = tf.placeholder(tf.float32, [27, 128, 28]) # first 27 rows of image
Y = tf.placeholder(tf.float32, [27, 128, 28]) # last 27 rows of image
m_output = tf.get_variable(tf.float32, [256, 28])
b_output = tf.get_variable(tf.float32, [28])
states = rnn(X)
output_img = tf.map_fn(lambda x: tf.nn.xw_plus_b(x, m_output,
b_output),
tf.pack(states))
cost =
tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(output_
img, Y))
Language Model

def generate(num_steps):
states = [tf.zeros([batch_size, hidden_dim])]
for _ in range(num_steps):
next_output = tf.sigmoid(tf.nn.xw_plus_b(states[-1],
m_output,
b_output))
outputs.append(next_output)
state = gru.step_(states[-1], outputs[-1]))
states.append(state)
return tf.pack(outputs)
Language Model (Generate)

Seq2Seq
Language
Model
(RNN)
Encoder
(RNN)
Input
Digit
Output
Digit
Take Final
State

X = tf.placeholder(tf.float32, [27, 128, 28]) # first 27 rows of image
Y_in = tf.placeholder(tf.float32, [27, 128, 28]) # first 27 rows of target image
Y_out = tf.placeholder(tf.float32, [27, 128, 28]) # last 27 rows of target image
m_output = tf.get_variable(tf.float32, [256, 28])
b_output = tf.get_variable(tf.float32, [28])
with tf.variable_scope(“encoder”) as scope:
encoded_states = rnn(X)
final_state = tf.reverse(encoded_states, [True, False, False])[0, :, :]
with tf.variable_scope(“decoder”) as scope:
output_states = rnn(Y_in, initializer=final_state)
output_img = tf.map_fn(lambda x: tf.nn.xw_plus_b(x, m_output, b_output),
tf.pack(output_states))
cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(output_img,
Y_out))
Seq2Seq

TensorFlow in Practice

Recommended

Recommended

More Related Content

What's hot

What's hot (20)

Viewers also liked

Viewers also liked (20)

Similar to TensorFlow in Practice

Similar to TensorFlow in Practice (20)

More from indico data

More from indico data (6)

Recently uploaded

Recently uploaded (20)

TensorFlow in Practice

Editor's Notes