IST 597 Tensorflow Tutorial Recurrent Neural Networks

Define hyperparameters Example 1 Vanilla RNN in MNIST classification import tensorflow as tf from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets('MNIST_data', one_hot=True) lr = 0.001 training_iters = 100000 batch_size = 128 n_inputs = 28 n_steps = 28 n_hidden_units = 128 n_classes = 10 x = tf.placeholder(tf.float32, [None, n_steps, n_inputs]) y = tf.placeholder(tf.float32, [None, n_classes]) Step 1 Load Data & Define hyperparameters

weights = { 'in': tf. Variable(tf weights = { 'in': tf.Variable(tf.random_normal([n_inputs, n_hidden_units])), 'out': tf.Variable(tf.random_normal([n_hidden_units, n_classes])) } biases = { 'in': tf.Variable(tf.constant(0.1, shape=[n_hidden_units, ])), 'out': tf.Variable(tf.constant(0.1, shape=[n_classes, ])) } def RNN(X, weights, biases): X = tf.reshape(X, [-1, n_inputs]) X_in = tf.matmul(X, weights['in']) + biases['in'] X_in = tf.reshape(X_in, [-1, n_steps, n_hidden_units]) cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden_units) init_state = cell.zero_state(batch_size, dtype=tf.float32) outputs, final_state = tf.nn.dynamic_rnn(cell, X_in, initial_state=init_state, time_major=False) outputs = tf.unstack(tf.transpose(outputs, [1,0,2])) results = tf.matmul(outputs[-1], weights['out']) + biases['out'] return results Step 2 Build Model

Loss Calculation & Start Training pred = RNN(x, weights, biases) cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y)) train_op = tf.train.AdamOptimizer(lr).minimize(cost) correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) step = 0 while step * batch_size < training_iters: batch_xs, batch_ys = mnist.train.next_batch(batch_size) batch_xs = batch_xs.reshape([batch_size, n_steps, n_inputs]) sess.run([train_op], feed_dict={x: batch_xs,y: batch_ys,}) if step % 20 == 0: print(sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys,})) step += 1 Step 3 Loss Calculation & Start Training

Example 2 LSTM in MNIST classification def LSTM(X, weights, biases): X = tf.reshape(X, [-1, n_inputs]) X_in = tf.matmul(X, weights['in']) + biases['in'] X_in = tf.reshape(X_in, [-1, n_steps, n_hidden_units]) cell = tf.nn.rnn_cell.BasicLSTMCell(n_hidden_units, forget_bias=1.0) init_state = cell.zero_state(batch_size, dtype=tf.float32) outputs, final_state = tf.nn.dynamic_rnn(cell, X_in, initial_state=init_state, time_major=False) outputs = tf.unstack(tf.transpose(outputs, [1,0,2])) results = tf.matmul(outputs[-1], weights['out']) + biases['out'] return results

Example 3 GRU in MNIST classification def GRU(X, weights, biases): X = tf.reshape(X, [-1, n_inputs]) X_in = tf.matmul(X, weights['in']) + biases['in'] X_in = tf.reshape(X_in, [-1, n_steps, n_hidden_units]) cell = tf.nn.rnn_cell.GRUCell(n_hidden_units) init_state = cell.zero_state(batch_size, dtype=tf.float32) outputs, final_state = tf.nn.dynamic_rnn(cell, X_in, initial_state=init_state, time_major=False) outputs = tf.unstack(tf.transpose(outputs, [1,0,2])) results = tf.matmul(outputs[-1], weights['out']) + biases['out'] return results

Example 4 Bidirectional RNN in MNIST classification def BiRNN(X, weights, biases): X = tf.reshape(X, [-1, n_inputs]) X_in = tf.matmul(X, weights['in']) + biases['in'] X_in = tf.reshape(X_in, [-1, n_steps, n_hidden_units]) lstm_fw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, forget_bias=1.0) lstm_bw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, forget_bias=1.0) outputs, _, _ = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell, lstm_bw_cell, X_in, dtype=tf.float32) outputs = tf.unstack(tf.transpose(outputs, [1,0,2])) results = tf.matmul(outputs[-1], weights['out']) + biases['out'] return results

Define hyperparameters Example 5 Deep LSTM in MNIST classification import tensorflow as tf from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets('MNIST_data', one_hot=True) lr = 1e-3 batch_size = tf.placeholder(tf.int32) input_size = 28 timestep_size = 28 hidden_size = 256 layer_num = 2 class_num = 10 _X = tf.placeholder(tf.float32, [None, 784]) y = tf.placeholder(tf.float32, [None, class_num]) keep_prob = tf.placeholder(tf.float32) X = tf.reshape(_X, [-1, 28, 28]) Step 1 Load Data & Define hyperparameters

lstm_cell = tf. nn. rnn_cell lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size, forget_bias=1.0, state_is_tuple=True) lstm_cell = tf.nn.rnn_cell.DropoutWrapper(cell=lstm_cell, input_keep_prob=1.0, output_keep_prob=keep_prob) mlstm_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell] * layer_num, state_is_tuple=True) init_state = mlstm_cell.zero_state(batch_size, dtype=tf.float32) outputs = list() state = init_state with tf.variable_scope('RNN'): for timestep in range(timestep_size): if timestep > 0: tf.get_variable_scope().reuse_variables() (cell_output, state) = mlstm_cell(X[:, timestep, :], state) outputs.append(cell_output) h_state = outputs[-1] W = tf.Variable(tf.truncated_normal([hidden_size, class_num], stddev=0.1), dtype=tf.float32) bias = tf.Variable(tf.constant(0.1,shape=[class_num]), dtype=tf.float32) y_pre = tf.nn.softmax(tf.matmul(h_state, W) + bias) cross_entropy = -tf.reduce_mean(y * tf.log(y_pre)) train_op = tf.train.AdamOptimizer(lr).minimize(cross_entropy) correct_prediction = tf.equal(tf.argmax(y_pre,1), tf.argmax(y,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) Step 2 Build Model & Loss Calculation

Step 3 Start Training sess = tf.Session() sess.run(tf.global_variables_initializer()) for i in range(2000): _batch_size = 128 batch = mnist.train.next_batch(_batch_size) if (i+1)%10 == 0: train_accuracy = sess.run(accuracy, feed_dict={_X:batch[0], y: batch[1], keep_prob: 1.0, batch_size: _batch_size}) print("Iter%d, step %d, training accuracy %g" % ( mnist.train.epochs_completed, (i+1), train_accuracy)) sess.run(train_op, feed_dict={_X: batch[0], y: batch[1], keep_prob: 0.5, batch_size: _batch_size}) print("test accuracy %g"% sess.run(accuracy, feed_dict={ _X: mnist.test.images, y: mnist.test.labels, keep_prob: 1.0, batch_size:mnist.test.images.shape[0]}))

Thanks !