conv_net

May 6, 2019

In [1]: import tensorflow as tf

import matplotlib.pyplot as plt
import numpy as np
%matplotlib inline

# Convert classes to indicator vectors

def one_hot(values,n_values=10):
n_v = np.maximum(n_values,np.max(values) + 1)
oh=np.eye(n_v)[values]
return oh

print(tf.__version__)

/Users/amit/anaconda2/envs/myenv/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: comp

return f(*args, **kwds)

1.10.1

1 Get Mnist data and split into train validation and test
In [ ]: def get_mnist():

data=np.float64(np.load('/Users/amit/Box Sync/tex/courses/LSDA/data_sets/mnist/MNIST
labels=np.float32(np.load('/Users/amit/Box Sync/tex/courses/LSDA/data_sets/mnist/MNI
print(data.shape)
data=np.float32(data)/255.
train_dat=data[0:50000]
train_labels=one_hot(np.int32(labels[0:50000]))
val_dat=data[50000:60000]
val_labels=one_hot(np.int32(labels[50000:60000]))
test_dat=data[60000:70000]
test_labels=one_hot(np.int32(labels[60000:70000]))

return (train_dat, train_labels), (val_dat, val_labels), (test_dat, test_labels)

1
2 Get CIFAR10 data and split into train, validation and test
In [3]: def get_cifar():
tr=np.float32(np.load('../../mnist/CIFAR_10.npy'))
tr_lb=np.int32(np.load('/project/cmsc25025/mnist/CIFAR_labels.npy'))
tr=tr.reshape((-1,np.prod(np.array(tr.shape)[1:4])))
train_data=tr[0:45000]/255.
train_labels=one_hot(tr_lb[0:45000])
val_data=tr[45000:]/255.
val_labels=one_hot(tr_lb[45000:])
test_data=np.float32(np.load('/project/cmsc25025/mnist/CIFAR_10_test.npy'))
test_data=test_data.reshape((-1,np.prod(np.array(test_data.shape)[1:4])))
test_data=test_data/255.
test_labels=one_hot(np.int32(np.load('/project/cmsc25025/mnist/CIFAR_labels_test.npy
return (train_data, train_labels), (val_data, val_labels), (test_data, test_labels)

3 Get the data

In [6]: def get_data(data_set):
if (data_set=="cifar"):
return(get_cifar())
elif (data_set=="mnist"):
return(get_mnist())
elif (data_set=="mnist_transform"):
return(get_mnist_trans())

4 Convolution layer with relu

In [3]: def conv_relu_layer(input,filter_size=[3,3],num_features=[1]):

# Get number of input features from input and add to shape of new layer
shape=filter_size+[input.get_shape().as_list()[-1],num_features]
W = tf.get_variable('W',shape=shape) # Default initialization is Glorot (the one exp
b = tf.get_variable('b',shape=[num_features],initializer=tf.zeros_initializer)
conv = tf.nn.conv2d(input, W, strides=[1, 1, 1, 1], padding='SAME')
relu = tf.nn.relu(conv + b)
return(relu)

5 Fully connected layer

In [4]: def fully_connected_layer(input,num_features):
# Make sure input is flattened.
flat_dim=np.int32(np.array(input.get_shape().as_list())[1:].prod())
input_flattened = tf.reshape(input, shape=[-1,flat_dim])
shape=[flat_dim,num_features]
W_fc = tf.get_variable('W',shape=shape)

2
 b_fc = tf.get_variable('b',shape=[num_features],initializer=tf.zeros_initializer)
fc = tf.matmul(input_flattened, W_fc) + b_fc
return(fc)

6 The network
In [5]: #tf.reset_default_graph()

def create_network():
pool_ksize=[1,2,2,1]
pool_strides=[1,2,2,1]
# The network:
with tf.variable_scope("conv1"):
# Variables created here will be named "conv1/weights", "conv1/biases".
relu1 = conv_relu_layer(x_image, filter_size=[5, 5],num_features=32)
pool1 = tf.nn.max_pool(relu1, ksize=pool_ksize, strides=pool_strides, paddin
with tf.variable_scope("conv2"):
# Variables created here will be named "conv1/weights", "conv1/biases".
relu2 = conv_relu_layer(pool1, filter_size=[5, 5],num_features=64)
pool2 = tf.nn.max_pool(relu2, ksize=pool_ksize, strides=pool_strides, paddin
with tf.variable_scope('dropout2'):
drop2=tf.nn.dropout(pool2,keep_prob)
with tf.variable_scope("fc1"):
fc1 = fully_connected_layer(drop2, num_features=256)
fc1r=tf.nn.relu(fc1)

with tf.variable_scope("fc2"):
fc2 = fully_connected_layer(fc1r, num_features=10)

# Names (OUT,LOSS, ACC) below added to make it easier to use this tensor when restor
fc2 = tf.identity(fc2, name="OUT")
# The loss computation
with tf.variable_scope('cross_entropy_loss'):
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_

# Accuracy computation
with tf.variable_scope('helpers'):
correct_prediction = tf.equal(tf.argmax(fc2, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32),name="ACC")
# We return the final functions (they contain all the information about the graph of
return cross_entropy, accuracy

7 Run one epoch

In [12]: # Run the iterations of one epoch
def run_epoch(sess,OPS,data,batch_size, type="train", num=None):
t1=time.time()

3
 n=data[0].shape[0]
if (num is not None):
n=np.minimum(n,num)
ii=np.array(np.arange(0,n,1))

# Randomly shuffle the training data

if (type=="train"):
np.random.shuffle(ii)
tr=data[0][ii]
y=data[1][ii]
loo=0.
acc=0.
nb=0.
# Run disjoint batches on shuffled data
for j in np.arange(0,len(y),batch_size):
if (np.mod(j,5000)==0):
print('Batch',j/batch_size)
batch=(tr[j:j+batch_size],y[j:j+batch_size])
if (type=="train"):
lo,ac,_=sess.run([OPS['cross_entropy'],OPS['accuracy'],OPS['train_step'
else:
lo,ac = sess.run([OPS['cross_entropy'],OPS['accuracy']],feed_dict={x: b
loo+=lo
acc+=ac
nb+=1
loo=loo/nb
acc=acc/nb
print('Epoch time',time.time()-t1)
return loo, acc

8 Run the training. Save the model and test at the end
In [ ]: import time
batch_size=500
step_size=.001
num_epochs=4
num_train=10000
minimizer="Adam"
data_set="mnist"
model_name="model"
keep_prob=.5
dim=28
nchannels=1
if (data_set=="cifar"):
dim=32
nchannels=3
train,val,test=get_data(data_set=data_set)

4
tf.reset_default_graph()
x = tf.placeholder(tf.float32, shape=[None, dim*dim*nchannels],name="x")
x_image = tf.reshape(x, [-1, dim, dim, nchannels])

# Dimensions of x_image: [Batch size, Column size, Row size, Number of incoming channels
# The number of incoming channels, for example, will be 3 if the image is color: RGB (re
# We will slide filter over this 2d picture with conv2d function.
y_ = tf.placeholder(tf.float32, shape=[None,10],name="y")
# Allows you to control the time step during the iterations
lr_ = tf.placeholder(tf.float32, shape=[],name="learning_rate")
keep_prob_=tf.placeholder(tf.float32, shape=[],name="keep_prob")

with tf.Session() as sess:

# Create the network architecture with the above placeholdes as the inputs.
cross_entropy, accuracy=create_network()

# Define the miminization method

if (minimizer=="Adam"):
train_step=tf.train.AdamOptimizer(learning_rate=lr_).minimize(cross_entropy)
elif (minimizer=="SGD"):
train_step = tf.train.GradientDescentOptimizer(learning_rate=lr_).minimize(cross
OPS={}
OPS['cross_entropy']=cross_entropy; OPS['accuracy']=accuracy; OPS['train_step']=trai
# Initialize variables
sess.run(tf.global_variables_initializer())
# Show trainable variables
for v in tf.trainable_variables():
print(v.name,v.get_shape().as_list(),np.std(v.eval()))

# Run epochs
for i in range(num_epochs): # number of epochs
lo,ac=run_epoch(sess,OPS,train,batch_size,type="train",num=num_train)
print('Epoch',i,'Train loss, accuracy',lo,ac)
vlo,vac = run_epoch(sess,OPS,val,batch_size,type="test")
print('Epoch',i,'Validation loss, accuracy',vlo,vac)
# Test set accuracy
telo,teac = run_epoch(sess,OPS,test,batch_size,type="test")
print('test accuracy',telo,teac)

# Save model
tf.add_to_collection("optimizer", train_step)
saver = tf.train.Saver()
save_path = saver.save(sess, "tmp/"+model_name)
print("Model saved in path: %s" % save_path)

5
9 Reload the model that was saved and continue training
In [ ]: # Reloading an existing model.

tf.reset_default_graph()
batch_size=500
step_size=.001
num_epochs=4
num_train=10000
data_set="mnist"
model_name="model"
Train=True
dim=28
nchannels=1
if (data_set=="cifar"):
dim=32
nchannels=3

with tf.Session() as sess:

# Get data
train, val, test=get_data(data_set=data_set)
# Load model info
saver = tf.train.import_meta_graph('tmp/'+model_name+'.meta')
saver.restore(sess,'tmp/'+model_name)
graph = tf.get_default_graph()
# Setup the placeholders from the stored model.
x = graph.get_tensor_by_name('x:0')
y_= graph.get_tensor_by_name('y:0')
lr_ = graph.get_tensor_by_name('learning_rate:0')
keep_prob_ = graph.get_tensor_by_name('keep_prob:0')
OPS={}
OPS['accuracy']=graph.get_tensor_by_name('helpers/ACC:0')
OPS['cross_entropy']=graph.get_tensor_by_name('cross_entropy_loss/LOSS:0')

# Get the minimization operation from the stored model

if (Train):
OPS['train_step'] = tf.get_collection("optimizer")[0]
# Confirm training accuracy of current model before additional training
acc=OPS['accuracy'].eval(feed_dict={x: train[0][0:num_train], y_:train[1][0:num_
print('train acc',acc)

ii=np.arange(0,num_train,1)
for i in range(num_epochs): # Run epochs
lo,ac=run_epoch(sess,OPS,train,batch_size,type="train",num=num_train)
print('Epoch',i,'Train loss, accuracy',lo,ac)
vlo,vac = run_epoch(sess,OPS,val,batch_size,type="test")
print('EPoch',i,'Validation loss, accuracy',vlo,vac)

6
# Test set accuracy
telo,teac = run_epoch(sess,OPS,test,batch_size,type="test")
print('test accuracy',telo,teac)

tf.add_to_collection("optimizer", train_step)
saver = tf.train.Saver()
save_path = saver.save(sess, "tmp/"+model_name)
print("Model saved in path: %s" % save_path)