deeplearning

July 7, 2024

[2]: import numpy as np

import matplotlib.pyplot as plt
import random

from keras.datasets import mnist

from keras.models import Sequential

from keras.layers import Dense, Dropout, Activation

from tensorflow.keras.utils import to_categorical

[3]: # The MNIST data is split between 60,000 28 x 28 pixel training images and␣
↪10,000 28 x 28 pixel images

(X_train, y_train), (X_test, y_test) = mnist.load_data()

print("X_train shape", X_train.shape)

print("y_train shape", y_train.shape)
print("X_test shape", X_test.shape)
print("y_test shape", y_test.shape)

Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-

datasets/mnist.npz
11490434/11490434 [==============================] - 0s 0us/step
X_train shape (60000, 28, 28)
y_train shape (60000,)
X_test shape (10000, 28, 28)
y_test shape (10000,)

[4]: plt.rcParams['figure.figsize'] = (9,9)

for i in range(9):
plt.subplot(3,3,i+1)
num = random.randint(0, len(X_train))
plt.imshow(X_train[num], cmap='gray', interpolation='none')
plt.title("Class {}".format(y_train[num]))

plt.tight_layout()

1
[5]: # just a little function for pretty printing a matrix
def matprint(mat, fmt="g"):
col_maxes = [max([len(("{:"+fmt+"}").format(x)) for x in col]) for col in␣
↪mat.T]

for x in mat:
for i, y in enumerate(x):
print(("{:"+str(col_maxes[i])+fmt+"}").format(y), end=" ")
print("")

# now print!
matprint(X_train[num])

2
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 114 12 89 207 253
176 253 253 84 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 29 85 101 225 253 152 197 252 252
253 252 252 84 0 0 0 0 0 0
0 0 0 0 19 166 197 197 198 215 252 252 252 196 161 43 149 195
253 252 252 84 0 0 0 0 0 0
0 0 0 0 178 252 252 252 253 252 252 138 84 0 47 0 0 0
253 252 252 84 0 0 0 0 0 0
0 0 0 0 146 252 252 252 253 252 141 37 0 0 0 0 0 0
253 252 220 37 0 0 0 0 0 0
0 0 0 0 0 57 240 140 141 94 0 0 0 0 0 0 13 191
255 253 196 0 0 0 0 0 0 0
0 0 0 0 0 6 24 0 0 0 0 0 0 0 0 0 154 252
253 252 148 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 13 187 252
253 186 12 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 79 252 252
253 151 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 85 252 252 252
112 12 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 147 253 253 253
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 225 252 242 89
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 57 231 252 192 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 16 253 252 214 28 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 140 253 252 118 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 92 253 255 215 31 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 19 196 252 253 167 0 0 0
0 0 0 0 0 0 0 0 0 0

3
 0 0 0 0 0 0 0 0 0 0 57 252 252 253 167 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 57 252 252 253 74 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 25 205 252 190 12 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0

[6]: X_train = X_train.reshape(60000, 784) # reshape 60,000 28 x 28 matrices into␣

↪60,000 784-length vectors.

X_test = X_test.reshape(10000, 784) # reshape 10,000 28 x 28 matrices into␣

↪10,000 784-length vectors.

X_train = X_train.astype('float32') # change integers to 32-bit floating␣

↪point numbers

X_test = X_test.astype('float32')

X_train /= 255 # normalize each value for each pixel for␣

↪the entire vector for each input

X_test /= 255

print("Training matrix shape", X_train.shape)

print("Testing matrix shape", X_test.shape)

Training matrix shape (60000, 784)

Testing matrix shape (10000, 784)

[7]: nb_classes = 10 # number of unique digits

Y_train = to_categorical(y_train, nb_classes)

Y_test = to_categorical(y_test, nb_classes)

[8]: # The Sequential model is a linear stack of layers and is very common.

model = Sequential()

[9]: # The first hidden layer is a set of 512 nodes (artificial neurons).
# Each node will receive an element from each input vector and apply some␣
↪weight and bias to it.

model.add(Dense(512, input_shape=(784,))) #(784,) is not a typo -- that␣

↪represents a 784 length vector!

[10]: # An "activation" is a non-linear function applied to the output of the layer␣

↪above.

4
 # It checks the new value of the node, and decides whether that artifical␣
↪neuron has fired.

# The Rectified Linear Unit (ReLU) converts all negative inputs to nodes in the␣
↪next layer to be zero.

# Those inputs are then not considered to be fired.

# Positive values of a node are unchanged.

model.add(Activation('relu'))

[11]: # Dropout zeroes a selection of random outputs (i.e., disables their activation)
# Dropout helps protect the model from memorizing or "overfitting" the training␣
↪data.

model.add(Dropout(0.2))

[12]: # The second hidden layer appears identical to our first layer.
# However, instead of each of the 512-node receiving 784-inputs from the input␣
↪image data,

# they receive 512 inputs from the output of the first 512-node layer.

model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.2))

[13]: # The final layer of 10 neurons in fully-connected to the previous 512-node␣

↪layer.

# The final layer of a FCN should be equal to the number of desired classes (10␣
↪in this case).

model.add(Dense(10))

[14]: # The "softmax" activation represents a probability distribution over K␣

↪different possible outcomes.

# Its values are all non-negative and sum to 1.

model.add(Activation('softmax'))

[15]: # Summarize the built model

model.summary()

Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 512) 401920

activation (Activation) (None, 512) 0

5
 dropout (Dropout) (None, 512) 0

dense_1 (Dense) (None, 512) 262656

activation_1 (Activation) (None, 512) 0

dropout_1 (Dropout) (None, 512) 0

dense_2 (Dense) (None, 10) 5130

activation_2 (Activation) (None, 10) 0

=================================================================
Total params: 669706 (2.55 MB)
Trainable params: 669706 (2.55 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

[16]: # Let's use the Adam optimizer for learning

model.compile(loss='categorical_crossentropy', optimizer='adam',␣
↪metrics=['accuracy'])

[17]: model.fit(X_train, Y_train,

batch_size=128, epochs=5,
verbose=1)

Epoch 1/5
469/469 [==============================] - 12s 23ms/step - loss: 0.2507 -
accuracy: 0.9239
Epoch 2/5
469/469 [==============================] - 14s 31ms/step - loss: 0.1006 -
accuracy: 0.9692
Epoch 3/5
469/469 [==============================] - 12s 26ms/step - loss: 0.0721 -
accuracy: 0.9776
Epoch 4/5
469/469 [==============================] - 11s 24ms/step - loss: 0.0566 -
accuracy: 0.9818
Epoch 5/5
469/469 [==============================] - 9s 20ms/step - loss: 0.0466 -
accuracy: 0.9843

[17]: <keras.src.callbacks.History at 0x7f7327872680>

[18]: score = model.evaluate(X_test, Y_test)

print('Test score:', score[0])
print('Test accuracy:', score[1])

6
 313/313 [==============================] - 1s 4ms/step - loss: 0.0714 -
accuracy: 0.9790
Test score: 0.07143609970808029
Test accuracy: 0.9789999723434448

[19]: # The predict_classes function outputs the highest probability class

# according to the trained classifier for each input example.
predictions = model.predict(X_test)
predicted_classes = np.argmax(predictions, axis=1)

# Check which items we got right / wrong

correct_indices = np.nonzero(predicted_classes == y_test)[0]

incorrect_indices = np.nonzero(predicted_classes != y_test)[0]

313/313 [==============================] - 2s 6ms/step

[20]: plt.figure()
for i, correct in enumerate(correct_indices[:9]):
plt.subplot(3,3,i+1)
plt.imshow(X_test[correct].reshape(28,28), cmap='gray',␣
↪interpolation='none')

plt.title("Predicted {}, Class {}".format(predicted_classes[correct],␣

↪y_test[correct]))

plt.tight_layout()

plt.figure()
for i, incorrect in enumerate(incorrect_indices[:9]):
plt.subplot(3,3,i+1)
plt.imshow(X_test[incorrect].reshape(28,28), cmap='gray',␣
↪interpolation='none')

plt.title("Predicted {}, Class {}".format(predicted_classes[incorrect],␣

↪y_test[incorrect]))

plt.tight_layout()

7
8
[41]: # import some additional tools

from keras.preprocessing.image import ImageDataGenerator

from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D,␣
↪GlobalAveragePooling2D, Flatten

from keras.layers import BatchNormalization

[22]: # Reload the MNIST data

(X_train, y_train), (X_test, y_test) = mnist.load_data()

[23]: # Again, do some formatting

9
 # Except we do not flatten each image into a 784-length vector because we want␣
↪to perform convolutions first

X_train = X_train.reshape(60000, 28, 28, 1) #add an additional dimension to␣

↪represent the single-channel

X_test = X_test.reshape(10000, 28, 28, 1)

X_train = X_train.astype('float32') # change integers to 32-bit␣

↪floating point numbers

X_test = X_test.astype('float32')

X_train /= 255 # normalize each value for each␣

↪pixel for the entire vector for each input

X_test /= 255

print("Training matrix shape", X_train.shape)

print("Testing matrix shape", X_test.shape)

Training matrix shape (60000, 28, 28, 1)

Testing matrix shape (10000, 28, 28, 1)

[24]: # one-hot format classes

nb_classes = 10 # number of unique digits

Y_train = to_categorical(y_train, nb_classes)

Y_test = to_categorical(y_test, nb_classes)

[25]: model = Sequential() # Linear stacking of layers

# Convolution Layer 1
model.add(Conv2D(32, (3, 3), input_shape=(28,28,1))) # 32 different 3x3 kernels␣
↪-- so 32 feature maps

model.add(BatchNormalization(axis=-1)) # normalize each feature␣

↪map before activation

convLayer01 = Activation('relu') # activation

model.add(convLayer01)

# Convolution Layer 2
model.add(Conv2D(32, (3, 3))) # 32 different 3x3 kernels␣
↪-- so 32 feature maps

model.add(BatchNormalization(axis=-1)) # normalize each feature␣

↪map before activation

model.add(Activation('relu')) # activation
convLayer02 = MaxPooling2D(pool_size=(2,2)) # Pool the max values over␣
↪a 2x2 kernel

10
 model.add(convLayer02)

# Convolution Layer 3
model.add(Conv2D(64,(3, 3))) # 64 different 3x3 kernels␣
↪-- so 64 feature maps

model.add(BatchNormalization(axis=-1)) # normalize each feature␣

↪map before activation

convLayer03 = Activation('relu') # activation

model.add(convLayer03)

# Convolution Layer 4
model.add(Conv2D(64, (3, 3))) # 64 different 3x3 kernels␣
↪-- so 64 feature maps

model.add(BatchNormalization(axis=-1)) # normalize each feature␣

↪map before activation

model.add(Activation('relu')) # activation
convLayer04 = MaxPooling2D(pool_size=(2,2)) # Pool the max values over␣
↪a 2x2 kernel

model.add(convLayer04)
model.add(Flatten()) # Flatten final 4x4x64␣
↪output matrix into a 1024-length vector

# Fully Connected Layer 5

model.add(Dense(512)) # 512 FCN nodes
model.add(BatchNormalization()) # normalization
model.add(Activation('relu')) # activation

# Fully Connected Layer 6

model.add(Dropout(0.2)) # 20% dropout of randomly␣
↪selected nodes

model.add(Dense(10)) # final 10 FCN nodes

model.add(Activation('softmax')) # softmax activation

[26]: model.summary()

Model: "sequential_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) (None, 26, 26, 32) 320

batch_normalization (Batch (None, 26, 26, 32) 128

Normalization)

activation_3 (Activation) (None, 26, 26, 32) 0

conv2d_1 (Conv2D) (None, 24, 24, 32) 9248

11
batch_normalization_1 (Bat (None, 24, 24, 32) 128
chNormalization)

activation_4 (Activation) (None, 24, 24, 32) 0

max_pooling2d (MaxPooling2 (None, 12, 12, 32) 0

D)

conv2d_2 (Conv2D) (None, 10, 10, 64) 18496

batch_normalization_2 (Bat (None, 10, 10, 64) 256

chNormalization)

activation_5 (Activation) (None, 10, 10, 64) 0

conv2d_3 (Conv2D) (None, 8, 8, 64) 36928

batch_normalization_3 (Bat (None, 8, 8, 64) 256

chNormalization)

activation_6 (Activation) (None, 8, 8, 64) 0

max_pooling2d_1 (MaxPoolin (None, 4, 4, 64) 0

g2D)

flatten (Flatten) (None, 1024) 0

dense_3 (Dense) (None, 512) 524800

batch_normalization_4 (Bat (None, 512) 2048

chNormalization)

activation_7 (Activation) (None, 512) 0

dropout_2 (Dropout) (None, 512) 0

dense_4 (Dense) (None, 10) 5130

activation_8 (Activation) (None, 10) 0

=================================================================
Total params: 597738 (2.28 MB)
Trainable params: 596330 (2.27 MB)
Non-trainable params: 1408 (5.50 KB)
_________________________________________________________________

12
[27]: # we'll use the same optimizer

model.compile(loss='categorical_crossentropy', optimizer='adam',␣
↪metrics=['accuracy'])

[28]: # data augmentation prevents overfitting by slightly changing the data randomly
# Keras has a great built-in feature to do automatic augmentation

gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08,␣

↪shear_range=0.3,

height_shift_range=0.08, zoom_range=0.08)

test_gen = ImageDataGenerator()

[29]: # We can then feed our augmented data in batches

# Besides loss function considerations as before, this method actually results␣
↪in significant memory savings

# because we are actually LOADING the data into the network in batches before␣
↪processing each batch

# Before the data was all loaded into memory, but then processed in batches.

train_generator = gen.flow(X_train, Y_train, batch_size=128)

test_generator = test_gen.flow(X_test, Y_test, batch_size=128)

[30]: # We can now train our model which is fed data by our batch loader
# Steps per epoch should always be total size of the set divided by the batch␣
↪size

# SIGNIFICANT MEMORY SAVINGS (important for larger, deeper networks)

model.fit_generator(train_generator, steps_per_epoch=60000//128, epochs=5,␣

↪verbose=1,

validation_data=test_generator, validation_steps=10000//128)

Epoch 1/5
<ipython-input-30-758777735d04>:6: UserWarning: `Model.fit_generator` is
deprecated and will be removed in a future version. Please use `Model.fit`,
which supports generators.
model.fit_generator(train_generator, steps_per_epoch=60000//128, epochs=5,
verbose=1,
468/468 [==============================] - 201s 425ms/step - loss: 0.1306 -
accuracy: 0.9590 - val_loss: 0.2779 - val_accuracy: 0.9135
Epoch 2/5
468/468 [==============================] - 211s 450ms/step - loss: 0.0506 -
accuracy: 0.9847 - val_loss: 0.0316 - val_accuracy: 0.9895

13
 Epoch 3/5
468/468 [==============================] - 206s 441ms/step - loss: 0.0393 -
accuracy: 0.9877 - val_loss: 0.0306 - val_accuracy: 0.9904
Epoch 4/5
468/468 [==============================] - 207s 442ms/step - loss: 0.0336 -
accuracy: 0.9897 - val_loss: 0.0228 - val_accuracy: 0.9923
Epoch 5/5
468/468 [==============================] - 195s 417ms/step - loss: 0.0293 -
accuracy: 0.9906 - val_loss: 0.0364 - val_accuracy: 0.9889

[30]: <keras.src.callbacks.History at 0x7f7327d1ea70>

[31]: score = model.evaluate(X_test, Y_test)

print('Test score:', score[0])
print('Test accuracy:', score[1])

313/313 [==============================] - 7s 22ms/step - loss: 0.0364 -

accuracy: 0.9888
Test score: 0.036419935524463654
Test accuracy: 0.9887999892234802

[46]: from keras import backend as K

# choose any image to want by specifying the index

img = X_test[3]
img = np.expand_dims(img, axis=0) # Keras requires the image to be in 4D, so we␣
↪add an extra dimension to it.

# Not important to understand how this function work -- It just plots a␣

↪convolution layer

def visualize(layer):
inputs = [K.learning_phase()] + model.inputs

_convout1_f = K.function(inputs, [layer.output])

def convout1_f(X):
# The [0] is to disable the training phase flag
return _convout1_f([0] + [X])

convolutions = convout1_f(img)
convolutions = np.squeeze(convolutions)

print ('Shape of conv:', convolutions.shape)

m = convolutions.shape[2]
n = int(np.ceil(np.sqrt(m)))

14
 # Visualization of each filter of the layer
fig = plt.figure(figsize=(15,12))
for i in range(m):
ax = fig.add_subplot(n,n,i+1)
ax.imshow(convolutions[:,:,i], cmap='gray')

[47]: plt.figure()
plt.imshow(X_test[3].reshape(28,28), cmap='gray', interpolation='none')

[47]: <matplotlib.image.AxesImage at 0x7f7319933310>

[44]: visualize(convLayer01, input_image)

15
 1/1 [==============================] - 0s 41ms/step

[ ]:

16