Scribd
Upload
2 views6 pages

Vertopal.com DLWP Chapter6

The document provides a comprehensive overview of implementing a Recurrent Neural Network (RNN) using TensorFlow and Keras, including the use of the ReLU activation function, training on sequences of input vectors, and generating predictions. It covers many-to-one predictions for numerical sequences and character predictions from text, along with the backpropagation process to update model weights. The document also includes code snippets demonstrating the RNN's functionality and outputs for various inputs.

Uploaded by

Papun Kumar Sahoo
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as PDF, TXT or read online on Scribd
2 views6 pages

Vertopal.com DLWP Chapter6

The document provides a comprehensive overview of implementing a Recurrent Neural Network (RNN) using TensorFlow and Keras, including the use of the ReLU activation function, training on sequences of input vectors, and generating predictions. It covers many-to-one predictions for numerical sequences and character predictions from text, along with the backpropagation process to update model weights. The document also includes code snippets demonstrating the RNN's functionality and outputs for various inputs.

Uploaded by

Papun Kumar Sahoo
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as PDF, TXT or read online on Scribd
You are on page 1/ 6

import numpy as np

#ReLU activation function


def relu(x):
return np.maximum(0,x)
#input vector at time t
x_t=np.array([[1], [2]]) #shape(2,1)
#previous hidden state(initially zero)
h_prev=np.array([[0.0], [0.0]]) #shape(2,1)
#weight matrices
W_xh=np.array([[0.1, 0.2],
[0.3, 0.4]]) #shape(2,2)
W_hh=np.array([[0.5, 0.6],
[0.7, 0.8]]) #shape(2,2)
W_hy=np.array([[0.9, 1.0]]) #shape(1,2)
#bias vectors
b_h=np.array([[0.1],
[0.2]]) #shape(2,1)
b_y=np.array([[0.3]]) #shape(1,1)
#step1: compute hidden state h_t using ReLU
xh=np.dot(W_xh, x_t)
hh=np.dot(W_hh, h_prev)
h_t=relu(xh+hh+b_h)
#step2: compute output y_t
y_t=np.dot(W_hy, h_t)+b_y
#print results
print("Hidden state h_t:")
print(h_t)
print("\nOutput y_t:")
print(y_t)

Hidden state h_t:


[[0.6]
[1.3]]

Output y_t:
[[2.14]]

import numpy as np
#ReLU activation
def relu(x):
return np.maximum(0,x)
#sequence of 3 input vectors, each with 2 features(shape:3 x 2 x 1)
sequence=[
np.array([[1], [2]]),
np.array([[0], [1]]),
np.array([[3], [1]])
]
#intial hidden state(2x1)
h_t=np.zeros((2,1))
#weights and biases
w_xh=np.array([[0.1, 0.2],
[0.3, 0.4]])
w_hh=np.array([[0.5, 0.6],
[0.7, 0.8]])
w_hy=np.array([[0.9, 1.0]])
b_h=np.array([[0.1],
[0.2]])
b_y=np.array([[0.3]])
#process each step
print("RNN time step outputs")
for t, x_t in enumerate(sequence,1):
#hidden state update
h_t=relu(np.dot(w_xh, x_t)+np.dot(w_hh, h_t)+b_h)
#output calculation
y_t=np.dot(w_hy, h_t)+b_y
print(f"Time step {t}:")
print("Input x_t:\n", x_t.flatten())
print("Hidden state h_t:\n",h_t.flatten())
print("Output y_t:\n", y_t,y_t.flatten())

RNN time step outputs


Time step 1:
Input x_t:
[1 2]
Hidden state h_t:
[0.6 1.3]
Output y_t:
[[2.14]] [2.14]
Time step 2:
Input x_t:
[0 1]
Hidden state h_t:
[1.38 2.06]
Output y_t:
[[3.602]] [3.602]
Time step 3:
Input x_t:
[3 1]
Hidden state h_t:
[2.526 4.114]
Output y_t:
[[6.6874]] [6.6874]

#many to one prediction


#predict the next no in the sequence:
#e.g.,[0.1,0.2,0.3]-> 0.4, [0.2,0.3,0.4] -> 0.5
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import SimpleRNN, Dense
#1. prepare dataset
X=np.array([
[0.1, 0.2, 0.3],
[0.2, 0.3, 0.4],
[0.3, 0.4, 0.5],
[0.4, 0.5, 0.6],
])
y=np.array([0.4, 0.5, 0.6, 0.7])
#reshape to(samples, time steps, features)
X= X.reshape((X.shape[0], X.shape[1], 1))
#2. Build model
model=Sequential([
SimpleRNN(10, activation='relu', input_shape=(3,1)),
Dense(1)
])
model.compile(optimizer='adam', loss='mse')
#3. Train model
model.fit(X, y, epochs=300, verbose=0)
#4. Make predictions
test=np.array([[0.5, 0.6, 0.7]]).reshape((1,3,1))
predicted=model.predict(test, verbose=0)
print(f"Predicted next value: { predicted[0][0]:.3f}")

Predicted next value: 0.803

#given a sequence of characters like "hel", predict the next


character: "l"
#then generate text like:"hello hello hello....."
import numpy as np
from tensorflow.keras.layers import SimpleRNN, Dense
from tensorflow.keras.models import Sequential
from tensorflow.keras.utils import to_categorical
#1. prepare text
text="hello welcome to ITER"
chars=sorted(list(set(text))) #unique characters
char_to_idx={c:i for i, c in enumerate(chars)}
idx_to_char={i: c for c, i in char_to_idx.items()}
print(char_to_idx)
print(idx_to_char)
#convert text ton integer indices
encoded= np.array([char_to_idx[c] for c in text])
#2. create input/output pairs(sequence length=3)
seq_length=3
X,y=[], []
for i in range(len(encoded) - seq_length):
X.append(encoded[i:i+seq_length])
y.append(encoded[i+seq_length])

X=np.array(X) #(num_samples, seq_len)


y=np.array(y) #(num_samples)
print(X)
print(y)
#one-hot encoded inputs and outputs
num_chars=len(chars)
X_oh=to_categorical(X, num_classes=num_chars)
y_oh=to_categorical(y, num_classes=num_chars)
model=Sequential([
SimpleRNN(64, activation='relu', input_shape=(seq_length,
num_chars)),
Dense(num_chars, activation='softmax')
])
#3.build the model
model.compile(loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy'])
model.fit(X_oh, y_oh, epochs=300, verbose=0)
#4.train
def generate_text(seed_text, length=20):
#5.Text generation
for _ in range(length):
input_seq = [char_to_idx[c] for c in seed_text[-seq_length:]]
input_oh = to_categorical([input_seq], num_classes=num_chars)
pred=model.predict(input_oh, verbose=0)
next_char=idx_to_char[np.argmax(pred)]
seed_text+=next_char
return seed_text
print("Generated Text:", generate_text("hell")) #test generation

{' ': 0, 'E': 1, 'I': 2, 'R': 3, 'T': 4, 'c': 5, 'e': 6, 'h': 7, 'l':
8, 'm': 9, 'o': 10, 't': 11, 'w': 12}
{0: ' ', 1: 'E', 2: 'I', 3: 'R', 4: 'T', 5: 'c', 6: 'e', 7: 'h', 8:
'l', 9: 'm', 10: 'o', 11: 't', 12: 'w'}
[[ 7 6 8]
[ 6 8 8]
[ 8 8 10]
[ 8 10 0]
[10 0 12]
[ 0 12 6]
[12 6 8]
[ 6 8 5]
[ 8 5 10]
[ 5 10 9]
[10 9 6]
[ 9 6 0]
[ 6 0 11]
[ 0 11 10]
[11 10 0]
[10 0 2]
[ 0 2 4]
[ 2 4 1]]
[ 8 10 0 12 6 8 5 10 9 6 0 11 10 0 2 4 1 3]
/usr/local/lib/python3.11/dist-packages/keras/src/layers/rnn/
rnn.py:200: UserWarning: Do not pass an `input_shape`/`input_dim`
argument to a layer. When using Sequential models, prefer using an
`Input(shape)` object as the first layer in the model instead.
super().__init__(**kwargs)

Generated Text: hello welcome to ITER T

#Back Propagation
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import SimpleRNN,Dense
X = np.array([[1.0],[2.0]]) #Shape: (1,2,1)
y_true = np.array([0.5]) #Shape: (1,1)
model = Sequential([
SimpleRNN(units=2, activation='tanh',
input_shape=(2,1),return_sequences=False),
Dense(1) #Output layer #2.Define a simple RNN model (many-to-
one)
])
loss_fn=tf.keras.losses.MeanSquaredError() #3.Forward and backward
pass using Gra
optimizer = tf.keras.optimizers.Adam(learning_rate=0.1)
with tf.GradientTape() as tape:
y_pred = model(X, training=True)
loss = loss_fn(y_true, y_pred)
gradients = tape.gradient(loss, model.trainable_variables)
#4.Compute Gradient
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
#5.Apply gradient
for var, grad in zip(model.trainable_variables, gradients):
print(f'Variable: {var.name}') #6.Show gradient value
print("Gradient:\n",grad.numpy())
print("\nPredicted:", y_pred.numpy())
print("Loss:", loss.numpy())

/usr/local/lib/python3.11/dist-packages/keras/src/layers/rnn/
rnn.py:200: UserWarning: Do not pass an `input_shape`/`input_dim`
argument to a layer. When using Sequential models, prefer using an
`Input(shape)` object as the first layer in the model instead.
super().__init__(**kwargs)

Variable: kernel
Gradient:
[[-0.98720163 -0.13503356]]
Variable: recurrent_kernel
Gradient:
[[0. 0.]
[0. 0.]]
Variable: bias
Gradient:
[-0.5919188 -0.10538915]
Variable: kernel
Gradient:
[[ 0.02860823]
[-0.45368445]]
Variable: bias
Gradient:
[0.51464826]

Predicted: [[0.6705861 ]
[0.84406215]]
Loss: 0.07373919

You might also like

pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy