0% found this document useful (0 votes)

23 views13 pages

Mlp-Fromscratch Sigmoid-Mse

The document outlines the course STAT 453: Deep Learning taught by Sebastian Raschka in Spring 2021, including details on the course website, GitHub repository, and Python environment setup. It provides a comprehensive implementation of a multilayer perceptron from scratch using PyTorch, covering model architecture, training, and evaluation processes with the MNIST dataset. The training process includes logging of costs and mean squared error metrics across multiple epochs.

Uploaded by

pnqanh.gdsciu

We take content rights seriously. If you suspect this is your content, claim it here.

Available Formats

Download as PDF, TXT or read online on Scribd

0% found this document useful (0 votes)

23 views13 pages

Mlp-Fromscratch Sigmoid-Mse

Uploaded by

pnqanh.gdsciu

We take content rights seriously. If you suspect this is your content, claim it here.

Available Formats

Download as PDF, TXT or read online on Scribd

You are on page 1/ 13

STAT 453: Deep Learning (Spring 2021)

Instructor: Sebastian Raschka (sraschka@wisc.edu)

Course website: http://pages.stat.wisc.edu/~sraschka/teaching/stat453-ss2021/

GitHub repository: https://github.com/rasbt/stat453-deep-learning-ss21

%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -p torch

Author: Sebastian Raschka

Python implementation: CPython

Python version : 3.9.2
IPython version : 7.20.0

torch: 1.9.0a0+d819a21

Multilayer Perceptron From Scratch (Sigmoid activation, MSE

Loss)
Implementation of a 1-hidden layer multi-layer perceptron from scratch using

sigmoid activation in the hidden layer

sigmoid activation in the output layer
Mean Squared Error loss function

Imports
import matplotlib.pyplot as plt
import pandas as pd
import torch
%matplotlib inline

import time
import numpy as np
from torchvision import datasets
from torchvision import transforms
from torch.utils.data import DataLoader
import torch.nn.functional as F
import torch

Settings and Dataset

##########################
### SETTINGS
##########################

RANDOM_SEED = 1
BATCH_SIZE = 100
NUM_EPOCHS = 50

##########################
### MNIST DATASET
##########################

# Note transforms.ToTensor() scales input images

# to 0-1 range
train_dataset = datasets.MNIST(root='data',
train=True,
transform=transforms.ToTensor(),
download=True)

test_dataset = datasets.MNIST(root='data',
train=False,
transform=transforms.ToTensor())

train_loader = DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=True)

test_loader = DataLoader(dataset=test_dataset,
batch_size=BATCH_SIZE,
shuffle=False)

# Checking the dataset

for images, labels in train_loader:
print('Image batch dimensions:', images.shape)
print('Image label dimensions:', labels.shape)
break

Image batch dimensions: torch.Size([100, 1, 28, 28])

Image label dimensions: torch.Size([100])

Model Implementation
##########################
### MODEL
##########################

class MultilayerPerceptron():

def init(self, num_features, num_hidden, num_classes):

super(MultilayerPerceptron, self).__init__()

self.num_classes = num_classes

# hidden 1
self.weight_1 = torch.zeros(num_hidden, num_features,
dtype=torch.float).normal_(0.0, 0.1)
self.bias_1 = torch.zeros(num_hidden, dtype=torch.float)

# output
self.weight_o = torch.zeros(self.num_classes, num_hidden,
dtype=torch.float).normal_(0.0, 0.1)
self.bias_o = torch.zeros(self.num_classes, dtype=torch.float)

def forward(self, x):

# hidden 1

# input dim: [n_hidden, n_features] dot [n_features, n_examples] .T

# output dim: [n_examples, n_hidden]
z_1 = torch.mm(x, self.weight_1.t()) + self.bias_1
a_1 = torch.sigmoid(z_1)

# hidden 2
# input dim: [n_classes, n_hidden] dot [n_hidden, n_examples] .T
# output dim: [n_examples, n_classes]
z_2 = torch.mm(a_1, self.weight_o.t()) + self.bias_o
a_2 = torch.sigmoid(z_2)
return a_1, a_2

def backward(self, x, a_1, a_2, y):

#########################
### Output layer weights
#########################

# onehot encoding
y_onehot = torch.FloatTensor(y.size(0), self.num_classes)
y_onehot.zero_()
y_onehot.scatter_(1, y.view(-1, 1).long(), 1)

# Part 1: dLoss/dOutWeights
## = dLoss/dOutAct * dOutAct/dOutNet * dOutNet/dOutWeight
## where DeltaOut = dLoss/dOutAct * dOutAct/dOutNet
## for convenient re-use

# input/output dim: [n_examples, n_classes]

dloss_da2 = 2.*(a_2 - y_onehot) / y.size(0)

# input/output dim: [n_examples, n_classes]

da2_dz2 = a_2 * (1. - a_2) # sigmoid derivative

# output dim: [n_examples, n_classes]

delta_out = dloss_da2 * da2_dz2 # "delta (rule) placeholder"

# gradient for output weights

# [n_examples, n_hidden]
dz2__dw_out = a_1

# input dim: [n_classlabels, n_examples] dot [n_examples, n_hidden]

# output dim: [n_classlabels, n_hidden]
dloss__dw_out = torch.mm(delta_out.t(), dz2__dw_out)
dloss__db_out = torch.sum(delta_out, dim=0)

#################################
# Part 2: dLoss/dHiddenWeights
## = DeltaOut * dOutNet/dHiddenAct * dHiddenAct/dHiddenNet * dHiddenNet/dWeight

# [n_classes, n_hidden]
dz2__a1 = self.weight_o

# output dim: [n_examples, n_hidden]

dloss_a1 = torch.mm(delta_out, dz2__a1)

# [n_examples, n_hidden]
da1__dz1 = a_1 * (1. - a_1) # sigmoid derivative

# [n_examples, n_features]
dz1__dw1 = x

# output dim: [n_hidden, n_features]

dloss_dw1 = torch.mm((dloss_a1 * da1__dz1).t(), dz1__dw1)
dloss_db1 = torch.sum((dloss_a1 * da1__dz1), dim=0)

return dloss__dw_out, dloss__db_out, dloss_dw1, dloss_db1

Training
####################################################
##### Training and evaluation wrappers
###################################################

def to_onehot(y, num_classes):

y_onehot = torch.FloatTensor(y.size(0), num_classes)
y_onehot.zero_()
y_onehot.scatter_(1, y.view(-1, 1).long(), 1).float()
return y_onehot

def loss_func(targets_onehot, probas_onehot):

return torch.mean(torch.mean((targets_onehot - probas_onehot)**2, dim=0))

def compute_mse(net, data_loader):

curr_mse, num_examples = torch.zeros(model.num_classes).float(), 0
with torch.no_grad():
for features, targets in data_loader:
features = features.view(-1, 28*28)
logits, probas = net.forward(features)
y_onehot = to_onehot(targets, model.num_classes)
loss = torch.sum((y_onehot - probas)**2, dim=0)
num_examples += targets.size(0)
curr_mse += loss

curr_mse = torch.mean(curr_mse/num_examples, dim=0)

return curr_mse

def train(model, data_loader, num_epochs,

learning_rate=0.1):

minibatch_cost = []
epoch_cost = []

for e in range(num_epochs):

for batch_idx, (features, targets) in enumerate(train_loader):

features = features.view(-1, 28*28)

#### Compute outputs ####

a_1, a_2 = model.forward(features)

#### Compute gradients ####

dloss__dw_out, dloss__db_out, dloss_dw1, dloss_db1 = \
model.backward(features, a_1, a_2, targets)

#### Update weights ####

model.weight_1 -= learning_rate * dloss_dw1
model.bias_1 -= learning_rate * dloss_db1
model.weight_o -= learning_rate * dloss__dw_out
model.bias_o -= learning_rate * dloss__db_out

#### Logging ####

curr_cost = loss_func(to_onehot(targets, model.num_classes), a_2)
minibatch_cost.append(curr_cost)
if not batch_idx % 50:
print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f'
%(e+1, NUM_EPOCHS, batch_idx,
len(train_loader), curr_cost))

#### Logging ####

curr_cost = compute_mse(model, train_loader)
epoch_cost.append(curr_cost)
print('Epoch: %03d/%03d |' % (e+1, NUM_EPOCHS), end="")
print(' Train MSE: %.5f' % curr_cost)

return minibatch_cost, epoch_cost

####################################################
##### Training
###################################################

torch.manual_seed(RANDOM_SEED)
model = MultilayerPerceptron(num_features=28*28,
num_hidden=50,
num_classes=10)

minibatch_cost, epoch_cost = train(model,

train_loader,
num_epochs=NUM_EPOCHS,
learning_rate=0.1)

Epoch: 001/050 | Batch 000/600 | Cost: 0.2386

Evaluation
plt.plot(range(len(minibatch_cost)), minibatch_cost)
plt.ylabel('Mean Squared Error')
plt.xlabel('Minibatch')
plt.show()

plt.plot(range(len(epoch_cost)), epoch_cost)
plt.ylabel('Mean Squared Error')
plt.xlabel('Epoch')
plt.show()

def compute_accuracy(net, data_loader):

correct_pred, num_examples = 0, 0
with torch.no_grad():
for features, targets in data_loader:
features = features.view(-1, 28*28)
_, outputs = net.forward(features)
predicted_labels = torch.argmax(outputs, 1)
num_examples += targets.size(0)
correct_pred += (predicted_labels == targets).sum()
return correct_pred.float()/num_examples * 100

print('Training Accuracy: %.2f' % compute_accuracy(model, train_loader))

print('Test Accuracy: %.2f' % compute_accuracy(model, test_loader))

Training Accuracy: 94.69

Test Accuracy: 94.53

Visual Inspection
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt

for features, targets in test_loader:

break

fig, ax = plt.subplots(1, 4)
for i in range(4):
ax[i].imshow(features[i].view(28, 28), cmap=matplotlib.cm.binary)

plt.show()

_, predictions = model.forward(features[:4].view(-1, 28*28))

predictions = torch.argmax(predictions, dim=1)
print('Predicted labels', predictions)

Predicted labels tensor([7, 2, 1, 0])

Loading [MathJax]/jax/output/CommonHTML/fonts/TeX/fontdata.js

Final DL
No ratings yet
Final DL
26 pages
BLDD VIT ResNet50v2 CustomCNN
No ratings yet
BLDD VIT ResNet50v2 CustomCNN
38 pages
LSTM From Scratch in Python
No ratings yet
LSTM From Scratch in Python
11 pages
MLP Pytorch Softmax Crossentr
No ratings yet
MLP Pytorch Softmax Crossentr
20 pages
PINN 1DBurgers
No ratings yet
PINN 1DBurgers
19 pages
The JavaScript Workbook - Download Edition
100% (6)
The JavaScript Workbook - Download Edition
221 pages
MLP Scratch
No ratings yet
MLP Scratch
8 pages
Video 5 - Building A Multilayer Perceptron For Regression in PyTorch
No ratings yet
Video 5 - Building A Multilayer Perceptron For Regression in PyTorch
17 pages
Softmax Regression Scratch
No ratings yet
Softmax Regression Scratch
5 pages
Perceptron Pytorch
No ratings yet
Perceptron Pytorch
3 pages
Logistic Regression
No ratings yet
Logistic Regression
4 pages
H1 AndresAlcivar
No ratings yet
H1 AndresAlcivar
4 pages
Lab Manual DL (New)
No ratings yet
Lab Manual DL (New)
89 pages
MLP Pytorch Sigmoid Mse
No ratings yet
MLP Pytorch Sigmoid Mse
20 pages
Experiment 6
No ratings yet
Experiment 6
3 pages
Chapter 1
No ratings yet
Chapter 1
37 pages
Softmax Regression Mnist
No ratings yet
Softmax Regression Mnist
3 pages
Ccnet Only
No ratings yet
Ccnet Only
6 pages
IBest DeepLearning
No ratings yet
IBest DeepLearning
123 pages
Skill 7
No ratings yet
Skill 7
11 pages
Experiment No 13 Final
No ratings yet
Experiment No 13 Final
9 pages
Lab 9
No ratings yet
Lab 9
29 pages
Cross Entropy
No ratings yet
Cross Entropy
1 page
(Deep Learning Using PyTorch) (Cheatsheet)
No ratings yet
(Deep Learning Using PyTorch) (Cheatsheet)
7 pages
Autoencoder From Scratch
No ratings yet
Autoencoder From Scratch
21 pages
Week 7 - Mnist-Mlp
No ratings yet
Week 7 - Mnist-Mlp
7 pages
CV Lab Final AwaisKhan EE A
No ratings yet
CV Lab Final AwaisKhan EE A
7 pages
Lesson 2
No ratings yet
Lesson 2
8 pages
Adaline SGD
No ratings yet
Adaline SGD
4 pages
CIFAR - 10 - Dataset - Using - CNN - Aniiiii - HTML
No ratings yet
CIFAR - 10 - Dataset - Using - CNN - Aniiiii - HTML
8 pages
Neural Networks MATH Explained
No ratings yet
Neural Networks MATH Explained
14 pages
Pytorch 101: Deep Learning PHD Course 2017/2018
No ratings yet
Pytorch 101: Deep Learning PHD Course 2017/2018
19 pages
NN From Scratch PDF 1735495327
No ratings yet
NN From Scratch PDF 1735495327
19 pages
Linear Regr GD
No ratings yet
Linear Regr GD
3 pages
Complete SQL Queries
100% (1)
Complete SQL Queries
61 pages
Autoencoder - MPL - Basic - Ipynb - Colaboratory PDF
No ratings yet
Autoencoder - MPL - Basic - Ipynb - Colaboratory PDF
21 pages
Pytorch Demo 1749471354
No ratings yet
Pytorch Demo 1749471354
10 pages
A-Simple-Neural-Network-From-Scratch - Jupyter Notebook
No ratings yet
A-Simple-Neural-Network-From-Scratch - Jupyter Notebook
9 pages
PyTorch CrashCourse
No ratings yet
PyTorch CrashCourse
16 pages
TXT
No ratings yet
TXT
7 pages
Lab Report 03
No ratings yet
Lab Report 03
14 pages
X OR Problem Using DNN
No ratings yet
X OR Problem Using DNN
3 pages
02 ML PDF
No ratings yet
02 ML PDF
5 pages
Deeplg 3
No ratings yet
Deeplg 3
8 pages
Ex No:1 Implementing A Perceptron Algorithm For Binary Classification Date: Aim
No ratings yet
Ex No:1 Implementing A Perceptron Algorithm For Binary Classification Date: Aim
41 pages
ML 0joh
No ratings yet
ML 0joh
2 pages
Notebook - Deep Neural Networks
No ratings yet
Notebook - Deep Neural Networks
28 pages
New Exp
No ratings yet
New Exp
12 pages
Assignment3 AL
No ratings yet
Assignment3 AL
23 pages
Lab 4
No ratings yet
Lab 4
2 pages
NN From Scratch
No ratings yet
NN From Scratch
5 pages
Experiment 2.4 DL
No ratings yet
Experiment 2.4 DL
4 pages
Intro To Pytorch
No ratings yet
Intro To Pytorch
12 pages
cs519 hw2
No ratings yet
cs519 hw2
15 pages
555610a19 DL Exp4
No ratings yet
555610a19 DL Exp4
11 pages
Assignment 3 DS5620
No ratings yet
Assignment 3 DS5620
11 pages
Ilovepdf Merged
No ratings yet
Ilovepdf Merged
10 pages
Using A Three Layer Deep Neural Network To Solve An Unsupervised Learning Problem
No ratings yet
Using A Three Layer Deep Neural Network To Solve An Unsupervised Learning Problem
13 pages
Trainina A NN Backpropagation
No ratings yet
Trainina A NN Backpropagation
6 pages
PyTorch Crash Course 1713016363
No ratings yet
PyTorch Crash Course 1713016363
15 pages
FDTD Getting Started Manual
No ratings yet
FDTD Getting Started Manual
63 pages
CIS Microsoft Azure Foundations Benchmark v4.0.0
No ratings yet
CIS Microsoft Azure Foundations Benchmark v4.0.0
561 pages
BAED-AI2121-2322S-Written Work 1-4th Quarter Grade 12
100% (1)
BAED-AI2121-2322S-Written Work 1-4th Quarter Grade 12
5 pages
Junit5 Notes
No ratings yet
Junit5 Notes
20 pages
SAP Simple Finance Training Course Content
No ratings yet
SAP Simple Finance Training Course Content
5 pages
IEEE 610-5-1990 - w2000 Glossary of Data Management Terminology
No ratings yet
IEEE 610-5-1990 - w2000 Glossary of Data Management Terminology
76 pages
Cpuguide
No ratings yet
Cpuguide
13 pages
Module 1 - Introduction To Hypershade
No ratings yet
Module 1 - Introduction To Hypershade
50 pages
3G&4G Upgrade and OM Configuration by Weblmt
No ratings yet
3G&4G Upgrade and OM Configuration by Weblmt
18 pages
Final Porfolio - Daniel Alcala
No ratings yet
Final Porfolio - Daniel Alcala
26 pages
ABI Product Brochure
No ratings yet
ABI Product Brochure
24 pages
Sage 50 Payroll Year End Guide
No ratings yet
Sage 50 Payroll Year End Guide
43 pages
Manually Remove A Click-to-Run Installation of Office
No ratings yet
Manually Remove A Click-to-Run Installation of Office
3 pages
Module 10
No ratings yet
Module 10
37 pages
Data Logging Vs Data Acquisition
No ratings yet
Data Logging Vs Data Acquisition
4 pages
Jedec Standard: DELPHI Compact Thermal Model Guideline
No ratings yet
Jedec Standard: DELPHI Compact Thermal Model Guideline
24 pages
Drawing Realistic Ipad2 - Photoshop Tutorial: Step 1: Ipad Basic Shape
No ratings yet
Drawing Realistic Ipad2 - Photoshop Tutorial: Step 1: Ipad Basic Shape
54 pages
Phases of Project Management
100% (1)
Phases of Project Management
20 pages
NSDC-Assessment Processes and Protocols - Guide For STT - Final
No ratings yet
NSDC-Assessment Processes and Protocols - Guide For STT - Final
88 pages
Reality Show Management - TutorialsDuniya
No ratings yet
Reality Show Management - TutorialsDuniya
19 pages
Etas Xetk s30.0c v31 Release Notes en 20240926
No ratings yet
Etas Xetk s30.0c v31 Release Notes en 20240926
17 pages
Ahsan Jamil: Career Objectives
No ratings yet
Ahsan Jamil: Career Objectives
4 pages
Chat Bot Mini Project
No ratings yet
Chat Bot Mini Project
4 pages
Azure Security Telescript - July 2021
No ratings yet
Azure Security Telescript - July 2021
30 pages
CMC Cartonwrap CW1000: CIS Release 2.3
No ratings yet
CMC Cartonwrap CW1000: CIS Release 2.3
47 pages
Steps On How To Create Pixel Art Using Conditional Formatting
No ratings yet
Steps On How To Create Pixel Art Using Conditional Formatting
2 pages
GFW0018 W6 Poster (S2116309)
No ratings yet
GFW0018 W6 Poster (S2116309)
3 pages
Embedded Event Manager
No ratings yet
Embedded Event Manager
5 pages
Profound Python Data Science
From Everand
Profound Python Data Science
Onder Teker
No ratings yet
TensorFlow深度学习项目实战: Chinese Edition
From Everand
TensorFlow深度学习项目实战: Chinese Edition
Posts & Telecom Press
No ratings yet

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.

Mlp-Fromscratch Sigmoid-Mse

Uploaded by

Mlp-Fromscratch Sigmoid-Mse

Uploaded by

STAT 453: Deep Learning (Spring 2021)

Instructor: Sebastian Raschka (sraschka@wisc.edu)

Course website: http://pages.stat.wisc.edu/~sraschka/teaching/stat453-ss2021/

Author: Sebastian Raschka

Python implementation: CPython

Multilayer Perceptron From Scratch (Sigmoid activation, MSE

sigmoid activation in the hidden layer

Settings and Dataset

# Note transforms.ToTensor() scales input images

# Checking the dataset

Image batch dimensions: torch.Size([100, 1, 28, 28])

def __init__(self, num_features, num_hidden, num_classes):

def forward(self, x):

# input dim: [n_hidden, n_features] dot [n_features, n_examples] .T

def backward(self, x, a_1, a_2, y):

# input/output dim: [n_examples, n_classes]

# input/output dim: [n_examples, n_classes]

# output dim: [n_examples, n_classes]

# gradient for output weights

# input dim: [n_classlabels, n_examples] dot [n_examples, n_hidden]

# output dim: [n_examples, n_hidden]

# output dim: [n_hidden, n_features]

return dloss__dw_out, dloss__db_out, dloss_dw1, dloss_db1

def to_onehot(y, num_classes):

def loss_func(targets_onehot, probas_onehot):

def compute_mse(net, data_loader):

curr_mse = torch.mean(curr_mse/num_examples, dim=0)

def train(model, data_loader, num_epochs,

for batch_idx, (features, targets) in enumerate(train_loader):

features = features.view(-1, 28*28)

#### Compute outputs ####

#### Compute gradients ####

#### Update weights ####

#### Logging ####

#### Logging ####

return minibatch_cost, epoch_cost

minibatch_cost, epoch_cost = train(model,

Epoch: 001/050 | Batch 000/600 | Cost: 0.2386

def compute_accuracy(net, data_loader):

print('Training Accuracy: %.2f' % compute_accuracy(model, train_loader))

Training Accuracy: 94.69

for features, targets in test_loader:

_, predictions = model.forward(features[:4].view(-1, 28*28))

Predicted labels tensor([7, 2, 1, 0])

You might also like

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

def init(self, num_features, num_hidden, num_classes):