Assignment # 01

Computer Vision (MSc. Electrical Engineering)

Submitted To:
Dr. Muhammad Haroon Yousaf
Submitted by:
Babar Ali 17F-MS-EE-15
Syed Mohsin 17F-MS-EE-11
Task 1:
 Both of the group members capture their own face snapshot in a
symmetrical pose. In case of individual/veil, you can take
snapshot of any friends or sibling.
 Apply image registration algorithm to align both of the images.
 Generate Gaussian pyramid of both snapshots and display in the
pattern as in lecture slide # 22.
 Generate Laplacian pyramid of both snapshots and display in the
pattern as in lecture slide # 22.
 (a).Apply image blending concept discussed in the lecture to blend
your faces w.r.t. vertical and horizontal half i.e. like apple and
orange case.
(b).Apply image blending concept discussed in the lecture and
swap one eye of both of your
faces.
 Present your results using GUI.

MATLAB Simulation Code:

Image_blending.m
close all;
clear all;
clc;
imga = im2double(imread('Awad_img.jpg'));
imgb = im2double(imread('Ehatisham_img.jpg')); % size(imga) = size(imgb)
register_image(imga,imgb);
imga = imresize(imga,[size(imgb,1) size(imgb,2)]);
[M N ~] = size(imga);
mask=im2bw(imread('image_mask.jpg'));
level = 5;
% blend by pyramid
img_output=blend_images(imga,imgb,mask,level);
figure,imshow(img_output) % blend by pyramid
Blend_images.m

function imgo=blend_images(imga,imgb,mask,level)
maska= repmat((1.*mask),[1 1 3]);
limga = generate_Pyramid(imga,'lap',level); % the Laplacian pyramid
limgb = generate_Pyramid(imgb,'lap',level);
maskb = 1-maska;
blurh = fspecial('gauss',30,15); % feather the border
maska = imfilter(maska,blurh,'replicate');
maskb = imfilter(maskb,blurh,'replicate');
limgo = cell(1,level); % the blended pyramid
for p = 1:level
[Mp Np ~] = size(limga{p});
maskap = imresize(maska,[Mp Np]);
maskbp = imresize(maskb,[Mp Np]);
limgo{p} = limga{p}.*maskap + limgb{p}.*maskbp;
end
for p = length(limgo)-1:-1:1
limgo{p} = limgo{p}+pyramid_expand(limgo{p+1});
end
imgo = limgo{1};
end
Generate_pyramid.m

function [ pyr ] = generate_Pyramid(image, type, level)

%This function generates Gaussian or Laplacian pyramid
pyr = cell(1,level);
pyr{1} = im2double(image);
for p = 2:level
pyr{p} = pyramid_reduce(pyr{p-1});
end
if strcmp(type,'gauss'), return;
end
for p = level-1:-1:1 % adjust the image size
osz = size(pyr{p+1})*2-1;
pyr{p} = pyr{p}(1:osz(1),1:osz(2),:);
end
for p = 1:level-1
pyr{p} = pyr{p}-pyramid_expand(pyr{p+1});
end
end

Pyramid_expand.m

function [ imgout ] = pyramid_expand(image)

image = im2double(image);
sz = size(image(:,:,1));
osz = sz*2-1;
imgout = zeros(osz(1),osz(2),size(image,3));
kw = 5; % default kernel width
cw = .375; % kernel centre weight
ker1d = [.25-cw/2 .25 cw .25 .25-cw/2];
kernel = kron(ker1d,ker1d')*4;
% Expansion
for p = 1:size(image,3)
img1 = image(:,:,p);
img1ph = padarray(img1,[0 1],'replicate','both'); % horizontally padded
img1pv = padarray(img1,[1 0],'replicate','both'); % horizontally padded
img00 = imfilter(img1,kernel(1:2:kw,1:2:kw),'replicate','same');
img01 = conv2(img1pv,kernel(1:2:kw,2:2:kw),'valid');
img10 = conv2(img1ph,kernel(2:2:kw,1:2:kw),'valid');
img11 = conv2(img1,kernel(2:2:kw,2:2:kw),'valid');
imgout(1:2:osz(1),1:2:osz(2),p) = img00;
imgout(2:2:osz(1),1:2:osz(2),p) = img10;
imgout(1:2:osz(1),2:2:osz(2),p) = img01;
imgout(2:2:osz(1),2:2:osz(2),p) = img11;
end
end
Pyramid_reduce.m

function [ imgout ] = pyramid_reduce(image)

%This function reduces image pyramid
%If input image A is M-by-N, then the size of output image is ceil(M/2)-
 byceil(N/2).
image = im2double(image);
sz = size(image);
imgout = [];
kernelWidth = 5; % default
cw = .375; % kernel centre weight
ker1d = [.25-cw/2 .25 cw .25 .25-cw/2];
kernel = kron(ker1d,ker1d');
%Reduction
for p = 1:size(image,3)
img1 = image(:,:,p);
imgFiltered = imfilter(img1,kernel,'replicate','same');
imgout(:,:,p) = imgFiltered(1:2:sz(1),1:2:sz(2));
end

Simulation Results:
Output results are shown in the form of figures

Fig 1.1: Image pair after Alignment

Fig. 1.2(a): Laplacian Pyramid of Babar Ali.jpg

Fig. 1.2(b): Laplacian Pyramid of Syed Mohsin.jpg

 Figure 1.2(c): Gaussian Pyramid of Babar Ali .Jpg

Task 2:
 Read images Search_Image.jpg and Template_Image.jpg.
 Write code for template matching scheme to find out the
template_Image.png in the Search_Image.png.
 III. Apply template matching algorithms using image pyramids
across 4-levels and propose a computational efficient way of
finding the template image in the search image.
  IV. Compare and present your results and computational cost in
case II and III.

Solution:
Proposed template matching schemes are:
 Template Matching by using Correlation in frequency domain
 Template matching by using Sum of Squared Difference
Correlation in time domain takes a lot of time for finding out the location
of a template in an image. Matlab has some built in function for frequency
domain processing such as FFT and IFFT which are very fast. So processing
image in frequency domain for template matching gives fast results and
the results are also accurate.

MATLAB Code:
Task Template Matching
function Task2_template_matching
clear all;
close all;
clc;
level=4;
I = rgb2gray(imread('Search_Image.png'));
I_pyr = generate_Pyramid_uint8((I),'gauss',level);
% Use a second similar chip as the template
T = rgb2gray(imread('Template_Image.png'));
T_pyr = generate_Pyramid_uint8((T),'gauss',level);

figure('Name','Input image with template','NumberTitle','off');

subplot(1,2,1); imshow(I); title('Input image');
subplot(1,2,2); imshow(T); title('Template');

for i=1:level
% Calculate NCC between Template and Image
tic;
I_NCC{i}=template_matching_by_Corr(T_pyr{i},I);
 toc;
I_NCC_img=I_NCC{i};
% Find maximum correspondence location in image
[x_Corr{i} y_Corr{i}]=(find(I_NCC_img==max(I_NCC_img(:))));
x1=x_Corr{i};
y1=y_Corr{i};

% Calculate SSD between Template and Image

tic;
I_SSD{i}=template_matching_by_SSD(T_pyr{i},I);
toc;
I_SSD_img=I_SSD{i};

% Find maximum correspondence location in image

[x_SSD{i} y_SSD{i}]=(find(I_SSD_img==max(I_SSD_img(:))));
x2=x_SSD{i};
y2=y_SSD{i};

%Mark matched template in input image

hmi1 = vision.MarkerInserter('Size', 10, 'Fill', true, 'FillColor', 'White','Opacity',
0.75);
loc1= int32([y1 x1]);
j1 = step(hmi1, I, loc1);
figure('Name',strcat('Matched template in input image by using Correlation at
Level-',num2str(i)),'NumberTitle','off');
imshow(j1); impixelinfo; title(strcat('Marked Target by using Correlation at
Level-',num2str(i)));
hmi2 = vision.MarkerInserter ('Size', 10, 'Fill', true, 'FillColor',
'Black','Opacity',0.75);
loc2= int32([y2 x2]);
j2 = step(hmi2, I, loc2);
figure('Name',strcat('Matched template in input image by using SSD at Level-
',num2str(i)),'NumberTitle','off');
imshow(j2); impixelinfo; title(strcat('Marked Target by using SSD at Level-',
num2str(i)));
end
end
Template_matching_by_SSD.m :

function [I_SSD]=template_matching_by_SSD(T,I)
% Convert images to double
T=double(T);
I=double(I);
[I_SSD]=template_matching_gray_SSD(T,I);
end
%%
function [I_SSD]=template_matching_gray_SSD(T,I)
% Calculate correlation output size = input size + padding template
T_size = size(T); I_size = size(I);
outsize = I_size + T_size-1;
% Calculate correlation in frequency domain
FT = fft2(rot90(T,2),outsize(1),outsize(2));
FI = fft2(I,outsize(1),outsize(2));
Icorr = real(ifft2(FI.* FT));
% Calculate Local Quadratic sum of Image and Template
LocalQSumI= local_sum(I.*I,T_size);
QSumT = sum(T(:).^2);
% SSD between template and image
I_SSD=LocalQSumI+QSumT-2*Icorr;
% Normalize to range 0..1
I_SSD=I_SSD-min(I_SSD(:));
I_SSD=1-(I_SSD./max(I_SSD(:)));
% Remove padding
I_SSD=unpadarray(I_SSD,size(I));
end
%%
function B=unpadarray(A,Bsize)
Bstart=ceil((size(A)-Bsize)/2)+1;
Bend=Bstart+Bsize-1;
B=A(Bstart(1):Bend(1),Bstart(2):Bend(2));
end
%%
function local_sum_I= local_sum(I,T_size)
% Add padding to the image
B = padarray(I,T_size);
% Calculate for each pixel the sum of the region around it,
% with the region the size of the template.
% 2D localsum
s = cumsum(B,1);
c = s(1+T_size(1):end-1,:)-s(1:end-T_size(1)-1,:);
s = cumsum(c,2);
local_sum_I= s(:,1+T_size(2):end-1)-s(:,1:end-T_size(2)-1);
end

Template_matching_by_Corr.m :

function [I_NCC]=template_matching_by_Corr(T,I)
%%
% Convert images to double
T=double(T);
I=double(I);
[I_NCC]=template_matching_gray_Corr(T,I);
end
%%
function [I_NCC]=template_matching_gray_Corr(T,I)
% Calculate correlation output size = input size + padding template
T_size = size(T); I_size = size(I);
outsize = I_size + T_size-1;
% Calculate correlation in frequency domain
FT = fft2(rot90(T,2),outsize(1),outsize(2));
FI = fft2(I,outsize(1),outsize(2));
Icorr = real(ifft2(FI.* FT));
% Normalized cross correlation STD
LocalSumI= local_sum(I,T_size);
% Calculate Local Quadratic sum of Image and Template
LocalQSumI= local_sum(I.*I,T_size);
% Standard deviation
stdI=sqrt(max(LocalQSumI-(LocalSumI.^2)/numel(T),0) );
stdT=sqrt(numel(T)-1)*std(T(:));
% Mean compensation
meanIT=LocalSumI*sum(T(:))/numel(T);
I_NCC= 0.5+(Icorr-meanIT)./ (2*stdT*max(stdI,stdT/1e5));
% Remove padding
I_NCC=unpadarray(I_NCC,size(I));
end
%%
function B=unpadarray(A,Bsize)
Bstart=ceil((size(A)-Bsize)/2)+1;
Bend=Bstart+Bsize-1;
B=A(Bstart(1):Bend(1),Bstart(2):Bend(2));
end
%%
function local_sum_I= local_sum(I,T_size)
% Add padding to the image
B = padarray(I,T_size);
% Calculate for each pixel the sum of the region around it,
% with the region the size of the template.
% 2D localsum
s = cumsum(B,1);
c = s(1+T_size(1):end-1,:)-s(1:end-T_size(1)-1,:);
s = cumsum(c,2);
local_sum_I= s(:,1+T_size(2):end-1)-s(:,1:end-T_size(2)-1);
end

Simulation Results:
Elapsed time is 1.762435 seconds.
Elapsed time is 0.134010 seconds.
Elapsed time is 0.098576 seconds.
Elapsed time is 0.060992 seconds.
Elapsed time is 0.101479 seconds.
Elapsed time is 0.077654 seconds.
Elapsed time is 0.184681 seconds.
Elapsed time is 0.082164 seconds.
It shows, as we go deeper into sub-sampled level of search image and
template image, results come quickly as there are no. of pixels to process
for template matching. Correlation and SSD gives different location of the
matching template for different sub-sampled levels of template image.
While processing search image at level-1 and template image at level-3,
both SSD and Correlation shows same results for matching template. So
the template is down-sampled to a level-3 and is matched in the search
image
Location of the center of the template in the search image is:
Fig. 2.1: Input Image &Template Image

Fig. 2.2(a): Matched Template by using Correlation at Level -1

Fig. 2.3(a): Matched Template using Correlation at Level-2

Fig. 2.4(a): Matched Template using Correlation at Level-3

Fig. 2.5 (a): Matched Template using Correlation at Level-4

Fig. 2.1 (b): Matched Template by using SSD at Level-1

Fig. 2.2 (b): Matched Template by using SSD at Level-2

Fig. 2.3(b): Matched Template by using SSD at Level-3

Fig. 2.4(b): Matched Template by using SSD at Level-4
Result For Best Match
Fig. 2.4(a): Matched Template using Correlation at Level-3

Fig. 2.3(b): Matched Template by using SSD at Level-3

Task 3:
Process Image 1, Image 2, Image 3 in your assigned images set as follows:
 Apply Harris Corner Detection Algorithm to detect the interest points in all
three images. Draw
any shape around the detected interest points. Show coordinates of
detected interest points in
all images in a Table.
 Apply SIFT descriptor to describe interest points detected in Part 1. Show
the no. of detected interest points and their respective feature vectors.
(Don’t use built-in functions for SIFT Descriptor).

Matlab Simulation Code:

img = imread('F:\usb data\MSCuet taxila\SEMESTER2\Cv\Assignment__1\Images\Set
8 & 9\Image 1.jpg'); %just image Number Replace & Loctaion
img1 = rgb2gray(img);
[Ix, Iy] = imgradientxy(img1); %Calculate x and y gradients,
Ixx = Ix.*Ix;
Iyy = Iy.*Iy;
Ixy = Ix.*Iy;
%applying gaussian filter
h= fspecial('gaussian',9,2); %dimension 9x9,sigma=2
Ix1 = filter2(h,Ixx);%size same ,same as conv
Iy1 = filter2(h,Iyy);
Ixy1 = filter2(h,Ixy);
[a,b] = size(img1);
mat = zeros(a,b);
R = zeros(a,b);
Rmaximum =1000000000; %threshold
k=0.04;
for l = 1:a
for m = 1:b
M = [Ix1(l,m) Ixy1(l,m);Ixy1(l,m) Iy1(l,m)];
R(l,m) = det(M)-k*(trace(M)).^2;
end;
end;
[rows,cols] = find(R>Rmaximum);%interest points=rows,cols
ip=[rows cols] %coordinates enter in table
imshow(img);
hold on;
plot(cols,rows,'ys');
new1=[];
feature=[];
s=size(img1);
%sift descriptor
for m=1:numel(ip)
k=ip(m,1);
l=ip(m,2);
if ((k>8) && (k<s(1)-8)) && ((l>8) && (l<s(2)-8))
 % for neighbourhood
new=img1(k-8:k+7,l-8:l+7);%getting neighbours 16x16 stored in
cell when take its gradient comes 16x16
for n=1:4
for o=1:4 %4x4 region
new1=new(1+(n-1)*4:n*4,1+(o-1)*4:o*4);
[Imag Idir]=imgradient(new1);
Idir=Idir+180; % make Idir 0 to 360 from -180 to 180
if Idir(n,o)<=22.5 && Idir(n,o)>337.5
Idir(n,o)=0;
elseif Idir(n,o)>22.5 && Idir(n,o)<=67.5
Idir(n,o)=45;
elseif Idir(n,o)>67.5 && Idir(n,o)<=112.5
Idir(n,o)=90;
elseif Idir(n,o)>112.5 && Idir(n,o)<=157.5
Idir(n,o)=135;
elseif Idir(n,o)>157.5 && Idir(n,o)<=202.5
Idir(n,o)=180;
elseif Idir(n,o)>202.5 && Idir(n,o)<=247.5
Idir(n,o)=225;
elseif Idir(n,o)>247.5 && Idir(n,o)<=292.5
Idir(n,o)=270;
elseif Idir(n,o)>292.5 && Idir(n,o)<=337.5
Idir(n,o)=315;
end

[r1 c1]=find(Idir(n,o)==0) %approximate and add magnitudes

zero=[r1 c1]
if isempty(zero)
sum(1)=0;
else
for p=1:numel(zero,1)
sum(1)=sum(1)+Imag(zero(p,:));
end
end
sum=0
[r1 c1]=find(Idir(n,o)==45)
zero=[r1 c1]
if isempty(zero)
sum(2)=0;
else
for p=1:numel(zero,1)
sum(2)=sum(2)+Imag(zero(p,:));
end
end

[r1 c1]=find(Idir(n,o)==90)
zero=[r1 c1]
if isempty(zero)
sum(3)=0;
else
for p=1:numel(zero,1)
sum(3)=sum(3)+Imag(zero(p,:));
end
end
[r1 c1]=find(Idir(n,o)==135)
 zero=[r1 c1]
if isempty(zero)
sum(4)=0;
else
for p=1:numel(zero,1)
sum(4)=sum(4)+Imag(zero(p,:));
end
end
[r1 c1]=find(Idir(n,o)==180)
zero=[r1 c1]
if isempty(zero)
sum(5)=0;
else
for p=1:numel(zero,1)
sum(5)=sum(5)+Imag(zero(p,:));
end
end
[r1 c1]=find(Idir(n,o)==225)
zero=[r1 c1]
if isempty(zero)
sum(6)=0;
else
for p=1:numel(r1)
sum(6)=sum(6)+Imag(zero(p,:));
end
end
[r1 c1]=find(Idir(n,o)==270)
zero=[r1 c1]
if isempty(zero)
sum(7)=0;
else
for p=1:numel(zero,1)
sum(7)=sum(7)+Imag(zero(p,:));
end
end
[r1 c1]=find(Idir(n,o)==315)
zero=[r1 c1]
if isempty(zero)
sum(8)=0;
else
for p=1:numel(zero,1)
sum(8)=sum(8)+Imag(zero(p,:)); %sum contain 8 values, no of
cells=16, 16x8=128 feature vector for 1 keypoint
end
end
f(1+n*(0-o))=sum;
end

end
sum=[];
feature=[feature;f]
end
end
Simulation Results:
Fig.0-1
Fig.1-3
Fig.3-5
Fig.6-8
Fig.8-10