GradientDescent - Single Variable

 

 

 

Followings are the code that I wrote in Octave to creates all the plots shown in this page. You may copy these code and play with these codes. Change variables and try yourself until you get your own intuitive understanding.

 

< Code 1 >

 

function main

  

t = -2:0.01:2;

y = t .^ 2;

 

v = -1.5;

xSpan = 1.0;

xStep = 0.01;

alpha = 0.995;

 

nStep = 60;

 

hFig = figure(1,'Position',[300 300 700 600]);

 

plot(t,y,'LineWidth',2);

axis([-2 2 -1 4]);

grid on;

hold on;

tStr = sprintf("alpha = %0.3f",alpha);

title(tStr);

 

for n = 1:nStep

  

   [tx, ty] = GetTangentLineAt(t,y,v,xSpan,xStep);

   vIdx =  GetLowerMaxIndex(t,v);

   vy = y(vIdx);

   vs = GetSlopAt(t,y,v);

 

   if n > 1

     plot(vp,vyp,'ro','MarkerFaceColor',[1 0 0]);

     line([vp v],[vyp vy],'color','black');

   end  

   plot(v,vy,'ro','MarkerFaceColor',[1 0 0]);

   

   vp = v;

   vyp = vy;

   v = v - (alpha * vs);

   

end;

 

hold off;

 

end

 

function [tx,ty] = GetTangentLineAt(x,y,v,xSpan,xStep)

          

          a = GetSlopAt(x,y,v);

          xIdx = GetLowerMaxIndex(x,v);

          x1 = x(xIdx);

          y1 = y(xIdx);

          b = (y1 - a*x1);

          

          tx = (v - 0.5*xSpan) : xStep : (v + 0.5*xSpan);

          ty = (a .* tx) + b;

          

endfunction  

 

function s = GetSlopAt(x,y,v)

  

   i = GetLowerMaxIndex(x,v);

   dx = x(i+1)-x(i);

   dy = y(i+1)-y(i);

   s = dy / dx;

  

endfunction  

 

function idx = GetLowerMaxIndex(x,v)

  

  idx = 1;

  for i = 1:length(x)

    if x(i) > v

       idx = i-1;

       break;

    end;     

  end;

  

endfunction