Common

fontLabels=Directive[FontFamily->"Libertinus Serif",FontSize->22];
fontTicks=Directive[FontFamily->"Libertinus Serif",FontSize->18];

(* https://mathematica.stackexchange.com/questions/54545/is-it-possible-to-define-a-new-plottheme *)
Begin["System`PlotThemeDump`"];
Themes`ThemeRules;(*preload Theme system*)

resolvePlotTheme["myTheme",def:_String]:=
Themes`SetWeight[Join[
{ImageSize->Large},
{LabelStyle->fontLabels},
{TicksStyle->fontTicks},
{FrameTicksStyle->fontTicks},
{PlotStyle->Directive[PointSize[0.013],Thick]},
(* 102 is the default grey level of Mathematica *)
{AxesStyle->Directive[AbsoluteThickness[0.3],GrayLevel[(102/255)]]}
]
,$ComponentWeight];

End[];

it[var_]:=Style[var,FontSlant->Italic]
bf[var_]:=Style[var,FontWeight->Bold]
bi[var_]:=bf[it[var]]

equalNumberForm[x_,numbLeft_,numbRight_]:=Module[{digits,exponent,lengthLeft},
If[!NumberQ[x],Return[x]];
{digits,exponent}=RealDigits[x];

(* Make sure that the list is long enough *)
digits=ArrayPad[digits,{({{\[Piecewise], ({{0, exponent≥0}, {Abs[exponent], True}})}}),numbRight}];
exponent=Max[exponent,0];
lengthLeft=Length[digits[[1;;exponent]]];

StringJoin[ToString[#]&/@ConstantArray["0",numbLeft-lengthLeft]]<>
StringJoin[ToString[#]&/@digits[[1;;lengthLeft]]]<>"."<>
StringJoin[ToString[#]&/@digits[[exponent+1;;exponent+numbRight]]]
]

rasterizeBackground[graphics_,res_: 450]:=Block[{g,fig},
fig=graphics;

If[Head@graphics==Legended,
g=fig[[1]],
g=fig
];

(* Rasterize only the image parts of the figure (https://mathematica.stackexchange.com/questions/13744/rasterized-image-together-with-vector-graphics-axes/13746#13746) *)
g=Show[
Rasterize[
Show[g,
PlotRangePadding->0,
ImagePadding->0,
ImageMargins->0,
LabelStyle->Opacity[0],
FrameTicksStyle->Opacity[0],
FrameStyle->Opacity[0],
AxesStyle->Opacity[0],
TicksStyle->Opacity[0],
PlotRangeClipping->False
],
ImageResolution->res
]/.Raster[data_,rect_,rest__]:>Raster[data,Transpose@OptionValue[AbsoluteOptions[g,PlotRange],PlotRange],rest],Sequence@@Options[g],Sequence@@Options[g,PlotRange]
];

If[Head@graphics==Legended,
fig[[1]]=g,
fig=g
];

fig
]

Data

data={{0.34601851851851845`,9.29045864275833`},{0.3834259259259259`,7.777297227353334`},{0.42083333333333334`,6.475978410105039`},{0.626574074074074`,4.902290538083845`},{0.47694444444444445`,3.5404452642193505`},{0.6452777777777776`,1.6943883374252575`},{2.010648148148148`,1.1799134561875597`},{3.020648148148148`,1.210176684495659`},{4.011944444444444`,1.02859731464706`},{5.9571296296296286`,1.0891237712632593`},{7.453425925925925`,1.1799134561875597`},{8.762685185185184`,1.210176684495659`},{9.828796296296296`,1.7246515657333572`},{9.941018518518518`,3.7522878623760496`},{9.772685185185184`,5.840450615634942`},{9.8475`,7.807560455661434`},{9.75398148148148`,9.865459980612226`},{8.18287037037037`,9.502301240915028`},{6.144166666666665`,9.593090925839327`},{4.7413888888888875`,9.623354154147428`},{3.1515740740740736`,9.804933523996027`},{1.580462962962963`,9.472038012606928`},{3.2450925925925924`,8.079929510434333`},{3.319907407407407`,7.383875259348036`},{3.3573148148148144`,6.839137149802238`},{3.3012037037037034`,6.475978410105039`},{3.263796296296296`,5.931240300559241`},{3.263796296296296`,5.386502191013443`},{3.2263888888888888`,4.962816994700045`},{3.2263888888888888`,4.327289200229948`},{3.2263888888888888`,3.7522878623760496`},{3.1702777777777773`,3.086496839597852`},{3.13287037037037`,2.4509690451277546`},{0.626574074074074`,7.958876597201934`},{0.6639814814814815`,7.020716519650837`},{0.6452777777777776`,6.173346127024041`},{0.6452777777777776`,5.295712506089144`},{0.626574074074074`,4.599658255002846`},{0.5891666666666666`,3.5404452642193505`},{0.5891666666666666`,2.7838645565168534`},{0.5891666666666666`,2.057547077122456`},{0.626574074074074`,8.352298565207231`},{0.47694444444444445`,7.353612031039936`},{0.3273148148148148`,3.4799188076031506`},{0.34601851851851845`,5.325975734397244`},{0.439537037037037`,4.448342113462347`},{0.3647222222222222`,3.358865894370751`},{0.42083333333333334`,2.1785999903548556`},{1.4121296296296295`,1.3009663694199594`},{0.7949074074074074`,9.562827697531228`},{2.440833333333333`,9.744407067379827`},{2.833611111111111`,9.13914250121783`},{2.4221296296296293`,9.472038012606928`},{1.4121296296296295`,9.683880610763627`},{1.991944444444444`,9.804933523996027`},{0.9632407407407406`,0.9983340863389598`},{1.6178703703703703`,0.8167547164903608`},{2.515648148148148`,0.8167547164903608`},{2.7400925925925925`,1.2707031411118592`},{1.7862037037037037`,7.535191400888536`},{1.7862037037037037`,5.265449277781044`},{1.6739814814814813`,3.5404452642193505`}};

σ=1;
K[x_]:=(1/(2*π*(σ^2)))(E^(-((((x[[1]])^2)+((x[[2]])^2))/(2*(σ^2)))))
h=0.8;
error[w1_,w2_]:=(1/((h^2)*Length[data]))(∑_(i=1)^(Length[data]))K[((data[[i]]-{w1,w2})/h)]

grad1=D[error[w1,w2],{w1}];
grad2=D[error[w1,w2],{w2}];

(*errorData=<||>;
errorData["w1"]=Table[w1,{w1,0,9,0.1}];
errorData["w2"]=Table[w2,{w2,1,8,0.1}];
errorData["error"]=Table[error[w1,w2],{w2,1,8,0.1},{w1,0,9,0.1}];
Export[FileNameJoin[{NotebookDirectory[],"AdamOptimizer_FirstOrderMomentsFunctionData.json"}],errorData,"Compact"->True];*)

w1Start=0.8;
w2Start=3.6;

clear[]:=Block[{},
(* This removes all subscript values (https://mathematica.stackexchange.com/a/38650/41589) *)
Clear["Subscript"];

(Subscript[grad, 1])[w1_,w2_]=grad1;
(Subscript[grad, 2])[w1_,w2_]=grad2;
]

(Subscript[w, i_])[t_]:=(Subscript[w, i])[t]=(Subscript[w, i])[t-1]-eta*((((Subscript[m, i])[t])/(1-(beta1^t)))/(Sqrt[(((Subscript[s, i])[t])/(1-(beta2^t)))+1]));

adam[beta1_,beta2_,eta_, iterations_:1000]:=Block[{},
clear[];

(Subscript[s, i_])[0]:=0;
(Subscript[m, i_])[0]:=0;
(Subscript[w, 1])[0]:=w1Start;
(Subscript[w, 2])[0]:=w2Start;

(Subscript[m, i_])[t_]:=(Subscript[m, i])[t]=beta1*(Subscript[m, i])[t-1]+(1-beta1)*(Subscript[grad, i])[(Subscript[w, 1])[t-1],(Subscript[w, 2])[t-1]];
(Subscript[s, i_])[t_]:=(Subscript[s, i])[t]=beta2*(Subscript[s, i])[t-1]+(1-beta2)*(((Subscript[grad, i])[(Subscript[w, 1])[t-1],(Subscript[w, 2])[t-1]])^2);
(Subscript[w, i_])[t_]:=(Subscript[w, i])[t]=(Subscript[w, i])[t-1]-eta*(((Subscript[m, i])[t])/(1-(beta1^t)));

Table[{(Subscript[w, 1])[t],(Subscript[w, 2])[t],error[(Subscript[w, 1])[t],(Subscript[w, 2])[t]]},{t,0,1000}]
]

Plots

Trajectory

color1=;
color2=;

vp=Options[Graphics3D,ViewPoint][[1,2]];

vp

trajectoryClassic=adam[0,1,20];
m1Classic=Table[(Subscript[m, 1])[t],{t,0,150}];
trajectoryAdam=adam[0.95,1,20];
m1Adam=Table[(Subscript[m, 1])[t],{t,0,150}];

plotSurface=Show[
Plot3D[error[w1,w2],{w1,0,8},{w2,1,8},
PlotTheme->"myTheme",
AxesLabel->{"(Subscript[w, 1])","(Subscript[w, 2])",Row[{it["E"],"((Subscript[w, 1]),(Subscript[w, 2]))"}]},
ImageSize->700,
PlotStyle->Opacity[0.7],
ColorFunction->(ColorData["DarkRainbow"][#3]&),
PlotLegends->Automatic,
MeshFunctions->{#3&},
Mesh->40,
PlotPoints->100,
MaxRecursion->10,
ViewPoint->{0.2828737948779217`,-2.6762223452154776`,2.051296267032315`},
PerformanceGoal->"Quality"
],
ListPointPlot3D[{trajectoryClassic,trajectoryAdam},
PlotTheme->"myTheme",
PlotStyle->{Directive[color1,PointSize[0.01]],Directive[PointSize[0.01],color2]},
PlotLegends->PointLegend[{color1,color2},{"(Subscript[β, 1]) = 0","(Subscript[β, 1]) = 0.9"},LegendMarkerSize->20]
]
]

(*β1Min=0;
β1Max=0.99;
β1Step=0.01;
Do[
Export[
FileNameJoin[{
NotebookDirectory[],
"beta1="<>IntegerString[Round[((β1-β1Min)/(β1Step)),1],10,2]<>".json"
}],
adam[β1,1,20,500]\[Transpose],
"Compact"->True
];
,{β1,β1Min,β1Max,β1Step}]*)

plotUpdate=ListLinePlot[{m1Classic,m1Adam},
PlotTheme->"myTheme",
PlotRange->{{0,Automatic},{-0.015,0.002}},
PlotStyle->{color1,color2},
FrameLabel->{it["t"],Row[{Subscript[it["m"],"1"],"(",it["t"],")"}]},
PlotLegends->{"(Subscript[β, 1]) = 0","(Subscript[β, 1]) = 0.95"},
GridLines->Automatic,
Frame->True,
Axes->False
]

(*Export[
FileNameJoin[{NotebookDirectory[],"AdamOptimizer_FirstOrderMomentsComponents.svg"}],
plotUpdate
];*)

Gradient function

adamMeasure[g1_,g2_,beta1_:0.9,beta2_:0.999]:=Module[{m2,s2,η=1},
m2=beta1*(1-beta1)*g1+(1-beta1)*g2;
s2=beta2*(1-beta2)*(g1^2)+(1-beta2)*(g2^2);

Abs[η*((m2/(1-(beta1^2)))/(Sqrt[(s2/(1-(beta2^2)))+(10^(-8))]))]-Abs[η*g2]
]

adamMeasureNoCorrection[g1_,g2_,beta1_:0.9,beta2_:0.999]:=Module[{m2,s2,η=1},
m2=beta1*(1-beta1)*g1+(1-beta1)*g2;
s2=beta2*(1-beta2)*(g1^2)+(1-beta2)*(g2^2);

Abs[η*(m2/(Sqrt[s2+(10^(-8))]))]-Abs[η*g2]
]

minValue=Min[
Minimize[{adamMeasure[g1,g2],-6≤g1≤6,-6≤g2≤6},{g1,g2}][[1]]//N,
Minimize[{adamMeasureNoCorrection[g1,g2],-6≤g1≤6,-6≤g2≤6},{g1,g2}][[1]]//N
];
maxValue=Max[
Maximize[{adamMeasure[g1,g2],-6≤g1≤6,-6≤g2≤6},{g1,g2}][[1]]//N,
Maximize[{adamMeasureNoCorrection[g1,g2],-6≤g1≤6,-6≤g2≤6},{g1,g2}][[1]]//N
];

scale[x_]:=({{\[Piecewise], ({{(x/maxValue)*0.5+0.5, x≥0}, {((x-minValue)/(0-minValue))*0.5, x<0}})}});

plotAdam=DensityPlot[adamMeasure[g1,g2],{g1,-6,6},{g2,-6,6},
PlotTheme->"myTheme",
ImageSize->500,
ColorFunctionScaling->False,
ColorFunction->Function[{x},ColorData["TemperatureMap"][scale[x]]],
Exclusions->None,
PlotPoints->200,
FrameLabel->{"(Subscript[g, 1])","(Subscript[g, 2])"},
PlotLegends->Placed[
BarLegend[Automatic,LabelStyle->fontTicks,LegendMarkerSize->462,LegendMargins->{{42,0},{0,0}},LegendLabel->Placed[Style["(Subscript[C, A])((Subscript[g, 1]),(Subscript[g, 2]))",fontLabels],Above]]
,Above]
]

plotAdamNoCorrection=DensityPlot[adamMeasureNoCorrection[g1,g2],{g1,-6,6},{g2,-6,6},
PlotTheme->"myTheme",
ImageSize->500,
ColorFunctionScaling->False,
ColorFunction->Function[{x},ColorData["TemperatureMap"][scale[x]]],
Exclusions->None,
PlotPoints->200,
FrameLabel->{"(Subscript[g, 1])","(Subscript[g, 2])"},
PlotLegends->Placed[
BarLegend[Automatic,LabelStyle->fontTicks,LegendMarkerSize->462,LegendMargins->{{42,0},{0,0}},LegendLabel->Placed[Style[Row[{Subscript["(OverTilde[C])",it["A"]],"((Subscript[g, 1]),(Subscript[g, 2]))"}],fontLabels],Above]]
,Above]
]

Round[((Sqrt[1-(0.999^2)])/(1-(0.9^2))),0.0001]

(1-(0.999^2))*(10^(-8))

(*Export[
FileNameJoin[{NotebookDirectory[],"AdamOptimizer_BiasCorrectionEnabled.svg"}],
rasterizeBackground[plotAdam]
];
Export[
FileNameJoin[{NotebookDirectory[],"AdamOptimizer_BiasCorrectionDisabled.svg"}],
rasterizeBackground[plotAdamNoCorrection]
];*)