(* https://mathematica.stackexchange.com/questions/54545/is-it-possible-to-define-a-new-plottheme *)
Begin["System`PlotThemeDump`"];
Themes`ThemeRules;(*preload Theme system*)

resolvePlotTheme["myTheme",def:_String]:=
Themes`SetWeight[Join[
{ImageSize->Large},
{LabelStyle->Directive[FontFamily->"Libertinus Serif",FontSize->20]},
{TicksStyle->Directive[FontSize->16]}
]
,$ComponentWeight];

End[];

Data

SeedRandom[18]

T={0,1,0,0};
x={-1,0,1,2};
M=Length[T]
η=0.8;

xClass1=x[[Flatten[Position[T,0]]]]

xClass2=x[[Flatten[Position[T,1]]]]

plotPoints=NumberLinePlot[{xClass1,xClass2},
PlotTheme->"myTheme",
Spacings->None,
PlotLegends->PointLegend[97,{"(T^μ) = 0","(T^μ) = 1"},
LabelStyle->Directive[FontFamily->"Libertinus Serif",FontSize->20],
LegendMarkerSize->15
],
AxesLabel->{Style["x",FontSlant->Italic]},
PlotStyle->Directive[PointSize[0.015]]
]

Functions

f[x_]:=LogisticSigmoid[x];
y[w_,b_,x_]:=f[w*x+b]
error[w_,b_]:=(1/2)(∑_(i=1)^M)(((T[[i]]-y[w,b,x[[i]]]))^2)

Batch learning rule (includes the gradient)

initBatch[w0_,b0_]:=Block[{},
ClearAll[wBatch];
ClearAll[bBatch];
wBatch[t_]:=wBatch[t]=wBatch[t-1]+η*(∑_(i=1)^M)(T[[i]]-y[wBatch[t-1],bBatch[t-1],x[[i]]])*f'[wBatch[t-1]*x[[i]]+bBatch[t-1]]*x[[i]];
bBatch[t_]:=bBatch[t]=bBatch[t-1]+η*(∑_(i=1)^M)(T[[i]]-y[wBatch[t-1],bBatch[t-1],x[[i]]])*f'[wBatch[t-1]*x[[i]]+bBatch[t-1]];
wBatch[0]:=w0;
bBatch[0]:=b0;
]

Incremental learning rule (includes the gradient)

initInc[w0_,b0_]:=Block[{},
ClearAll[weightsInc];
ctn=1;
weightsInc[t_]:=weightsInc[t]=Block[{i,w,b},
(*i=RandomInteger[{1,M}];*)
(* Examples are selected in round-robin mammer *)
i=Mod[ctn,M]+1;
ctn++;
{w,b}=weightsInc[t-1];

{w+η*(T[[i]]-y[w,b,x[[i]]])*f'[w*x[[i]]+b]*x[[i]],
b+η*(T[[i]]-y[w,b,x[[i]]])*f'[w*x[[i]]+b]}
];
weightsInc[0]:={w0,b0};
]

Manual calculation

min=FindMinimum[error[w,b],{w,b}]

η=0.8;
w0=-1;
b0=3;

styleVar[var_]:=Style[var,FontSlant->Italic]

plotErrorSurface[w0_:w0,b0_:b0,rangeW_:{-5,5},rangeB_:{-5,5}]:=Show[
Plot3D[error[w,b],{w,rangeW[[1]],rangeW[[2]]},{b,rangeB[[1]],rangeB[[2]]},
PlotTheme->"myTheme",
PlotStyle->Opacity[0.5],
AxesLabel->{styleVar["w"],styleVar["b"],Row[{styleVar["E"],"(",styleVar["w"],",",styleVar["b"],")"}]},
PerformanceGoal->"Quality",
ViewPoint->{0, -2, 2}
],
ListPointPlot3D[{{w0,b0,error[w0,b0]}},
PlotTheme->"myTheme",
PlotLegends->{"Startposition"},
PlotStyle->Directive[,PointSize[0.025]]
],
ListPointPlot3D[{{min[[2,1,2]],min[[2,2,2]],min[[1]]}},
PlotTheme->"myTheme",
PlotLegends->{"Minimum"},
PlotStyle->Directive[,PointSize[0.025]]
],

ImageSize->700
]

plotErrorSurface[]

Calculation of the gradients

gradW=D[error[w,b],{w}]/.{w->w0,b->b0}//N

gradB=D[error[w,b],{b}]/.{w->w0,b->b0}//N

Gradient in the plot

Show[
plotErrorSurface[],
Graphics3D[Arrow[Tube[{
{w0,b0,error[w0,b0]},
{w0-gradW,b0-gradB,error[w0-gradW,b0-gradB]}
}]]]
]

One iteration of the learning rule

initBatch[w0,b0]

Table[wBatch[t],{t,0,6}]//N

Table[bBatch[t],{t,0,6}]//N

Following the gradients further, we get the following path

Show[
plotErrorSurface[w0,b0],
ListPointPlot3D[Table[{wBatch[t],bBatch[t],error[wBatch[t],bBatch[t]]},{t,0,800}],
PlotTheme->"myTheme",
PlotStyle->Directive[,Opacity[0.5]],
PlotLegends->LineLegend[{"Pfad"}]
]/.Point[a___]:>{Thick,Line[a]},
ListPointPlot3D[Table[{wBatch[t],bBatch[t],error[wBatch[t],bBatch[t]]},{t,0,800}],
PlotStyle->Directive[,PointSize[0.01]]
]
]

Batch vs. incremental

plot[w0_,b0_,η0_]:=Block[{},
η=η0;
initInc[w0,b0];
initBatch[w0,b0];

Show[
plotErrorSurface[w0,b0],
ListPointPlot3D[Table[{wBatch[t],bBatch[t],error[wBatch[t],bBatch[t]]},{t,0,400}],
PlotTheme->"myTheme",
PlotStyle->,
PlotLegends->LineLegend[{"Pfad 1"}]
]/.Point[a___]:>{Thick,Line[a]},
ListPointPlot3D[Table[{weightsInc[t][[1]],weightsInc[t][[2]],error[weightsInc[t][[1]],weightsInc[t][[2]]]},{t,0,2000}],
PlotTheme->"myTheme",
PlotStyle->,
PlotLegends->LineLegend[{"Pfad 2"}]
]/.Point[a___]:>{Thick,Line[a]}
]
]

plot[0.7,3.9,3.6]

Learning rate

plotLearningRate[w0_,b0_,η0_]:=Block[{},
η=η0;
initInc[w0,b0];
initBatch[w0,b0];

Column[{
Show[
plotErrorSurface[w0,b0,{-5,2},{-5,0}]/.{"Startposition"->"Start"},

ListPointPlot3D[Table[{wBatch[t],bBatch[t],error[wBatch[t],bBatch[t]]},{t,0,400}],
PlotTheme->"myTheme",
PlotStyle->Directive[,Opacity[0.5]],
PlotLegends->LineLegend[{"Trajectory"}]
]/.Point[a___]:>{Thick,Line[a]},
ListPointPlot3D[Table[{wBatch[t],bBatch[t],error[wBatch[t],bBatch[t]]},{t,0,400}],
PlotStyle->Directive[,PointSize[0.01]]
]
],

DiscretePlot[error[wBatch[t],bBatch[t]],{t,0,400},
PlotTheme->"myTheme",
PlotRange->{0,2},
AxesLabel->{styleVar["t"],Row[{styleVar["E"],"(",styleVar["w"],"(",styleVar["t"],"),",styleVar["b"],"(",styleVar["t"],"))"}]},
ImageSize->700
]
},Center]
]

wStart=-2;
bStart=-2;
Manipulate[
plotLearningRate[wStart,bStart,lern]
,{{lern,0.5,"η"},0.2,20}]

(*Export[
FileNameJoin[{
NotebookDirectory[],
"frames/eta=00.png"
}],
Flatten[Table[plotLearningRate[wStart,bStart,ηControl],{ηControl,0.2,20,0.2}],1],
"VideoFrames",
Antialiasing->True
];*)

Network result

yMin[x_]:=y[min[[2,1,2]],min[[2,2,2]],x]

colorFun=({Opacity[0.3],ColorData["GrayYellowTones"][#]}&);

Show[
ListPlot[{
{xClass1,ConstantArray[0,Length[xClass1]]}\[Transpose],
{xClass2,ConstantArray[1,Length[xClass2]]}\[Transpose]
},
PlotTheme->"myTheme",
AxesLabel->{styleVar["x"],Row[{"(Subscript[y, min])(",styleVar["x"],")"}]},
PlotStyle->Opacity[0]
],

DensityPlot[yMin[x],{x,-1.5,2.5},{x2,-1,2},
PlotTheme->"myTheme",
ColorFunction->colorFun,
(* It is unclear why, but using a different image size for the density plot makes the bar legend wider *)
ImageSize->595,
PlotLegends->Placed[
BarLegend[{colorFun,{yMin[2.5],yMin[-1.5]}},
LegendLayout->"ReversedRow",
LegendLabel->Placed["Netzwerkausgabe (farblich codiert)",Bottom],
(* Manually adjust so that the bar legend fits roughly together with the plot *)
LegendMargins->{{-30,0},{0,0}}
],
Bottom
]
],

Plot[yMin[x],{x,-1.5,2.5},PlotTheme->"myTheme",PlotStyle->,PlotLegends->{"Netzwerkausgabe"}],

(* Plot the points again so that they appear on top *)
ListPlot[{
{xClass1,ConstantArray[0,Length[xClass1]]}\[Transpose],
{xClass2,ConstantArray[1,Length[xClass2]]}\[Transpose]
},
PlotLegends->PointLegend[97,{"(T^μ) = 0","(T^μ) = 1"},
LabelStyle->Directive[FontFamily->"Libertinus Serif",FontSize->20],
LegendMarkerSize->15
]
]
]

Appendix

ColorData[97,"ColorList"]

Manipulate[
plot[init[[1]]+0.001,init[[2]]+0.002,lern]
,{{init,{w0,b0}},{-5,-5},{5,5}},{lern,0.5,10}]