\section{Introduction}

Gradient descent (GD) optimization methods for the training of artificial neural networks (ANNs) belong these days to the most heavily employed numerical schemes in the digital world. In particular, such computational techniques are significantly employed in a multiplicity of applications ranging from text processing and speech recognition to computer vision, just to name a few (cf., e.g., Le Cun et al.~\cite{LeCunBengioHinton2015} and Bottou et al.~\cite[Section 2]{Bottou2018optimization}).

Despite the ubiquitous presence of GD optimization methods in the training of ANNs, it remains a fundamental open problem of research to provide a rigorous theoretical justification for the practical success of such methods. GD optimization methods (by which we mean the negative gradient flow dynamics, deterministic GD schemes, as well as stochastic GD schemes) walk along the negative gradient with the aim to minimize the risk function of the considered supervised learning problem. The fundamental challenge, which arises in the mathematical analysis of such methods, is the fact that there are many non-optimal critical points (many saddle points as well as many non-global local minimum points) in the optimization landscape (cf., e.g.,~\cite{ShokhrukhLocalMinima,Safran2018spurious,Swirszcz2017local}) whose risk values are strictly larger than the optimal risk value (the smallest possible risk value) and that the probability of the domain of attraction of the non-global local minimum points does not vanish (see, e.g., Cheridito et al.~\cite{FlorianEscapeSaddles2022}).


\def2.5cm{4cm}
\begin{figure}[H]
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tikzpicture}[shorten >=1pt,->,draw=black!50, node distance=2.5cm]
\tikzstyle{every pin edge}=[<-,shorten <=1pt]
\tikzstyle{input neuron}=[very thick, circle,draw=red, fill=red!20, minimum size=15pt,inner sep=0pt]
\tikzstyle{output neuron}=[very thick, circle, draw=green,fill=green!20,minimum size=30pt,inner sep=0pt]
\tikzstyle{hidden neuron}=[very thick, circle,draw=blue,fill=blue!20,minimum size=20pt,inner sep=0pt]
\tikzstyle{annot} = [text width=9em, text centered]
\tikzstyle{annot2} = [text width=4em, text centered]

\node[input neuron] (I) at (0,-1) {$x$};

\path[yshift = 1.5cm]
node[hidden neuron](H0-1) at (2.5cm, -1 cm) {$1^{\text{st}}$};
\path[yshift = 1.5cm]
node[hidden neuron](H0-2) at (2.5cm, -2 cm) {$2^{\text{nd}}$};
\path[yshift = 1.5cm]
node(H0-dots) at (2.5cm, -2.9 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H0-3) at (2.5cm, -4 cm) {$\mathfrak{h}^{\text{th}}$};


\path[yshift = 1.5cm]
node[output neuron](O) at (2*2.5cm,-2.5 cm) {$\mathcal{N}^{\theta}(x)$};   

\foreach \dest in {1,2,3}
\path[line width = 0.8 ] (I) edge (H0-\dest);

\foreach \source in {1,2,3}
\path[line width = 0.8 ] (H0-\source) edge (O);

\node[annot,above of=H0-1, node distance=1cm, align=center] (hl) {Hidden layer\\($2^{\text{nd}}$ layer)};
\node[annot,left of=hl, align=center] {Input layer\\ ($1^{\text{st}}$ layer)};
\node[annot,right of=hl, align=center] {Output layer\\($3^{\text{rd}}$ layer)};

\node[annot2,below of=H0-3, node distance=1cm, align=center] (sl) {$\mathfrak{h}$ neurons};
\node[annot2,left of=sl, align=center] {$1$ neuron};
\node[annot2,right of=sl, align=center] {$1$ neuron};
\end{tikzpicture}
\end{adjustbox}
\caption{Graphical illustration of the ANN architecture considered 
in \cref{theorem_intro_2} and \cref{theorem_intro}: 
ANNs with a $ 1 $-dimensional input layer, $ \mathfrak{h} $-dimensional hidden layer, and $ 1 $-dimensional output layer.}
\label{figure_shallow_nns_general_introduction}
\end{figure}


It is a key contribution of this work to overcome this difficulty in certain simplified training situations where the considered ANNs are shallow with just one hidden layer and a one-dimensional input (see also \cref{figure_shallow_nns_general_introduction} for a graphical illustration of the analyzed ANN architecture), where certain special random initialization schemes are employed, and where only the \emph{bias parameter} of the ANNs but not the \emph{weight parameters} of the ANNs are trained. Specifically, in such simplified ANN training situations we prove under one random initialization
\begin{enumerate}[label = (\Roman*)]
\item
\label{intro_item_I}
that the gradient flow (GF) dynamics overcomes with high probability all \emph{bad} non-global local minimum points (overcomes all non-global local minimum points whose risk values are far away from the optimal risk value of the global minimum points) and
\item
\label{intro_item_II}
that the GF dynamics does converge with high probability to a \emph{good} critical point (does converge with high probability to a critical point whose risk value is very close to the optimal risk value of the global minimum points).
\end{enumerate}
This analysis allows to conclude that with high probability we have that the risk of the GF trajectory $(\Theta_t)_{t \in [0,\infty)}$ converges \emph{to zero} as, both, the training time $t \geq 0$ \emph{and} the width $\mathfrak{h} \in \mathbb{N} = \{1, 2, 3, \dots \}$ of the ANN converge to infinity.


In \cref{theorem_intro_2} below we disclose such a result for ANNs with the clipping activation function $\mathbb{R} \ni x \mapsto \max\{ \min\{x, 1\}, 0\} \in \mathbb{R}$ (see \cref{theorem_intro_2:eq:risk}) and in \cref{theorem_intro} below we present such a result for ANNs with the rectified linear unit (ReLU) activation function $\mathbb{R} \ni x \mapsto \max\{ x, 0 \} \in \mathbb{R}$ (see \cref{eq:theorem_intro_risk_function}). We now first present the precise statement of \cref{theorem_intro_2}, thereafter, we comment on the statement and the mathematical objects in \cref{theorem_intro_2}, and, thereafter, we finally present \cref{theorem_intro}.

\begin{samepage}
\cfclear
\begin{theorem}\label{theorem_intro_2}
Let $\scra \in \mathbb{R}$, $ \scrb \in (\scra, \infty)$, let $f, \mathfrak{p} \in C(\mathbb{R}, \mathbb{R})$ be piecewise polynomial, assume for all $ x \in \mathbb{R} $ that $\mathfrak{p}(x) \ge 0$ and $\mathfrak{p}^{ -1 }( \mathbb{R} \backslash \{ 0 \} ) = ( \scra, \scrb ) $, assume that $ f $ is strictly increasing, let $ ( \Omega, \mathcal{F}, \P ) $ be a probability space, for every $ \mathfrak{h} \in \mathbb{N} $ let $ \mathfrak{d}_\mathfrak{h} = 3 \mathfrak{h} + 1 $, let $ \mathcal{L}_{ \mathfrak{h} } \colon \mathbb{R}^{ \mathfrak{d}_{ \mathfrak{h} } } \to \mathbb{R} $ satisfy for all $ \theta = ( \theta_1, \dots, \theta_{\mathfrak{d}_{ \mathfrak{h} } } ) \in \mathbb{R}^{ \mathfrak{d}_{ \mathfrak{h} } } $ that
\begin{equation}\label{theorem_intro_2:eq:risk}
\textstyle \mathcal{L}_{ \mathfrak{h} }( \theta ) = \int_{\mathbb{R}} (f(x) - \theta_{\mathfrak{d}_\mathfrak{h}} - \sum_{j = 1}^{\mathfrak{h}} \theta_{2 \mathfrak{h} + j} \max\cu{\min\cu{\theta_{\mathfrak{h} + j} x + \theta_{j}, 1}, 0})^2 \mathfrak{p}(x) \, \d x,
\end{equation}
let $\mathcal{G}_\mathfrak{h} = ( \mathcal{G}_\mathfrak{h}^1, \dots, \mathcal{G}_\mathfrak{h}^{ \mathfrak{d}_\mathfrak{h} } )\colon \mathbb{R}^{ \mathfrak{d}_\mathfrak{h} } \to \mathbb{R}^{ \mathfrak{d}_\mathfrak{h} }$ satisfy for all $ k \in \{ 1, 2, \dots, \mathfrak{d}_\mathfrak{h} \} $, $\theta = ( \theta_1, \dots, \theta_{ \mathfrak{d}_{ \mathfrak{h} } } ) \in \mathbb{R}^{ \mathfrak{d}_\mathfrak{h} }$ that $\mathcal{G}_\mathfrak{h}^k(\theta) = ( \frac{ \partial^{ - } }{ \partial \theta_k }\mathcal{L}_\mathfrak{h} )( \theta ) \indicator{ \mathbb{N} \backslash ( \mathfrak{h}, 3 \mathfrak{h}]}( k )$, and let $\Theta^\mathfrak{h} = (\Theta^{\mathfrak{h}, 1}, \ldots, \Theta^{\mathfrak{h}, \mathfrak{d}_\mathfrak{h}}) \colon [0, \infty) \times \Omega \to \mathbb{R}^{\mathfrak{d}_\mathfrak{h}}$ be a stochastic process with continuous sample paths, assume for all $\mathfrak{h} \in \mathbb{N}$, $t \in [0, \infty)$ that
\begin{equation}\label{theorem_intro_2:eq:assgf}
\textstyle \P\bigl(\Theta_t^\mathfrak{h} = \Theta_0^\mathfrak{h} - \int_0^t \mathcal{G}_\mathfrak{h} (\Theta_s^\mathfrak{h}) \, \d s \bigr) = 1,
\end{equation}
assume for all $ \mathfrak{h} \in \mathbb{N} $ that $ \Theta_0^{ \mathfrak{h}, 1 }, \dots, \Theta_0^{ \mathfrak{h}, \mathfrak{d}_\mathfrak{h} } $ are independent, let $ \alpha \in (\nicefrac{ 3 }{ 4 }, 1 ) $, $ \beta \in ( \alpha + 2, \infty ) $, assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \{1, 2, \ldots, \mathfrak{h}\}$ that $\mathfrak{h}^{- \beta} \Theta_0^{\mathfrak{h}, k}$ is standard normal, and assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \{1, 2, \ldots, \mathfrak{h}\}$, $x \in \mathbb{R}$ that $\P(\mathfrak{h}^{- \beta} \Theta_0^{\mathfrak{h}, \mathfrak{h} + k} \leq x) = \P(\mathfrak{h}^{\alpha} \Theta_0^{\mathfrak{h}, 2 \mathfrak{h} + k} \le x) = [\frac{2}{\pi}]^{1/2} \int_0^x \exp(- \frac{y^2}{2}) \, \d y$ and $\mathbb{E}[\abs{\Theta_0^{\mathfrak{h}, \mathfrak{d}_\mathfrak{h}}}^2] < \infty $. Then there exist $\mathfrak{c}, \kappa \in (0, \infty)$ and random variables $\mathfrak{C}_{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $\mathfrak{h} \in \mathbb{N}$, such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $\limsup_{t \to \infty} \mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}})] \leq \mathfrak{c} \mathfrak{h}^{- \kappa}$ and
\begin{equation}\label{eq:theorem_2_intro_statement}
\textstyle \P\bigl(\forall\, t \in (0, \infty) \colon \mathcal{L}_{ \mathfrak{h} }(\Theta_t^{ \mathfrak{h} } ) \le \mathfrak{c} \mathfrak{h}^{- \kappa} + \mathfrak{C}_{\mathfrak{h}} t^{- 1} \bigr) \ge 1 - \mathfrak{c} \mathfrak{h}^{- \kappa}.
\end{equation}
\end{theorem}
\end{samepage}
\cref{theorem_intro_2} is a direct consequence of \cref{theo:main:conv} in \cref{sec:convergence_of_GF_with_random_init_clipping} below and in \cref{sec:anns_with_clipping} below we present the detailed proof of \cref{theo:main:conv}. Next let us add some comments on the statement and the mathematical objects appearing in \cref{theorem_intro_2}.


The real numbers $ \scra, \scrb \in \mathbb{R} $ with $ \scra < \scrb $ in \cref{theorem_intro_2} specify the borders of the support $[\scra, \scrb]$ of the probability distribution of the input data of the considered supervised learning problem. The function $ f \colon \mathbb{R} \to \mathbb{R} $ in \cref{theorem_intro_2} is the target function (the function which we intend to approximate via ANNs) of the considered supervised learning problem and the function $ \mathfrak{p} \colon \mathbb{R} \to \mathbb{R} $ in \cref{theorem_intro_2} is an unnormalized density function of the probability distribution of the input data of the considered supervised learning problem. Both $ f $ and $ \mathfrak{p} $ are assumed to be continuous and piecewise polynomial, by which we mean that there exist $ K \in \mathbb{N} $, $ \tau_0, \tau_1, \dots, \tau_K \in [ - \infty, \infty]$ with
\begin{equation}
- \infty = \tau_0 < \tau_1 < \dots < \tau_K = \infty
\end{equation}
such that for all $ k \in \{ 1, 2, \dots, K \} $ we have that $ f|_{ (\tau_{ k - 1 }, \tau_k ) } $ and $ \mathfrak{p}|_{ ( \tau_{ k - 1 }, \tau_k)}$ are polynomials (cf.\ \cref{def:piecewise_polynomial} in \cref{sec:anns_with_clipping} below). In addition, we also assume in \cref{theorem_intro_2} that the target function $ f $ is strictly increasing.


In \cref{theorem_intro_2} we study the GF dynamics with one random initialization and the triple $ ( \Omega, \mathcal{F}, \P ) $ in \cref{theorem_intro_2} serves as the underlying probability space to specify this random initialization. In \cref{theorem_intro_2} we analyze \emph{shallow fully-connected feedforward ANNs} with $ 1 $ neuron on the input layer (with an $ 1 $-dimensional input layer), with $ \mathfrak{h} \in \mathbb{N} $ neurons on the hidden layer (with an $\mathfrak{h}$-dimensional hidden layer), and with $ 1 $ neuron on the output layer (with an $ 1 $-dimensional output layer). There are thus $ \mathfrak{h} $ weight parameters and $ \mathfrak{h} $ bias parameters to describe the affine linear transformation between the $ 1 $-dimensional input layer and the $ \mathfrak{h} $-dimensional hidden layer and there are thus $\mathfrak{h}$ weight parameters and $ 1 $ bias parameter to describe the affine linear transformation between the $ \mathfrak{h} $-dimensional input layer and the $ 1 $-dimensional output layer. Altogether there are thus
\begin{equation}
\mathfrak{h} + \mathfrak{h} + \mathfrak{h} + 1 = 3 \mathfrak{h} + 1
\end{equation}
real parameters to describe the considered ANNs and it is thus precisely the role of the natural numbers $ \mathfrak{d}_{ \mathfrak{h} } = 3 \mathfrak{h} + 1 $ for $ \mathfrak{h} \in \mathbb{N} $ to specify the number of real parameters in the considered ANNs. We also refer to \cref{figure_shallow_nns_general_introduction} for a graphical illustration of the ANN architecture used in \cref{theorem_intro_2}.


In \cref{theorem_intro_2:eq:risk} we consider the realization of ANNs with the clipping activation function, which is for every $ \theta = (\theta_1, \dots, \theta_{ \mathfrak{d}_{ \mathfrak{h} } } ) \in \mathbb{R}^{ \mathfrak{d}_\mathfrak{h} } $ given by
\begin{equation}
\textstyle \mathbb{R} \ni x \mapsto \theta_{\mathfrak{d}_\mathfrak{h}} + \sum_{j = 1}^{\mathfrak{h}} \theta_{2 \mathfrak{h} + j} \max\cu{\min\cu{\theta_{\mathfrak{h} + j} x + \theta_{j}, 1}, 0} \in \mathbb{R}.
\end{equation}
\Nobs that the clipping function is a continuous sigmoidal function in the sense of Cybenko~\cite{Cybenko1989} and, in particular, admits the universal approximation property (cf., e.g.,~\cite{Cybenko1989,Hornik1989,LeshnoLinPinkusSchocken1993}). Similarly to the more commonly used ReLU activation function, the clipping function is piecewise linear and almost everywhere differentiable. Thus it is possible to define a suitable generalized gradient in an analogous way. In the setting considered in this article, the risk function is actually continuously differentiable due to the integral appearing in \cref{theorem_intro_2:eq:risk}. In addition, the clipping function has the convenient property that it is bounded and its derivative has bounded support, which makes it easier to establish boundedness of the considered GF trajectories in some cases. For further uses of the clipping activation for ANNs we refer, e.g., to \cite{ChoiWang2018PACT} and \cite{SakrChoi2018}.


The functions $ \mathcal{L}_{ \mathfrak{h} } \colon \mathbb{R}^{ \mathfrak{d}_{ \mathfrak{h} } } \to \mathbb{R} $ for $ \mathfrak{h} \in \mathbb{N} $ in \cref{theorem_intro_2} are the risk functions which specify the squared $ L^2 $-distance between the target function $ f \colon \mathbb{R} \to \mathbb{R} $ and its ANN approximations. The functions $\mathcal{G}_\mathfrak{h} = ( \mathcal{G}_\mathfrak{h}^1, \dots, \mathcal{G}_\mathfrak{h}^{ \mathfrak{d}_\mathfrak{h} } ) \colon \mathbb{R}^{\mathfrak{d}_\mathfrak{h}} \to \mathbb{R}^{\mathfrak{d}_\mathfrak{h}} $ for $ \mathfrak{h} \in \mathbb{N} $ in \cref{theorem_intro_2} describes the generalized gradient functions that are used in the training process. Note that the indicator functions $\mathbb{N} \ni k \mapsto \indicator{ \mathbb{N} \backslash ( \mathfrak{h}, 3 \mathfrak{h} ] }( k ) \in \{ 0, 1 \}$ for $ \mathfrak{h} \in \mathbb{N} $ in the description of the generalized gradient functions $\mathcal{G}_{ \mathfrak{h} }\colon \mathbb{R}^{ \mathfrak{d} } \to \mathbb{R}^{ \mathfrak{d} }$, $ \mathfrak{h} \in \mathbb{N} $, ensures that only the bias parameters but not the weight parameters are moved in the training process. We would like to point out that, even though only the bias parameters are trained in \cref{theorem_intro_2}, there are still a large number of \emph{non-global local minimum points in the optimization landscape} in the setup of \cref{theorem_intro_2}.



The stochastic processes $\Theta^{ \mathfrak{h} } = ( \Theta^{ \mathfrak{h}, 1 }, \dots, \Theta^{ \mathfrak{h}, \mathfrak{d}_\mathfrak{h} } ) \colon [0, \infty) \times \Omega \to \mathbb{R}^{ \mathfrak{d}_{ \mathfrak{h} } }$ for $ \mathfrak{h} \in \mathbb{N} $ in \cref{theorem_intro_2} are the randomly initialized solutions of the GF differential equations in \cref{theorem_intro_2:eq:assgf} and the real numbers $ \alpha \in ( \nicefrac{ 3 }{ 4 }, 1 ) $, $ \beta \in (\alpha + 2, \infty ) $ in \cref{theorem_intro_2} are two parameters for the distribution of the GF trajectories $ ( \Theta^{ \mathfrak{h} } )_{t \in [0,\infty) } $ at initial time $ t = 0 $. In \cref{eq:theorem_2_intro_statement} in \cref{theorem_intro_2} we establish with high probability that the risk of the GF trajectory $\mathcal{L}_{ \mathfrak{h} }( \Theta^{ \mathfrak{h} }_t ) $ converges with the convergence rates $ \kappa $ and $ 1 $ to zero as the training time $t$ and the width of the ANN $ \mathfrak{h} $ converge to infinity. In analogy to traditional numerical analysis, say, numerical discretization analyses for partial differential equations (PDEs), the convergence rate $ \kappa $ in \cref{eq:theorem_2_intro_statement} can be regarded as a \emph{spatial approximation order} and the convergence rate $ 1 $ in \cref{eq:theorem_2_intro_statement} can be regarded as a \emph{temporal approximation order}.


In the next result, \cref{theorem_intro} below, we establish a similar but in a certain sense more restrictive result than \cref{theorem_intro_2} above for the ReLU activation function $ \mathbb{R} \ni x \mapsto \max\{ x, 0 \} \in \mathbb{R} $ (see \cref{eq:theorem_intro_risk_function} below) instead of for the clipping activation function $\mathbb{R} \ni x \mapsto \max\{ \min\{ x, 1 \}, 0\} \in \mathbb{R}$ (see \cref{theorem_intro_2:eq:risk} above).


\begin{samepage}
\cfclear
\begin{theorem}\label{theorem_intro}
Let $\scra \in \mathbb{R}$, $\scrb \in (\max\{\scra, 0\}, \infty)$, let $f \in C^2(\mathbb{R}, \mathbb{R})$, $\mathfrak{p} \in C(\mathbb{R}, \mathbb{R})$ be piecewise polynomial, assume for all $x \in (\scra, \scrb)$ that $\min\{f'(x), \allowbreak f''(x)\} \allowbreak > f(\scra) = 0 \le \mathfrak{p}(x)$ and $\mathfrak{p}^{-1}(\mathbb{R} \backslash \{0\}) = (\scra, \scrb)$, let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $\mathfrak{h} \in \mathbb{N}$ let $\mathfrak{d}_{\mathfrak{h}} = 3 \mathfrak{h} + 1$, let $\mathcal{L}_{\mathfrak{h}} \colon \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}} \to \mathbb{R}$ satisfy for all $\theta = (\theta_1, \ldots, \theta_{\mathfrak{d}_{\mathfrak{h}}}) \in \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ that
\begin{equation}\label{eq:theorem_intro_risk_function}
\textstyle \mathcal{L}_{\mathfrak{h}}(\theta) = \int_{\mathbb{R}} (f(x) - \theta_{\mathfrak{d}_{\mathfrak{h}}} - \sum_{j = 1}^{\mathfrak{h}} \theta_{2 \mathfrak{h} + j} \max\{\theta_{\mathfrak{h} + j} x + \theta_{j}, 0\})^2 \mathfrak{p}(x) \, \d x,
\end{equation}
let $\mathcal{G}_{\mathfrak{h}} = (\mathcal{G}_{\mathfrak{h}}^1, \ldots, \mathcal{G}_{\mathfrak{h}}^{\mathfrak{d}_{\mathfrak{h}}}) \colon \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}} \to \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ satisfy for all $k \in \{1, 2, \ldots, \mathfrak{d}_{\mathfrak{h}}\}$, $\theta = (\theta_1, \ldots, \theta_{\mathfrak{d}_{\mathfrak{h}}}) \in \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ that $\mathcal{G}_{\mathfrak{h}}^{k}(\theta) = (\frac{\partial^{-}}{\partial \theta_k} \mathcal{L}_{\mathfrak{h}})(\theta) \mathbbm{1}_{\mathbb{N} \cap (0, \mathfrak{h}]}(k)$, and let $\Theta^{\mathfrak{h}} = (\Theta^{\mathfrak{h}, 1}, \ldots, \Theta^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}) \colon [0, \infty) \times \Omega \to \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ be a stochastic process with continuous sample paths, assume for all $\mathfrak{h} \in \mathbb{N}$, $t \in [0, \infty)$ that 
\begin{equation}
\label{eq:def_GF_dynamics2}
\textstyle \P\bigl(\Theta_t^{\mathfrak{h}} = \Theta_0^{\mathfrak{h}} - \int_0^{t} \mathcal{G}_{\mathfrak{h}}(\Theta_s^{\mathfrak{h}}) \, \d s\bigr) = 1,
\end{equation}
assume for all $\mathfrak{h} \in \mathbb{N}$ that $\Theta_0^{\mathfrak{h}, 1}, \ldots, \Theta_0^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}$ are independent, let $\alpha \in (0, 1)$, $\beta \in (0, \nicefrac{\alpha}{3})$, assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \mathbb{N} \cap (0, \mathfrak{h}] \cup \{\mathfrak{d}_{\mathfrak{h}}\}$ that $\mathfrak{h}^{\alpha} \Theta_0^{\mathfrak{h}, k}$ is standard normal, and assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \mathbb{N} \cap (\mathfrak{h}, 3 \mathfrak{h}]$, $x \in \mathbb{R}$ that $\P(\mathfrak{h}^{\beta} \Theta_0^{\mathfrak{h}, k} \le x) = \int_{0}^{x} [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y$. Then there exist $\mathfrak{c}, \kappa \in (0, \infty)$ and random variables $\mathfrak{C}_{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $\mathfrak{h} \in \mathbb{N}$, such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $\limsup_{t \to \infty} \mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}})] \leq \mathfrak{c} \mathfrak{h}^{- \kappa}$ and
\begin{equation}
\textstyle \P\bigl(\forall\, t \in (0, \infty) \colon \mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}}) \le \mathfrak{c} \mathfrak{h}^{- \kappa} + \mathfrak{C}_{\mathfrak{h}} t^{-1}\bigr) \ge 1 - \mathfrak{c} \mathfrak{h}^{- \kappa}.
\end{equation}
\end{theorem}
\end{samepage}
\cref{theorem_intro} is a direct consequence of \cref{cor:main} in \cref{sec:ANNs_with_ReLU_final_result} below and in \cref{sec:anns_with_relu} we present a detailed proof of \cref{cor:main}. The statement of \cref{theorem_intro} is similar to the statement of \cref{theorem_intro_2} but \cref{theorem_intro} considers the ReLU activation function instead of the clipping activation function and somehow imposes more restrictive assumptions. In particular, we note that in \cref{theorem_intro} \emph{only the inner bias parameters} (the $ \mathfrak{h} $ bias parameters between the $1$-dimensional input layer and the $ \mathfrak{h} $-dimensional hidden layer) but not the outer bias parameter (the bias parameter between the $ \mathfrak{h} $-dimensional hidden layer and the $ 1 $-dimensional output layer) are trained while in \cref{theorem_intro_2} \emph{all bias parameters} are trained.


A result similar to \cref{theorem_intro_2} and \cref{theorem_intro}, respectively, 
has recently been established independently in Gentile \& Welper~\cite{GentileWelper2022} 
where only the inner bias parameters of a shallow neural network with one-dimensional input and ReLU activation are trained by GF,
where the inner weight parameters are assumed to be $1$,
where the outer weight parameters are initialized randomly and uniformly in $\{ - 1 , 1 \}$,
 and where the true risk with respect to a uniform input distribution is considered. 
While the results in \cite{GentileWelper2022} are 
regarding the probability distribution of the input data 
and the values of the inner weight parameters
more restrictive than ours, the target function in \cite{GentileWelper2022} 
is not assumed to be strictly increasing as in 
\cref{theorem_intro_2} and \cref{theorem_intro} 
and the article \cite{GentileWelper2022} even establishes exponential convergence of the GF with high probability. 
The arguments of proof of the main convergence result in \cite{GentileWelper2022} 
follow a completely different approach than the arguments in the proofs of \cref{theorem_intro_2} and \cref{theorem_intro}. 
Specifically, the main convergence result in \cite{GentileWelper2022} is proved by analyzing the neural tangent kernel operator 
on the infinite-dimensional Hilbert space $ L^2( [ -1, 1 ] ) $ and various Sobolev subspaces. 


Let us also comment on some of the shortcomings of \cref{theorem_intro_2} and \cref{theorem_intro}, respectively. First, we would like to point out that \cref{theorem_intro_2} and \cref{theorem_intro} only establish convergence to zero for the time-continuous GF dynamics (see \cref{theorem_intro_2:eq:assgf} and \cref{eq:def_GF_dynamics2}) -- which is, of course, not implementable on a computer -- but not for \emph{time-discrete GD optimization methods}, in particular, not for the practically relevant \emph{stochastic GD method}. Moreover, instead of merely using a large number of input-output data pairs, \cref{theorem_intro_2} and \cref{theorem_intro} also exploit the explicit knowledge of the probability distribution of the input data in the gradient dynamics (see \cref{theorem_intro_2:eq:risk}--\cref{theorem_intro_2:eq:assgf} and \cref{eq:theorem_intro_risk_function}--\cref{eq:def_GF_dynamics2}). However, in our subjective point of view, the key difficulty in the mathematical analysis of the training of ANNs is the issue to overcome with high probability all bad non-global local minimum points which is already fully present in the case of time-continuous GF processes and we expect that it will be possible to extend \cref{theorem_intro_2} and \cref{theorem_intro}, respectively, to cover also these more practically relevant situations  by using, among other things, the findings in Dereich \& Kassing~\cite{DereichKassing2021}. Another key shortcoming of \cref{theorem_intro_2} and \cref{theorem_intro}, respectively, is the fact that only the (inner) bias parameters of the ANNs are trained but not the weight parameters and this shortcoming seems indeed to be extremely hard to relax. Nonetheless, we would like to point out that the optimization landscapes associated to \cref{theorem_intro_2} and \cref{theorem_intro} still admit a large number of bad non-global local minimum points whose risk values are far away from the optimal risk value of the global minimum points. Roughly speaking, our approach in this work is to simplify the training scenario for ANNs in a way
\begin{itemize}
\item so that, one the one hand, the optimization problem becomes significantly \emph{simplified} which enabled us to establish \cref{theorem_intro_2} and \cref{theorem_intro} and
\item so that, one the other hand, the optimization problem is still \emph{difficult} enough to provide a complicated optimization landscape with a large number of bad non-global local minimum points.
\end{itemize}
We conjecture that the central reasons/arguments for the convergence results in \cref{theorem_intro_2} and \cref{theorem_intro} developed in \cref{sec:anns_with_clipping} (in the situation of \cref{theorem_intro_2}) and \cref{sec:anns_with_relu} (in the situation of \cref{theorem_intro}) -- with high probability \emph{overcoming all bad non-global local minimum points} and \emph{convergence to good critical points}; see \cref{intro_item_I,intro_item_II} above -- is not only the reason for the convergence in the simplified shallow ANN training situations in \cref{theorem_intro_2} and \cref{theorem_intro} but is also the reason for the success of GD optimization methods in the usual non-simplified training situations of ANNs. In particular, in \cref{sec:numerical_simulations} we present a series of numerical simulation results for the \emph{standard stochastic GD optimization method} as well as for the popular \emph{Adam optimizer} (see \cite{DBLP:journals/corr/KingmaB14}) for several ANN architectures (including shallow ANNs with just one hidden layer and deep ANNs with two or three hidden layers) and each of these numerical simulations suggests that with high probability the considered GD optimization method \emph{overcomes all bad non-global local minimum points}, \emph{does not converge to a global minimum point}, but \emph{does converge to a good non-optimal critical point} whose risk value is very close to the optimal risk value of the global minimum points.


\textbf{Literature overview} -- Let us also add some comments on the literature on SGD optimization methods as well as on the training of ANNs. In the situation of a convex objective function, convergence of GD and SGD type optimization methods to the global minimum point with explicit convergence rates can be proved; see, e.g.,~\cite{BachMoulines2013,JentzenKuckuckNeufeldVonWurstemberger2021,BachMoulines2011,Nesterov2015,Nesterov2004}. For non-convex objective functions, generally one cannot even ensure convergence of the iterates of GD methods; see Absil et al.~\cite{AbsilMahonyAndrews2005} for a counterexample. Nevertheless, in the case where the objective function satisfies a {\L}ojasiewicz type inequality the findings, e.g., in \cite{AbsilMahonyAndrews2005,AttouchBolte2009,BolteDaniilidis2006,DereichKassing2021} show convergence to a critical point for GF and GD type processes under the assumption that the process does not diverge to infinity. In the context of ANNs, such convergence results have also been established in \cite{Davis2018stochastic,EberleJentzenRiekertWeiss2021,JentzenRiekert2021ExistenceGlobMin} and in this work we use similar arguments based on a {\L}ojasiewicz inequality for the risk function as in Eberle et al.~\cite{EberleJentzenRiekertWeiss2021}. However, the critical limit points in the above mentioned references might well be bad saddle points or bad local minimum points whose risk values are far away from the optimal risk values of the global minimum points. Thus, in order to establish convergence to a \emph{good} (non-optimal) critical point one also needs to employ more specific arguments related to the structure of the risk landscape in the training of ANNs as performed in this work in the simplified ANN training situations in \cref{theorem_intro_2} and \cref{theorem_intro}.


It should also be noted that establishing non-divergence or boundedness of GF and GD processes in the training of ANNs, which is necessary to apply {\L}ojasiewicz type arguments, seems to be a difficult problem (cf.\ Eberle et al.~\cite{EberleNormalizedGF2022}). In particular, as the counterexamples in Gallon et al.~\cite{GallonJentzen2022} show, such a proof can only be possible under certain regularity assumptions on the target function. In the setting considered in this article, we can show boundedness of the considered GF trajectories since only the bias parameters are trained.


In general, the problem of \emph{ruling out convergence to saddle points} seems to be more accessible than the problem of overcoming bad local minimum points. Indeed, sufficient conditions which ensure that the convergence of GD type optimization methods to saddle points can be excluded have been revealed, e.g., in~\cite{LeePanageasRecht2019,LeeJordanRecht2016,PanageasPiliouras2017,Panageas2019firstorder}. Concretely, these results show that GD processes can (up to a Lebesgue zero set of initial values) not converge to \emph{strict} saddles, i.e., saddle points at which the Hessian matrix of the objective function admits at least one strictly negative eigenvalue. They thus require the objective function to be at least twice continuously differentiable. For a similar result regarding time-continuous GF processes we refer to Bah et al.~\cite{BahRauhutTerstiege2021}. In fact, a key step in our proofs consists in verifying the necessary regularity conditions and applying \cite[Theorem~28]{BahRauhutTerstiege2021} in our setting to exclude convergence of GF trajectories to certain bad saddle points. In the context of ANNs, the fact that GD processes avoid certain saddle points also has recently been established in a special situation in Cheridito et al.~\cite{FlorianEscapeSaddles2022}. More generally, the results in Zhang et al.~\cite{Zhang2022embedding} suggest that strict saddle points appear regularly in the risk landscape related to ANNs.


On the other hand, with high probability \emph{ruling out convergence to bad local minimum points} remains a challenging problem in essentially all practically relevant cases. A particular setup where positive results have been established is the so-called overparametrized regime. In this situation one considers the empirical risk, which is computed with respect to a finite set of input-output data pairs, and assumes that the number of neurons is much larger than the number of data pairs. Under such a condition, convergence of GD type optimization methods to a global minimum point with high probability has been demonstrated, e.g., in~\cite{AroraDuHuLiWang2019,DuZhaiPoczosSingh2018,EMaWu2020,JentzenKroeger2021,LiLiang2019,RotskoffEijnden2018,ZhangMartensGrosse2019} for the case of shallow ANNs and, e.g., in~\cite{AllenzhuLiLiang2019,AllenzhuLiSong2019,DuLeeLiWangZhai2019,SankararamanDeXuHuangGoldstein2020,ZouCaoZhouGu2019} for the case of deep ANNs.
The limiting behavior of GD processes associated to neural networks as the width increases to infinity has also been investigated, e.g., in Chizat \& Bach~\cite{ChizatBach2018,ChizatBach2020}, Jacot et al.~\cite{JacotGabrielHongler2020}, and Wojtowytsch~\cite{Wojtowytsch2020}.

Some further progress has been made in the case of training ANNs for particularly simple target functions. Specifically, in the case of a constant target function the articles \cite{Riekert2021ConvergenceConstTF,RiekertJentzenConvergenceConstTargetfn2021,DNNReLUarXiv} establish convergence of GF and GD processes to a global minimum point. In fact, although the risk function is not convex one can show that all critical points are global minimum points in this simplified situation. In the case of an affine linear one-dimensional target function a complete characterization of all local minimum points and saddle points has been established in Cheridito et al.~\cite{Cheridito2022Landscape}.


A further approach is to show that certain global minima of the risk function satisfy suitable regularity properties and conclude convergence of GD optimization methods provided that the initial value is contained in a suitable region of attraction. In order to obtain convergence with high probability, a large number of independent random initializations is then required. Such local convergence results can, e.g., be found in Chatterjee~\cite{Chatterjee2022}, Fehrman et al.~\cite{FehrmanGessJentzen2020}, and \cite{JentzenRiekert2022Piecewise,JentzenRiekert2021ExistenceGlobMin}. By contrast, it is a key strength of the results in this article that \emph{a single random initialization is sufficient to guarantee convergence}.


Finally, more detailed overviews and further references regarding gradient-based optimization methods can be found, e.g., in Bercu \& Fort~\cite{BercuFort2013}, Bottou et al.~\cite{Bottou2018optimization}, E et al.~\cite{EMaWojtowytschWu2020}, Fehrman et al.~\cite[Section~1.1]{FehrmanGessJentzen2020}, \cite[Section~1]{JentzenKuckuckNeufeldVonWurstemberger2021}, Ruder~\cite{Ruder2017overview}, and the references mentioned therein.


The remainder of this work is organized as follows. In \cref{sec:anns_with_clipping} we study the training of ANNs with the clipping activation function and present the detailed proof of \cref{theorem_intro_2} above and \cref{theo:main:conv} below, respectively. In \cref{sec:anns_with_relu} we study the training of ANNs with the ReLU activation function and provide the detailed proof of \cref{theorem_intro} above and \cref{cor:main} below, respectively. Finally, in \cref{sec:numerical_simulations} we present several numerical simulation results which complement the analytical findings of this work.




\section{Artificial neural networks (ANNs) with clipping activation}
\label{sec:anns_with_clipping}


In this section we study ANNs with the clipping activation function.
The main result of this section is \cref{theo:main:conv} below, which in particular implies \cref{theorem_intro_2} from the introduction. 
Broadly speaking, the proof of \cref{theo:main:conv} consists of three parts:

\begin{enumerate} [label = (\Roman*)]
	\item We show in \cref{cor:limit_point} that the considered GF process converges to a critical point and that the risk values converge with rate $\mathcal{O} ( t^{-1} )$. We also analyze properties of such critical points of the risk function.
	\item Secondly, we establish in \cref{theo:conv:bias:deterministic} an upper bound for the risk value at the limit point, provided that the weight parameters satisfy suitable estimates.
	\item Thirdly, we prove in \cref{lem:random:init} that the required conditions on the weights in \cref{theo:conv:bias:deterministic} are satisfied with high probability for our random initialization.
\end{enumerate}



\subsection{The risk function and its derivatives}

We first introduce the ANNs with clipping activation we consider throughout this section. Afterwards we establish some basic regularity properties of the risk function.


\subsubsection{Mathematical setup for bias training of clipping ANNs}



In \cref{setting:ann:clip} we describe the mathematical framework 
which we frequently employ throughout this section. 


\begin{setting} 
\label{setting:ann:clip}
Let $ \scra \in \mathbb{R} $, $ \scrb \in (\scra , \infty) $, $ \mathfrak{h} \in \mathbb{N} $, $v_1, v_2, \dots, v_{ \mathfrak{h} }, w_1, w_2, \dots, w_\mathfrak{h} \in (0, \infty)$,
$f, \mathfrak{p}, \mathfrak{c} \in C(\mathbb{R}, \allowbreak \mathbb{R})$ satisfy for all $x \in \mathbb{R}$ that $\mathfrak{p}(x) \ge 0$, $ \mathfrak{p}^{-1}( (0, \infty) ) = (\scra, \scrb) $, and 
$\mathfrak{c}( x ) = \min\allowbreak\cu{\max \allowbreak \cu{x, \allowbreak 0}, \allowbreak 1}$, let $\operatorname{Lip} \colon C(\mathbb{R}, \mathbb{R}) \to [0, \infty]$ satisfy for all $F \in C(\mathbb{R}, \mathbb{R})$ that
\begin{equation}
\textstyle 
\operatorname{Lip}(F) = \sup_{x, y \in [\scra, \scrb], x \neq y}\frac{\abs{F(x) - F(y)}}{\abs{x - y}},
\end{equation}
for every $i \in \{ 1, 2, \dots, \mathfrak{h} \}$ let $\psi_i \colon \mathbb{R}^{\mathfrak{h} + 1} \to \mathbb{R}$ satisfy for all $\theta = (\theta_1, \ldots, \theta_{\mathfrak{h} + 1}) \in \mathbb{R}^{\mathfrak{h} + 1}$ that $\psi_i(\theta) = \allowbreak - [ w_i ]^{ - 1 } \allowbreak \theta_i$, for every $\theta = (\theta_1, \ldots, \theta_{\mathfrak{h} + 1}) \in \mathbb{R}^{ \mathfrak{h} + 1 }$, $i \in \{1, 2, \dots, \mathfrak{h}\}$ let $I_i^{\theta} \subseteq \mathbb{R}$ satisfy $I_i^{\theta} \allowbreak = ( \psi_i(\theta), \allowbreak \psi_i(\theta) \allowbreak + [ w_i ]^{ - 1 } ) \cap ( \scra, \scrb )$, for every $\theta = (\theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1}$ let $\realization{ \theta } \colon \mathbb{R} \to \mathbb{R}$ satisfy for all $x \in \mathbb{R}$ that
\begin{equation}
\label{eq:realization_function_ANNs_clipping}
\textstyle \realization{ \theta }( x ) = \theta_{ \mathfrak{h} + 1 } + \sum_{ i = 1 }^{ \mathfrak{h} } v_i \mathfrak{c}( w_i x + \theta_i ),
\end{equation}
and let 
$\mathcal{L} \colon \mathbb{R}^{\mathfrak{h} + 1} \to \mathbb{R}$ satisfy for all $\theta \in \mathbb{R}^{\mathfrak{h} + 1}$ that
\begin{equation}
\label{eq:risk_function_ANNs_clipping}
\textstyle \mathcal{L}( \theta ) = \int_{ \scra }^{ \scrb } (\realization{ \theta }(x) - f(x))^2 \mathfrak{p}(x) \, \d x.
\end{equation}
\end{setting}


In the following we add some brief explanatory comments 
regarding some of the mathematical objects 
appearing in \cref{setting:ann:clip}. 
As in the introduction, 
we also consider 
in \cref{setting:ann:clip} 
\emph{shallow ANNs} with $ 1 $ neuron on the input layer, 
with $ \mathfrak{h} \in \mathbb{N} $ neurons on the hidden layer, 
and with $ 1 $ neuron on the output layer 
(see \cref{figure_shallow_nns_general_introduction}). 
The function 
$ \mathfrak{c} \colon \mathbb{R} \to \mathbb{R} $
in \cref{setting:ann:clip} 
is used as a short notation for the clipping function 
\begin{equation}
  \mathbb{R} \ni x \mapsto \min\{ \max\{ x, 0 \}, 1 \} \in \mathbb{R} 
\end{equation}
and serves as the activation function 
of the considered ANNs. 
In \cref{setting:ann:clip} 
the \emph{inner weight parameters} 
of the considered 
ANNs are fixed positive real numbers 
denoted by 
$ w_1, w_2, \dots, w_{ \mathfrak{h} } \in (0,\infty) $, 
the \emph{outer weight parameters} of the considered 
ANNs are fixed positive real numbers 
denoted by 
$ v_1, v_2, \dots, v_{ \mathfrak{h} } \in (0,\infty) $, 
and the $ \mathfrak{h} $ \emph{inner bias parameters} 
and the \emph{outer bias parameter} are trainable parameters 
and thus appear as arguments 
$ 
  \theta_1, \theta_2, \dots, \theta_{ \mathfrak{h} + 1 } 
  \in \mathbb{R} 
$ 
of the risk function 
$  
  \mathcal{L} = 
  ( \mathcal{L}( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) )_{ 
    ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) 
    \in \mathbb{R}^{ \mathfrak{h} + 1 } 
  } 
  \colon \mathbb{R}^{ \mathfrak{h} + 1 } \to \mathbb{R} 
$  
specified 
in \cref{eq:risk_function_ANNs_clipping}. 



For every ANN parameter vector 
$ 
  \theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } 
$ 
we have that the function 
$ \realization{ \theta } \colon \mathbb{R} \to \mathbb{R} $ 
in \cref{eq:realization_function_ANNs_clipping}
specifies the \emph{realization function} 
of the considered ANN. 
For every ANN parameter vector 
$ \theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } $
and every $ i \in \{ 1, 2, \dots, \mathfrak{h} \} $ 
we have that 
the $ i $-th summand in the sum on the right hand side 
of \cref{eq:realization_function_ANNs_clipping} 
coincides with the function 
\begin{equation}
  \mathbb{R} \ni x \mapsto 
  v_i \mathfrak{c}( w_i x + \theta_i )
  \in \mathbb{R}
\end{equation}
and specifies the contribution 
of the $ i $-th neuron on the hidden layer. 
Observe in \cref{setting:ann:clip} 
that for every ANN parameter vector $\theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } $ and every $ i \in \{ 1, 2, \dots, \mathfrak{h} \} $, $ x \in \mathbb{R} $ we have that $  0 < \mathfrak{c}( w_i x + \theta_i ) < 1$ if and only if 
\begin{equation}
\textstyle \psi_i( \theta ) = - \theta_i [w_i]^{-1} < x < [w_i]^{-1}  - \theta_i [w_i]^{-1} = \psi_i( \theta ) + [w_i]^{- 1}
  .
\end{equation}
For every ANN parameter vector 
$ \theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } $
and every $ i \in \{ 1, 2, \dots, \mathfrak{h} \} $
we thus have that the set 
$
  I^{ \theta }_i \subseteq \mathbb{R}
$ 
in \cref{setting:ann:clip} 
describes precisely 
the largest open subset of 
$ ( \scra, \scrb ) $ 
on which the function 
\begin{equation}
  ( \scra, \scrb ) \ni x \mapsto 
  v_i \mathfrak{c}( w_i x + \theta_i )
  \in \mathbb{R}
\end{equation}
is not constant 
and, in this sense, 
the intervals 
$
  I^{ \theta }_i \subseteq \mathbb{R}
$ 
for 
$
  i \in \{ 1, 2, \dots, \mathfrak{h} \}
$
and 
$ \theta \in \mathbb{R}^{ \mathfrak{h} + 1 } $ 
may be referred as 
\emph{activity intervals}. 


In the next subsection, 
\cref{ssub:activity_intervals}, 
we collect a few elementary properties 
of such activity intervals. 
Thereafter, in \cref{ssub:differentiability_risk_clipping} 
we provide explicit representations 
for the first and second order partial derivatives of the risk function 
$
  \mathcal{L} \colon \mathbb{R}^{ \mathfrak{h} + 1 } \to \mathbb{R}
$ 
from \cref{setting:ann:clip}. 



\subsubsection{Properties of activity intervals}
\label{ssub:activity_intervals}
\begin{prop}
\label{prop:active:intervals}
Assume \cref{setting:ann:clip} and let 
$ 
  \theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } 
$.  
Then
\begin{enumerate}[label = (\roman*)]
\item
\label{prop:active:intervals:item1} 
it holds 
for all $ i \in \{ 1, 2, \dots, \mathfrak{h} \} $
that 
$
  I_i^{ \theta } 
  = \cu{ x \in ( \scra, \scrb ) \colon 0 < w_i x + \theta_i < 1 } 
$,

\item
\label{prop:active:intervals:item2} 
it holds 
for all $ i \in \{ 1, 2, \dots, \mathfrak{h} \} $
that 
$
  ( 
    ( 
      I_i^{ \theta } \neq \emptyset 
    )
    \Longleftrightarrow 
    ( 
      \scra - [ w_i ] ^{ - 1 } < \psi_i( \theta ) < \scrb 
    ) 
  )
$, 
and 

\item
\label{prop:active:intervals:item3} 
it holds for all 
$ \mathcal{J} \subseteq \{ 1, 2, \ldots, \mathfrak{h} \} $, 
$ x, y \in ( \cap_{ i \in \mathcal{J} } I_i^{ \theta } ) $ 
with $ y - x \ge 0 $ that
\begin{equation}
\textstyle 
  \realization{ \theta }( y ) 
  -
  \realization{ \theta }( x ) 
  \ge 
  ( \sum_{ i \in \mathcal{J} } v_i w_i ) 
  ( y - x ) 
  .
\end{equation}
\end{enumerate}
\end{prop}



\begin{cproof}{prop:active:intervals}
\Nobs that the fact that 
for all 
$ i \in \{ 1, 2, \dots, \mathfrak{h} \} $
it holds that 
\begin{equation}
  I_i^{ \theta } 
  = 
  ( 
    - \theta_i [ w_i ]^{ - 1 }, 
    ( 1 - \theta_i ) [ w_i ]^{ - 1 } 
  ) 
  \cap ( \scra, \scrb )
\end{equation}
establishes \cref{prop:active:intervals:item1,prop:active:intervals:item2}. 
Next \nobs that the fact that 
$ \mathfrak{c} \colon \mathbb{R} \to \mathbb{R} $ 
is non-decreasing assures 
that for all 
$ \mathcal{J} \subseteq \{ 1, 2, \dots, \mathfrak{h} \} $, 
$ x, y \in ( \cap_{ i \in \mathcal{J} } I_i^{ \theta } ) $ 
with $ x \le y $ 
it holds that
\begin{equation}
\begin{split}
\textstyle \realization{\theta} ( y ) - \realization{\theta}(x) 
& \textstyle = \br[\big]{\theta_{ \mathfrak{h} + 1 } + \textstyle\sum_{j=1}^\mathfrak{h} v_j \mathfrak{c} ( w_j y + \theta_j)} - \br[\big]{\theta_{ \mathfrak{h} + 1 } +  \textstyle\sum_{j=1}^\mathfrak{h} v_j \mathfrak{c} ( w_j x + \theta_j)} \\
& \textstyle = \sum_{j = 1}^\mathfrak{h} v_j \br{\mathfrak{c} ( w_j y + \theta_j ) - \mathfrak{c} ( w_j x + \theta_j ) } \\
& \textstyle \ge \sum_{j \in \mathcal{J} } v_j \br{\mathfrak{c} ( w_j y + \theta_j ) - \mathfrak{c} ( w_j x + \theta_j)}
\\
& \textstyle = \sum_{j \in \mathcal{J}} v_j \br{(w_j y + \theta_j ) - ( w_ j x + \theta_j)} = \rbr[\big]{\sum_{j \in \mathcal{J} }  v_j w_j } ( y - x ).
\end{split}
\end{equation}
This establishes \cref{prop:active:intervals:item3}.
\end{cproof}



\subsubsection{First and second order differentiability properties of the risk function}
\label{ssub:differentiability_risk_clipping}

We next show in \cref{prop:risk:gradient} that the considered risk function is once continuously differentiable. The proof is similar to our previous works \cite{ShokhrukhLocalMinima,JentzenRiekert2022Piecewise} and uses the facts that the clipping activation $\mathfrak{c}$ is almost everywhere differentiable and that the weights are nonzero.


\begin{prop}[First order partial derivatives of the risk function] 
\label{prop:risk:gradient}
Assume \cref{setting:ann:clip}. Then
\begin{enumerate} [label = (\roman*)]
\item
\label{prop:risk:gradient:item1} 
it holds that 
$ \mathcal{L} \in C^1( \mathbb{R}^{ \mathfrak{h} + 1 }, \mathbb{R} ) $,

\item 
\label{prop:risk:gradient:item2} 
it holds for all 
$ 
  \theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } 
$, 
$ 
  i \in \{ 1, 2, \dots, \mathfrak{h} \}
$ 
that
\begin{equation} \label{prop:risk:gradient:item2:eq}
\begin{split}
\textstyle 
  ( \frac{ \partial }{ \partial \theta_i } \mathcal{L} )( \theta ) 
& 
  \textstyle 
  = 2 v_i \int_{ \scra }^{ \scrb } 
  ( \realization{ \theta }( x ) - f( x ) ) 
  \mathfrak{p}(x) 
  \indicator{ (0, 1) }( w_i x + \theta_i ) 
  \, \d x 
\\
& \textstyle 
  = 2 v_i \int_{I_i^\theta} (\realization{\theta} (x) - f(x)) \mathfrak{p}(x) \, \d x
  ,
\end{split}
\end{equation}
and
\item
\label{prop:risk:gradient:item3} 
it holds for all 
$ 
  \theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } 
$
that
\begin{equation} \label{prop:risk:gradient:item3:eq}
\textstyle 
  ( 
    \frac{\partial}{\partial \theta_{\mathfrak{h} + 1}} 
    \mathcal{L} 
  )( \theta ) 
  = 2 \int_{\scra}^{\scrb} (\realization{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x.
\end{equation}
\end{enumerate}
\end{prop}
\begin{cproof}{prop:risk:gradient}
\Nobs that for all 
$ \theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } $, 
$ i \in \{ 1, 2, \dots, \mathfrak{h} \} $, 
$ 
  x \in [\scra, \scrb] \allowbreak \backslash 
  \cu{ \psi_i (\theta), \allowbreak \psi_i (\theta) \allowbreak + [w_i]^{-1}} 
$ 
it holds that $\mathbb{R} \ni \vartheta \mapsto (\realization{(\theta_1, \ldots, \theta_{i-1}, \vartheta, \theta_{i+1}, \ldots, \theta_{\mathfrak{h} + 1})}(x) - f(x))^2 \mathfrak{p}(x) \in \mathbb{R}$ is differentiable at $\theta_i$ with
\begin{equation}
\textstyle \frac{\partial}{\partial \theta_{i}} \rbr[\big]{(\realization{\theta}(x) - f(x))^2 \mathfrak{p} (x)}
= 2 v_i (\realization{\theta}(x) - f(x)) \mathfrak{p}(x) \indicator{(0, 1)} (w_i x + \theta_i).
\end{equation}
Combining this with \cite[Corollary~2.3]{ShokhrukhLocalMinima} establishes \cref{prop:risk:gradient:item2:eq}. We obtain \cref{prop:risk:gradient:item3:eq} in the same way. Furthermore, \nobs that, e.g., \cite[Lemma 2.6]{JentzenRiekert2022Piecewise} implies for all $i \in \{1, 2, \ldots, \mathfrak{h} + 1\}$ that $\mathbb{R}^{\mathfrak{h} + 1} \allowbreak \ni \theta \mapsto \rbr{\frac{\partial}{\partial \theta_i} \mathcal{L}}(\theta) \in \mathbb{R}$ is continuous.
\end{cproof}

Next we show that the risk function is even twice continuously differentiable. This follows from the Leibniz integral rule and the assumption that $\mathfrak{p}^{-1} ( \mathbb{R} \backslash \cu{0} ) = ( \scra , \scrb )$, which means that no regularity problems can occur at the boundary.

\begin{prop}[Second order partial derivatives of the risk function]  
\label{prop:risk:hess}
Assume \cref{setting:ann:clip}. Then
\begin{enumerate} [label = (\roman*)]


\item
\label{item1:prop:risk:hess} 
it holds that 
$ \mathcal{L} \in C^2( \mathbb{R}^{ \mathfrak{h} + 1 }, \mathbb{R} ) $,

\item
\label{item2:prop:risk:hess} 
it holds for all $\theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } $, $i, j \in \{ 1, 2, \ldots, \mathfrak{h} \}$ that
\begin{equation}
\begin{split}
\textstyle (\frac{ \partial^2 }{ \partial \theta_i \partial \theta_j } \mathcal{L} )( \theta ) & \textstyle = 2 v_i v_j \int_{ \mathbb{R} } \mathfrak{p}(x) \indicator{ (0, 1) }( w_i x + \theta_i ) 
  \indicator{ (0, 1) }( w_j x + \theta_j ) 
  \, \d x 
\\
& \textstyle \quad - 2 v_i (w_i)^{- 1} [\mathbbm{1}_{\{i\}}(j)] \br[\big]{( \realization{ \theta }(x) - f(x) ) \mathfrak{p}(x)}_{x = \min\cu{\max\cu{\scra, \psi_i(\theta)}, \scrb}}^{x = \min \cu{\max \cu{\scra, \psi_i(\theta) + [w_i]^{-1}}, \scrb}} \\
& \textstyle = 2 v_i v_j \int_{ I_i^{ \theta } \cap I_j^{ \theta } } \mathfrak{p}(x) \, \d x - 2 v_i ( w_i )^{- 1} [\mathbbm{1}_{\{i\}}(j)] \br[\big]{(\realization{ \theta }( x ) - f( x ) ) \mathfrak{p}(x)}_{x = \psi_i( \theta ) }^{ x = \psi_i( \theta ) + [ w_i ]^{ - 1 } },
\end{split}
\end{equation}


\item
\label{item3:prop:risk:hess} 
it holds for all 
$ 
  \theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } 
$, 
$ i \in \{ 1, 2, \dots, \mathfrak{h} \} $ 
that
\begin{equation}
\textstyle 
  ( 
    \frac{ \partial^2 }{ \partial \theta_i \partial \theta_{ \mathfrak{h} + 1 } } \mathcal{L} 
  )( \theta ) 
  = 2 v_i 
  \int_{ \scra }^{ \scrb } \mathfrak{p}(x) \indicator{ (0, 1) }( w_i x + \theta_i ) \, \d x
  = 2 v_i 
  \int_{ I_i^{ \theta } } 
  \mathfrak{p}(x) \, \d x,
\end{equation}
and


\item
\label{item4:prop:risk:hess} 
it holds for all 
$ 
  \theta = ( \theta_1, \dots, \theta_{ \mathfrak{h} + 1 } ) \in \mathbb{R}^{ \mathfrak{h} + 1 } 
$
that
\begin{equation}
  \textstyle 
  ( \frac{ \partial^2 }{ \partial \theta_{ \mathfrak{h} + 1}^2 } \mathcal{L} )( \theta ) 
  = 2 \int_{ \scra }^{ \scrb } \mathfrak{p}(x) \, \d x .
\end{equation}
\end{enumerate}
\end{prop}
\begin{cproof}{prop:risk:hess}
\Nobs that the assumption that $ \mathfrak{p}^{ - 1 }( (0, \infty) ) = (\scra, \scrb) $, 
\cref{prop:risk:gradient}, 
\cite[Corollary~2.3]{ShokhrukhLocalMinima}, 
and the Leibniz integral rule establish \cref{item1:prop:risk:hess,item2:prop:risk:hess,item3:prop:risk:hess,item4:prop:risk:hess}.
\end{cproof}



\subsubsection{Properties of critical points of the risk function}

In this part we analyze activity intervals of critical points of the risk function. The main result is \cref{cor:non:strict:saddle} below, which establishes a pointwise estimate for the slope of the realization function at non-descending critical points (cf.\ \cref{def:strict_saddle}). This will later be used in the convergence analysis, since convergence to descending critical points can be shown to occur with probability zero.


\begin{lemma}[Overlapping activity intervals of critical points must be nested]
\label{lem:crit:intervals:intersect}
Assume \cref{setting:ann:clip}, assume $\operatorname{Lip}(f) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$, let $\theta \in \mathbb{R}^{\mathfrak{h} + 1}$ satisfy $(\nabla \mathcal{L}) \allowbreak (\theta) \allowbreak = 0$, and let $i, j \in \{1, 2, \ldots, \mathfrak{h}\}$ satisfy 
\begin{equation}
\label{eq:assumption_overlapping}
  ( I_i^{\theta} \cap I_j^{\theta} ) \neq \emptyset
  .
\end{equation}
Then 
$
    ( I_i^{ \theta } \subseteq I_j^{ \theta } ) 
    \vee 
    ( I_j^{ \theta } \subseteq I_i^{ \theta } )
$.
\end{lemma}
\begin{cproof}{lem:crit:intervals:intersect}
Throughout this proof let $q^i, r^i, q^j, r^j \in \mathbb{R}$ satisfy $I_i^{\theta} = (q^i, r^i)$ and $I_j^{\theta} = (q^j , r^j)$. 
\Nobs that for all $x, y, z, t \in \mathbb{R}$ with $(x - z)(y - t) \le 0$ and $(x, y) \cap (z, t) \neq \emptyset$ it holds that 
\begin{equation}
  [(x, y) \subseteq (z, t)] \vee [(z, t) \subseteq (x, y)]
  .
\end{equation}
Hence, for the sake of contradiction 
we assume without loss of generality that $q^i < q^j$ and $r^i < r^j$. 
\Nobs that \cref{prop:risk:gradient} and 
the assumption that $(\nabla \mathcal{L})(\theta) = 0$ imply that
\begin{equation} \label{lem:crit:intervals:intersect:eq1}
\textstyle \int_{q^i}^{r^i } ( \realization{\theta} ( x ) - f ( x ) ) \mathfrak{p} ( x ) \, \d x = 
\int_{q^j}^{r^j } ( \realization{\theta} ( x ) - f ( x ) ) \mathfrak{p} ( x ) \, \d x = 0 .
\end{equation}
\Nobs that \cref{eq:assumption_overlapping}
and the assumption that 
$
  (q^i < q^j) \wedge (r^i < r^j)
$ 
imply that $q^j < r^i$. 
In addition, \cref{lem:crit:intervals:intersect:eq1} 
and the mean value theorem ensure that 
there exists $u \in (q^j , r^j)$ which satisfies 
\begin{equation}
  \realization{\theta}(u) = f(u)
  .
\end{equation}
This, the assumption that $\operatorname{Lip} ( f ) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$, and \cref{prop:active:intervals} show for all $x \in (q^i, u)$ that $\realization{\theta}(x) < f(x)$. Combining this with \cref{lem:crit:intervals:intersect:eq1} ensures that $u < r^i$. Hence, we obtain for all $x \in (r^i , r^j )$ that $\realization{\theta}(x) > f(x)$. \cref{lem:crit:intervals:intersect:eq1} therefore demonstrates that
\begin{equation}
\begin{split}
0 & \textstyle < \int_{ r^i}^{r^j} (\realization{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x = - \int_{q^j}^{r^i} (\realization{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x \\
& \textstyle = \int_{q^i}^{q^j} (\realization{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x < 0,
\end{split}
\end{equation}
which is a contradiction.
\end{cproof}


\cfclear
\begin{definition}[Descending critical point]\label{def:strict_saddle}
Let $ \mathfrak{h} \in \mathbb{N} $, $ f \in C^2( \mathbb{R}^{ \mathfrak{h} }, \mathbb{R} ) $, $ x \in \mathbb{R}^{\mathfrak{h}} $. Then we say that $ x $ is a \cfadd{def:strict_saddle}\text{descending } critical point of $ f $ if and only if it holds that 
\begin{enumerate}[label = (\roman*)]
\item we have that 
$
  ( \nabla f )( x ) = 0
$
and 
\item 
we have that there exists $ v \in \mathbb{R}^{ \mathfrak{h} } $ such that
\begin{equation}
\textstyle 
  \scalar{v, ((\operatorname{Hess} f)(x))v} < 0.
\end{equation}
\end{enumerate}
\end{definition}


In the situation of \cref{def:strict_saddle} 
\nobs that 
the assumption that $ f \in C^2( \mathbb{R}^\mathfrak{h} , \mathbb{R}) $ 
ensures that 
the matrix $ ( \operatorname{Hess} f)( x ) $ is symmetric and therefore diagonalizable with real eigenvalues. 
In the context of \cref{def:strict_saddle} we hence obtain 
that $ x $ is a \cfadd{def:strict_saddle}\text{descending } critical point of $ f $ if and only if 
$ x $ is a critical point with the property that 
$ ( \operatorname{Hess} f)( x ) $ 
admits at least one negative eigenvalue.

In the scientific literature descending critical points are often called strict saddle points (cf., e.g., \cite[Definition 27]{BahRauhutTerstiege2021} and \cite[Definition 1]{LeePanageasRecht2019}).
However, note that a descending critical point in the sense of \cref{def:strict_saddle} is not necessarily a saddle point, but might as well be a local maximum.



\cfclear
\begin{lemma}[Upper bounds for the slopes of realization 
functions of suitable non-descending critical points 
under local regularity assumptions on the density]
\label{lem:crit:dens:bounded:below}
Assume \cref{setting:ann:clip}, 
assume $ \operatorname{Lip}( f ) < \min_{ i \in \{ 1, 2, \dots, \mathfrak{h} \} } v_i w_i $, 
let $ \theta \in \mathbb{R}^{ \mathfrak{h} + 1 } $ 
satisfy 
$
  ( \nabla \mathcal{L} ) \allowbreak ( \theta ) \allowbreak = 0 
$, 
assume that $ \theta $ is not a \cfadd{def:strict_saddle}\text{descending } critical point of $ \mathcal{L} $ 
\cfadd{def:strict_saddle}, 
let 
$
  \mathscr{x} \in ( \scra, \scrb ) 
$, 
and assume that there exists 
$ 
  i \in \{ 1, 2, \dots, \mathfrak{h} \} 
$ 
such that 
\begin{equation} 
\textstyle 
  \mathscr{x} \in I_i^{\theta} 
\qquad \text{and} \qquad
  \inf_{ y \in I_i^{ \theta } } 
  \mathfrak{p}(y) 
  \ge (\min_{j \in \{1, 2, \ldots, \mathfrak{h}\}} w_j)^{-1} \operatorname{Lip}(\mathfrak{p})
\end{equation}
\cfload. 
Then
\begin{equation}
\textstyle 
\sum_{j \in \{ 1, 2, \dots, \mathfrak{h} \}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j \le 3 \bigl[\max_{j \in \{ 1, 2, \dots, \mathfrak{h} \}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j \bigr].
\end{equation}
\end{lemma}
\begin{cproof}{lem:crit:dens:bounded:below}
Throughout this proof assume that
\begin{equation}
\label{lem:crit:dens:bounded:below:eq1}
\textstyle 
\sum_{ j \in \{ 1, 2, \dots, \mathfrak{h} \}, \, \mathscr{x} \in I_j^{ \theta }} v_j w_j > 3 \bigl[\max_{j \in \{ 1, 2, \dots, \mathfrak{h} \}, \, \mathscr{x} \in I_j^{ \theta } } v_j w_j\bigr]. 
\end{equation}
\Nobs that \cref{lem:crit:intervals:intersect} ensures 
that there exists $ i \in \{ 1, 2, \dots, \mathfrak{h} \} $ 
which satisfies $\mathscr{x} \in I_i^\theta$, $\inf_{y \in I_i^\theta} \mathfrak{p}(y) \ge [w_i]^{-1} \operatorname{Lip}(\mathfrak{p}) > 0$, and which satisfies for all $j \in \{1, 2, \ldots, \mathfrak{h}\}$ with $\mathscr{x} \in I_j^{\theta}$ that $I_i^{\theta} \subseteq I_j^{\theta}$. Let $q, r \in \mathbb{R}$ satisfy $I_i^{\theta} = (q, r)$. \Nobs that \cref{prop:risk:hess} implies that
\begin{equation}\label{eqn:lem:crit:dens:bounded:below:parderi2_ii}
\textstyle \rbr[\big]{\frac{\partial^2}{\partial \theta_i^2} \mathcal{L}}(\theta) = 2 v_i^2 \int_q^r \mathfrak{p}(x) \, \d x - 2 \frac{v_i}{w_i} \br{( \realization{\theta}(x) - f(x)) \mathfrak{p}(x)}_{x=q}^{x=r}.
\end{equation}
Furthermore, the assumption that 
$ \operatorname{Lip}(f) < \min_{ j \in \{ 1, 2, \dots, \mathfrak{h} \} } v_j w_j $, 
\cref{prop:active:intervals}, and \cref{lem:crit:dens:bounded:below:eq1} show for all $x, y \in [q, r]$ with $x < y$ that $( \realization{\theta}(y) - f(y)) - (\realization{\theta}(x) - f(x)) > 2 v_i w_i (y - x) > 0 $. This and the fact that $\int_q^r ( \realization{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x  = 0$ assure that $\realization{\theta}(r) - f(r) > 0 > \realization{\theta}(q) - f(q)$. Therefore, we obtain that
\begin{equation}\label{eqn:lem:crit:dens:bounded:below:int_bound}
\begin{split}
\textstyle \br*{(\realization{\theta}(x) - f(x)) \mathfrak{p}(x)}_{x=q}^{x=r} & \textstyle \ge \rbr[\big]{(\realization{\theta}(r) - f(r)) - (\realization{\theta}(q) - f(q))} \inf\nolimits_{x \in [q,r] } \mathfrak{p}(x) \\
& > 2 v_i w_i (r - q) \inf\nolimits_{x \in [q,r]} \mathfrak{p}(x).
\end{split}
\end{equation}
Moreover, the fact that $0 < r - q \le (-[w_i]^{-1} \theta_i + [w_i]^{-1}) - (- [w_i]^{-1} \theta_i) = [w_i]^{-1}$ and the fact that $\inf_{x \in I_i^\theta} \mathfrak{p}(x) \ge [w_i]^{-1} \operatorname{Lip}(\mathfrak{p})$
prove that
\begin{equation} 
\begin{split}
\textstyle \sup_{x \in [q,r] } \mathfrak{p} ( x ) 
& \textstyle \le  \br[\big]{\inf_{x \in [q, r]} \mathfrak{p}(x)} + (\operatorname{Lip}(\mathfrak{p}))(r - q) \le \br[\big]{\inf_{x \in [q, r]} \mathfrak{p}(x)} + (\operatorname{Lip}(\mathfrak{p})) [w_i]^{-1} \\
& \textstyle \le 2 \inf_{x \in [q, r]} \mathfrak{p}(x).
\end{split}
\end{equation}
Combining this with \cref{eqn:lem:crit:dens:bounded:below:parderi2_ii} and \cref{eqn:lem:crit:dens:bounded:below:int_bound} ensures that
\begin{equation}
\rbr[\big]{ \tfrac{ \partial ^2 }{ \partial \theta_i ^2 } \mathcal{L} } ( \theta )
< \tfrac{2v_i}{w_i} \rbr*{ v_i w_i ( r - q ) \br[\big]{ \sup\nolimits_{x \in [q,r]} \mathfrak{p} ( x ) } - 2 v_i w_i ( r - q ) \br[\big]{ \inf\nolimits_{x \in [q,r]} \mathfrak{p} ( x ) } } \le 0.
\end{equation}
This shows that $\theta$ is a \cfadd{def:strict_saddle}\text{descending } critical point of $\mathcal{L}$, which is a contradiction.
\end{cproof}


\begin{lemma} 
\label{lem:zero:mean:help}
Let $ a \in \mathbb{R} $, $ b \in (a, \infty) $, 
let $ \mathfrak{p} \in C ( [a, b] , [0, \infty) ) $ be strictly increasing, 
let $ g \in C( [a, b], \mathbb{R} ) $, $ L \in (0, \infty) $ satisfy 
for all $ x, y \in [a,b] $ with $ x < y $ that 
\begin{equation}\label{eqn:lem:zero:mean:help}
\textstyle 
  g(y) - g(x) > 3 L (y - x),
\end{equation}
and assume $\int_a^b g(x) \mathfrak{p}(x) \, \d x = 0$.
Then 
\begin{equation}
\label{lem:zero:mean:help:eqclaim}
\textstyle L \int_a^b \mathfrak{p}(x) \, \d x < g(b) \mathfrak{p}(b).
\end{equation}
\end{lemma}
\begin{cproof}{lem:zero:mean:help}
\Nobs that \cref{eqn:lem:zero:mean:help} ensures that $g$ is strictly increasing. Combining this with the assumption that $\int_a^b g ( x ) \mathfrak{p} ( x ) \, \d x = 0$ shows that there exists $\xi \in (a,b)$ which satisfies $g(a) < 0 = g (\xi) < g(b)$. \Nobs that \cref{eqn:lem:zero:mean:help} shows that $g(b) > 3L(b - \xi)$.

In the following we distinguish between the case $g(b) \ge - g(a)$ and the case $g(b) < - g(a)$.
 First we prove \cref{lem:zero:mean:help:eqclaim} in the case 
 \begin{equation}
 \label{lem:zero:mean:help:case1}
 g(b) \ge - g(a).
 \end{equation}
  \Nobs that \cref{eqn:lem:zero:mean:help} and the fact that $g(a) < 0 = g (\xi) < g(b)$ assure that $g(a) < 3 L (a - \xi)$. Combining this with \cref{lem:zero:mean:help:case1} shows that $g(b) > 3 L (\xi - a)$. This, the fact that $g(b) > 3L(b - \xi)$, and the assumption that $\mathfrak{p}$ is strictly increasing demonstrate that
\begin{equation}
\begin{split}
\textstyle g(b) \mathfrak{p}(b) & \textstyle = 2 \frac{g(b) \mathfrak{p}(b)}{2} > \frac{3L}{2} (\xi - a) \mathfrak{p}(b) + \frac{3L}{2} (b - \xi) \mathfrak{p}(b) \\
& \textstyle \ge  L (b - a) \mathfrak{p}(b) > L \int_a^b \mathfrak{p}(x) \, \d x.
\end{split}
\end{equation}
This establishes \cref{lem:zero:mean:help:eqclaim} in the case $g(b) \ge - g(a)$.
 Next we prove \cref{lem:zero:mean:help:eqclaim} in the case 
 \begin{equation}
 \label{lem:zero:mean:help:case2}
 g(b) < - g(a).
 \end{equation} 
 \Nobs that \cref{lem:zero:mean:help:case2} and the intermediate value theorem assure that there exists $\zeta \in (a, \xi)$ which satisfies $g(\zeta) = - g(b)$. Moreover, \cref{eqn:lem:zero:mean:help} shows that $g(b) = g(\xi) - g(\zeta) > 3 L(\xi - \zeta)$. The fact that for all $x \in (a, \zeta )$,
$y \in (\zeta , \xi )$ it holds that $g(x) < g(\zeta ) = - g(b) = g(\zeta) < g(y) < 0$, the fact that $g$ is strictly increasing, and the assumption that $\int_a^b g(x) \mathfrak{p}(x)\, \d x = 0$ therefore demonstrate that
\begin{equation}
\textstyle g(b) \int_a^\zeta \mathfrak{p}(x) \, \d x \textstyle \le \int_a^\zeta(- g(x)) \mathfrak{p}(x) \, \d x = \int_\zeta^b g(x) \mathfrak{p}(x) \, \d x \le \int_\xi^b g(x) \mathfrak{p}(x) \, \d x \le g(b) \int_\xi^b \mathfrak{p}(x) \, \d x.
\end{equation}
Hence, we obtain that $\int_a^\zeta \mathfrak{p}(x) \, \d x \le \int_\xi ^b \mathfrak{p} ( x ) \, \d x$. This and the fact that $g ( b ) >3 L \max \cu{ \xi - \zeta , b - \xi }$ assure that
\begin{equation}
\begin{split}
\textstyle g(b) \mathfrak{p}(b) & \textstyle > 2 L \mathfrak{p}(b) (b - \xi) + L \mathfrak{p}(b) (\xi - \zeta) \ge 2 L \int_\xi^b \mathfrak{p}(x) \, \d x + L \int_\zeta^\xi \mathfrak{p}(x) \, \d x \\
& \textstyle \ge L \int_\xi^b \mathfrak{p}(x) \, \d x + L \int_a^\zeta \mathfrak{p}(x) \, \d x + L \int_\zeta^\xi \mathfrak{p}(x) \, \d x = L \int_a^b \mathfrak{p}(x) \, \d x .
\end{split}
\end{equation}
This establishes \cref{lem:zero:mean:help:eqclaim} in the case $g(b) < - g(a)$.
\end{cproof}



\begin{cor}
\label{cor:zero:mean:help}
Let $a \in \mathbb{R}$, $b \in (a, \infty )$, let $\mathfrak{p} \in C( [a,b] , [0, \infty ) )$ be strictly monotonous, 
let $g \in C([a, b] , \mathbb{R} )$, $L \in (0, \infty)$ satisfy for all 
$ x, y \in [a,b] $
with $ x < y $ that
\begin{equation}
  g(y) - \allowbreak g(x) \allowbreak > 3 L (y - x)
  ,
\end{equation}
and assume $\int_a^b g(x) \mathfrak{p}(x) \, \d x = 0$. Then 
\begin{equation}
\label{cor:zero:mean:help:eqclaim}
\textstyle L \int_a^b \mathfrak{p}(x) \, \d x < g(b) \mathfrak{p}(b) - g(a) \mathfrak{p}(a).
\end{equation}
\end{cor}
\begin{cproof}{cor:zero:mean:help}
Throughout this proof let 
$
  \overline{\mathfrak{p}}, \overline{g} \colon [a, b] \to \mathbb{R}
$ 
satisfy for all $x \in [a, b]$ that 
$\overline{\mathfrak{p}} ( x ) = \mathfrak{p} ( a + b - x )$ and $\overline{g} ( x ) = - g (a + b - x ) $.


In the following we distinguish between the case where $ \mathfrak{p} $ is strictly increasing 
and the case where $ \mathfrak{p} $ is strictly decreasing. 
First we prove \cref{cor:zero:mean:help:eqclaim} in the case where $ \mathfrak{p} $ is strictly increasing. 
\Nobs that \cref{lem:zero:mean:help} and the fact that $ g(a) \mathfrak{p}(a) \le 0 $ show that 
$ L \int_a^b \mathfrak{p}(x) \, \d x < g(b) \mathfrak{p}(b) - g(a) \mathfrak{p}(a) $, 
which establishes \cref{cor:zero:mean:help:eqclaim}. 
 
 Next we prove \cref{cor:zero:mean:help:eqclaim} in the case where $\mathfrak{p}$ is strictly decreasing. \Nobs that \cref{lem:zero:mean:help} (applied with $\mathfrak{p} \with \overline{\mathfrak{p}}$, $g \with \overline{g}$ in the notation of \cref{lem:zero:mean:help}) and the integral transformation theorem demonstrate that
\begin{equation}
\textstyle L \int_a^b \mathfrak{p}(x) \, \d x = L \int_a^b \overline{\mathfrak{p}} (x) \, \d x < \overline{g}(b) \overline{\mathfrak{p}}(b) = - g(a) \mathfrak{p}(a) \le g(b) \mathfrak{p}(b) - g(a) \mathfrak{p}(a).
\end{equation}
\end{cproof}



\cfclear
\begin{lemma}[Upper bounds for the slopes of realization 
functions of suitable non-descending critical points 
under local regularity assumptions on the density]
\label{lem:crit:dens:mono} 
Assume \cref{setting:ann:clip}, assume $\operatorname{Lip}(f) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$, let $\theta \in \mathbb{R}^{\mathfrak{h} + 1}$ satisfy $(\nabla \mathcal{L}) \allowbreak(\theta) \allowbreak = 0$, let $ \mathscr{x} \in (\scra, \scrb) $, assume that there exists 
$ i \in \{ j \in \{ 1, 2, \dots, \mathfrak{h} \} \colon \mathscr{x} \in I_j^\theta \} $ 
such that $\mathfrak{p}|_{I_i^\theta }$ is strictly monotonous, and assume that $\theta$ is not a \cfadd{def:strict_saddle}\text{descending } critical point of $\mathcal{L}$ \cfadd{def:strict_saddle}\cfload. Then
\begin{equation}
\textstyle \sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j \le 4 \max_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j.
\end{equation}
\end{lemma}
\begin{cproof}{lem:crit:dens:mono}
Throughout this proof assume
\begin{equation}
\label{lem:crit:dens:mono:eq1}
\textstyle \sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j > 4 \max_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j.
\end{equation}
\Nobs that \cref{lem:crit:intervals:intersect} ensures that there exists $i \in \{1, 2, \ldots, \mathfrak{h}\}$ which satisfies for all $j \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathfrak{h}\}$ with $\mathscr{x} \in I_j^{\theta}$ that $\mathscr{x} \in I_i^\theta \subseteq I_j^\theta$ and $\mathfrak{p} |_{I_i^\theta }$ 
is strictly monotonous. Let $q, r \in \mathbb{R}$ satisfy $I_i^\theta = (q, r)$. \Nobs that \cref{prop:risk:hess} implies that
\begin{equation}
\label{lem:crit:dens:mono:eq2}
\textstyle \rbr[\big]{ \tfrac{ \partial ^2 }{ \partial \theta_i ^2 } \mathcal{L} } ( \theta ) = 2 v_i ^2 \int_q^r \mathfrak{p} ( x ) \, \d x - 2 \tfrac{v_i}{w_i} \br*{ ( \realization{\theta} ( x ) - f ( x ) ) \mathfrak{p} ( x ) } _{x=q}^{x=r}.
\end{equation}
Furthermore, \cref{lem:crit:dens:mono:eq1} ensures for all $x \in [q,r]$, $y \in (x, r]$ that $( \realization{\theta} ( y ) - f ( y ) ) - ( \realization{\theta} ( x ) - f ( x ) ) > 3 v_i w_i ( y - x ) $. This, the fact that $\int_q^r ( \realization{\theta} ( x ) - f ( x ) ) \mathfrak{p} ( x ) \, \d x = 0$, and \cref{cor:zero:mean:help} assure that
\begin{equation}
\textstyle v_i w_i \int_q^r \mathfrak{p} ( x ) \, \d x < \br*{ ( \realization{\theta} ( x ) - f ( x ) ) \mathfrak{p} ( x ) } _{x=q}^{x=r}.
\end{equation}
Combining this with \cref{lem:crit:dens:mono:eq2} demonstrates that 
$\rbr[\big]{ \tfrac{ \partial ^2 }{ \partial \theta_i ^2 } \mathcal{L} } ( \theta ) < 0$.
This shows that $\theta$ is a \cfadd{def:strict_saddle}\text{descending } \allowbreak critical point of $\mathcal{L}$, which is a contradiction.
\end{cproof}


\cfclear
\begin{cor}[Upper bounds for the slopes of realization 
functions of suitable non-descending critical points 
under global regularity assumptions on the density]
\label{cor:non:strict:saddle}
Assume \cref{setting:ann:clip}, assume $\operatorname{Lip} ( f ) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$, let $\theta \in \mathbb{R}^{\mathfrak{h} + 1 }$ satisfy $( \nabla \mathcal{L} ) ( \theta ) \allowbreak = 0$,
 let $ \varepsilon, \allowbreak \delta \in \allowbreak (0, \infty)$ satisfy 
\begin{equation}
\textstyle 
  \varepsilon - \delta > 2 \allowbreak [\min_{j \in \{1, 2, \ldots, \mathfrak{h}\}} w_j]^{-1} 
\qquad \text{and} \qquad 
  \inf_{x \in [ \scra + \delta , \scrb - \delta ] } \mathfrak{p} ( x ) \ge [\min_{j \in \{1, 2, \ldots, \mathfrak{h}\}} w_j]^{-1} \allowbreak \operatorname{Lip}( \mathfrak{p} )
  ,
\end{equation}
assume that $\mathfrak{p}_{[\scra , \scra + \varepsilon ] }$ is strictly increasing, assume that $\mathfrak{p}_{[\scrb -  \varepsilon  , \scrb ] }$ is strictly decreasing,
and assume that $\theta$ is not a \cfadd{def:strict_saddle}\text{descending } critical point of $\mathcal{L}$ \cfadd{def:strict_saddle}\cfload. Then it holds for all $x \in ( \scra , \scrb )$ that
\begin{equation}\label{eqn:cor:non:strict:saddle}
\textstyle 
\sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_j^{\theta}} v_j w_j \le 4 \max_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_j^{\theta}} v_j w_j.
\end{equation}
\end{cor}
\begin{cproof}{cor:non:strict:saddle}
Throughout this proof let $\mathscr{x} \in (\scra , \scrb )$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$ satisfy $\mathscr{x} \in I_i^\theta$ and let $\mathscr{w} \in \mathbb{R}$ satisfy $\mathscr{w} = [\min_{j \in \{1, 2, \ldots, \mathfrak{h}\}} w_j]^{-1}$. In the following we distinguish between the case $\mathscr{x} \in [ \scra + \delta + \mathscr{w}, \scrb - \delta - \mathscr{w}]$ and the case $\mathscr{x} \in (\scra ,  \scra + \delta + \mathscr{w}) \cup (\scrb - \delta - \mathscr{w}, \scrb )$. First we prove \cref{eqn:cor:non:strict:saddle} in the case $\mathscr{x} \in [\scra + \delta + \mathscr{w}, \scrb - \delta - \mathscr{w}]$. In this case, we have that $I_i^\theta \subseteq [\scra + \delta , \scrb - \delta]$. This and the assumption that $\inf_{x \in [ \scra + \delta , \scrb - \delta ] } \mathfrak{p}(x) \ge \mathscr{w} \operatorname{Lip}(\mathfrak{p})$ ensure that $\inf_{x \in I_i^\theta } \mathfrak{p}(x) \ge \mathscr{w} \operatorname{Lip}(\mathfrak{p})$. Combining this with \cref{lem:crit:dens:bounded:below} shows that 
\begin{equation}
\textstyle \sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j \le 3 \max_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j \le 4 \max_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j.
\end{equation}
This establishes \cref{eqn:cor:non:strict:saddle} in the case $\mathscr{x} \in [\scra + \delta + \mathscr{w}, \scrb - \delta - \mathscr{w}]$. In the next step we prove \cref{eqn:cor:non:strict:saddle} in the case $\mathscr{x} \in (\scra ,  \scra + \delta + \mathscr{w}) \cup (\scrb - \delta - \mathscr{w}, \scrb )$. \Nobs that the assumption that $\varepsilon - \delta > 2 \mathscr{w}$ assures that $I_i^\theta \allowbreak \subseteq \allowbreak (\scra, \allowbreak \scra + \varepsilon) \cup (\scrb - \varepsilon, \scrb)$. \cref{lem:crit:dens:mono} therefore demonstrates that $\sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j \le 4 \max_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{x} \in I_j^{\theta}} v_j w_j$. This establishes \cref{eqn:cor:non:strict:saddle} in the case $\mathscr{x} \in (\scra ,  \scra + \delta + \mathscr{w}) \cup (\scrb - \delta - \mathscr{w}, \scrb )$.
\end{cproof}




\subsubsection{Pointwise estimates for realization functions at critical points}

We next derive in \cref{lem:realization:pointwise:bound} an estimate for the pointwise distance between the realization function $\mathcal{N}^\vartheta$ and the target function $f$ at a critical point $\vartheta$. If the outer weights $v_i$ are chosen to be small this will result in a useful estimate for the risk value $\mathcal{L} ( \vartheta )$.
Due to the fact that $\mathcal{N} ^\vartheta$ is constant outside the activity intervals $I_i^\vartheta$ it suffices to prove this inside these intervals, the bound at the remaining points then follows from the monotonicity of $f$.

\begin{lemma}
\label{lem:realization:pointwise:bound}
Assume \cref{setting:ann:clip}, assume that $f$ is non-decreasing and satisfies $\operatorname{Lip} ( f ) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$, let $\vartheta \in \mathbb{R}^{\mathfrak{h} + 1 }$ satisfy $(\nabla \mathcal{L})(\vartheta) = 0$, and let $V \in \mathbb{R}$ satisfy 
\begin{equation}
\textstyle V = \sup_{x \in (\scra, \scrb)} \sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_j^{\theta}} v_j.
\end{equation}
Then
\begin{enumerate}[label = (\roman*)]
\item
\label{lem:realization:pointwise:bound:item1} it holds for all $j \in \{1, 2, \ldots, \mathfrak{h}\}$, $x \in I_j^\vartheta$ that $\abs{\realization{\vartheta} ( x ) - f ( x ) } \le V$, 

\item
\label{lem:realization:pointwise:bound:item2} it holds for all $j , k \in \{1, 2, \ldots, \mathfrak{h}\}$ with $I_j^\vartheta, I_k^\vartheta \not= \emptyset$ and all $x \in [ \sup I_j^\vartheta, \inf I_k^\vartheta ]$
that $\abs{ \realization{\vartheta} ( x ) - f ( x ) } \le V$,

\item
\label{lem:realization:pointwise:bound:item3} it holds for all $j \in \{1, 2, \ldots, \mathfrak{h}\}$ with $I_j^\vartheta \not= \emptyset$ and all
$x \in [ \sup I_j^\vartheta , \scrb ]$ that $\abs{\realization{\vartheta} ( x ) - f ( x ) } \le \max \cu{ V , \abs{ f ( \scrb ) - \realization{\vartheta} ( \scrb ) } }$, and

\item
\label{lem:realization:pointwise:bound:item4} it holds for all $j \in \{1, 2, \ldots, \mathfrak{h}\}$ with $I_j^\vartheta \not= \emptyset$ and all
$x \in [ \scra ,\inf I_j^\vartheta ]$ that $\abs{\realization{\vartheta} ( x ) - f ( x ) } \le \max \cu{ V , \abs{ f ( \scra ) - \realization{\vartheta} ( \scra ) } }$.
\end{enumerate}
\end{lemma}
\begin{cproof}{lem:realization:pointwise:bound}
Throughout this proof let $k, l, m \in \{1, 2, \ldots, \mathfrak{h}\}$, $q, s \in [\scra, \scrb]$, $r \in [q, \scrb]$, $\mathscr{x} \allowbreak \in \allowbreak (\scra, \scrb)$ satisfy $I_k^\vartheta \neq \emptyset$, $I_l^\vartheta \neq \emptyset$, $I_m^\vartheta \not= \emptyset$, $q = \sup I_k^\vartheta$, $s = \sup I_m^\vartheta$, $r = \inf I_l^\vartheta$, and $\cu{ j \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathfrak{h}\} \allowbreak \colon \allowbreak \mathscr{x} \allowbreak \in \allowbreak I_j^\vartheta} \allowbreak \allowbreak \neq \emptyset$. \Nobs that \cref{lem:crit:intervals:intersect} assures that there exists $\mathscr{i} \in \{1, 2, \ldots, \mathfrak{h}\}$ which satisfies for all $j \in \allowbreak \{1, \allowbreak 2, \allowbreak \ldots, \mathfrak{h}\}$ with $\mathscr{x} \in I_j^\vartheta$ that $\mathscr{x} \in I_{\mathscr{i}}^\vartheta \subseteq I_j^\vartheta$. Moreover, the integral mean value theorem and the fact that 
\begin{equation}
\textstyle \int_{I_{\mathscr{i}}^\vartheta} ( \realization{\vartheta} ( x ) - f ( x ) ) \allowbreak \mathfrak{p} ( x ) \, \d x \allowbreak = 0
\end{equation}
demonstrate that there exists $\mathscr{y} \in I_{\mathscr{i}}^\vartheta$ which satisfies $\realization{\vartheta} ( \mathscr{y} ) = f ( \mathscr{y} )$. In the following we assume without loss of generality that $\mathscr{y} \le \mathscr{x}$ and let $\mathscr{u} \in \mathbb{R}$ satisfy $\mathscr{u} = \sup \cu{ z \in I_{\mathscr{i}}^\vartheta \cap ( \scra , \mathscr{x} ] \colon \realization{\vartheta} ( z ) = f(z)}$. \Nobs that
\begin{equation} \label{lem:realization:pointwise:bound:eq1}
\begin{split}
& \textstyle \realization{\vartheta} ( \mathscr{x} ) \le \realization{\vartheta} ( \mathscr{u} ) + \sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, I_j^\vartheta \cap (\mathscr{u} , \mathscr{x} ) \neq \emptyset} v_j \\
& \textstyle = f ( \mathscr{u} ) + \sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, I_j^\vartheta \cap (\mathscr{u} , \mathscr{x} ) \neq \emptyset} v_j \le f ( \mathscr{x} ) + \sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, I_j^\vartheta \cap (\mathscr{u} , \mathscr{x} ) \neq \emptyset} v_j.
\end{split}
\end{equation}
In addition, the fact that $\operatorname{Lip} ( f ) < \min_{j \in \{1, 2, \ldots, \mathfrak{h}\}} v_j w_j $, the fact that $\mathscr{x}, \mathscr{u} \in I_{\mathscr{i}}^\vartheta$, and \cref{prop:active:intervals} show that $\realization{\vartheta} ( \mathscr{x} ) \ge f ( \mathscr{x} )$. Next let $\mathscr{j} \in \{1, 2, \ldots, \mathfrak{h}\}$ satisfy $I_{\mathscr{j}}^\vartheta \cap ( \mathscr{u} , \mathscr{x} ) \neq \emptyset$. We now prove that $\mathscr{u} \in I_{\mathscr{j}}^\vartheta$. \Nobs that the integral mean value theorem and the fact that
\begin{equation}
\textstyle \int_{I_{\mathscr{j}}^\vartheta} ( \realization{\vartheta} ( x ) - f ( x ) ) \allowbreak \mathfrak{p} ( x ) \, \d x \allowbreak = 0
\end{equation}
show that there exists $\mathscr{z} \in I_{\mathscr{j}}^\vartheta$ which satisfies $\realization{\vartheta} ( \mathscr{z} ) = f ( \mathscr{z} )$.

In the following we distinguish between the case $\mathscr{z} \le \mathscr{u}$ and the case $\mathscr{z} > \mathscr{u}$. First we prove $\mathscr{u} \in I_{\mathscr{j}}^\vartheta$ in the case $\mathscr{z} \le \mathscr{u}$. \Nobs that the assumption that $I_{\mathscr{j}}^\vartheta \cap ( \mathscr{u} , \mathscr{x} ) \neq \emptyset$ and the fact that $I_{\mathscr{j}}^\vartheta$ is connected imply that $\mathscr{u} \in I_{\mathscr{j}}^\vartheta$. This proves $\mathscr{u} \in I_{\mathscr{j}}^\vartheta$ in the case $\mathscr{z} \le \mathscr{u}$.

 In the next step we prove $\mathscr{u} \in I_{\mathscr{j}}^\vartheta$ in the case $\mathscr{z} > \mathscr{u}$. \Nobs that the fact that $\mathscr{u} = \sup \cu{ z \in I_{\mathscr{i}}^\vartheta \cap ( \scra , \mathscr{x} ] \colon \realization{\vartheta} ( z ) = f(z)}$ implies that $\mathscr{z} > \mathscr{x}$. Hence, we obtain that $\mathscr{x} \in I_{\mathscr{j}}^\vartheta$. Combining this with the definition of $\mathscr{i}$ ensures that $I_{\mathscr{i}}^\vartheta \subseteq I_{\mathscr{j}}^\vartheta$. Therefore, we obtain that $\mathscr{u} \in I_{\mathscr{i}}^\vartheta \subseteq I_{\mathscr{j}}^\vartheta$. This proves $\mathscr{u} \in I_{\mathscr{j}}^\vartheta$ in the case $\mathscr{z} > \mathscr{u}$. Hence, we obtain that
\begin{equation}
\textstyle \sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, I_j^\vartheta \cap (\mathscr{u} , \mathscr{x} ) \neq \emptyset} v_j \le \textstyle\sum_{j \in \{1, 2, \ldots, \mathfrak{h}\}, \, \mathscr{u} \in I_j^\vartheta} v_j \le V.
\end{equation}
Combining this with \cref{lem:realization:pointwise:bound:eq1} and the fact that $\realization{\vartheta} ( \mathscr{x} ) \ge f (\mathscr{x})$ establishes \cref{lem:realization:pointwise:bound:item1}. Next we assume without loss of generality that 
\begin{equation}\label{eqn:lem:realization:pointwise:bound:k_l}
\textstyle \forall \, i \in \{1, 2, \ldots, \mathfrak{h}\} \backslash \cu{k, l} \colon I_i^\vartheta \cap [q, r] = \emptyset.
\end{equation}
\Nobs that \cref{eqn:lem:realization:pointwise:bound:k_l} ensures for all $x \in [q, r]$ that $\realization{\vartheta} (x) = \realization{\vartheta}(q) = \realization{\vartheta}(r)$ and $f(q) \le f(x) \le f(r)$. Combining this with \cref{lem:realization:pointwise:bound:item1} assures for all
$x \in [q, r]$ that
\begin{equation}
\begin{split}
\textstyle \abs{\realization{\vartheta} ( x ) - f(x) } & \textstyle \le \max \cu[\big]{\abs{\realization{\vartheta} ( x ) - f(q) } , \abs{\realization{\vartheta} ( x ) - f(r) }} \\
& \textstyle = \max \cu[\big]{\abs{\realization{\vartheta} ( q ) - f(q) } , \abs{\realization{\vartheta} ( r ) - f(r) }}  \le V.
\end{split}
\end{equation}
This establishes \cref{lem:realization:pointwise:bound:item2}. In the next step we assume without loss of generality that $\forall \, i \in \allowbreak \{1, \allowbreak 2, \allowbreak \ldots, \mathfrak{h}\} \backslash \cu{ m } \colon I_i^\vartheta \cap [s, \scrb] = \emptyset$ (otherwise the claim in \cref{lem:realization:pointwise:bound:item3} follows from \cref{lem:realization:pointwise:bound:item1,lem:realization:pointwise:bound:item2}). \Nobs that for all $x \in [s, \scrb]$ we have that $\realization{\vartheta} ( x ) = \realization{\vartheta} ( s ) = \realization{\vartheta} ( \scrb )$ and $f( s ) \le f ( x ) \le f ( \scrb )$. Combining this with \cref{lem:realization:pointwise:bound:item1} assures for all $x \in [s , \scrb ]$ that 
\begin{equation}
\begin{split}
\textstyle \abs{\realization{\vartheta} ( x ) - f(x) } & \textstyle \le \max \cu[\big]{\abs{\realization{\vartheta} ( x ) - f(s) } , \abs{\realization{\vartheta} ( x ) - f(\scrb) }} \\
& \textstyle = \max \cu[\big]{\abs{\realization{\vartheta} ( s ) - f(s) } , \abs{\realization{\vartheta} (\scrb) - f(\scrb) }}  \le \max \cu{ V , \abs{f(\scrb) - \realization{\vartheta}(\scrb)}}.
\end{split}
\end{equation}
This establishes \cref{lem:realization:pointwise:bound:item3}. \Cref{lem:realization:pointwise:bound:item4} can be shown analogously as \cref{lem:realization:pointwise:bound:item3}.
\end{cproof}



\begin{cor} \label{cor:realization:endpoint:bound}
Assume \cref{setting:ann:clip}, assume that $f$ is non-decreasing and satisfies $\operatorname{Lip} ( f ) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$, let $\vartheta \in \mathbb{R}^{\mathfrak{h} + 1 }$ satisfy $( \nabla \mathcal{L} ) ( \vartheta ) = 0$, and let $V \in \mathbb{R}$ satisfy 
\begin{equation}
\textstyle V = \sup_{x \in ( \scra , \scrb ) } \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_i^\vartheta } v_i.
\end{equation}
Then
\begin{equation} \label{cor:realization:endpoint:bound:eq1}
\max \cu[\big]{ \realization{\vartheta} ( \scrb ) - f ( \scrb )  , f ( \scra ) - \realization{\vartheta} ( \scra ) } \le V.
\end{equation}
\end{cor} 

\begin{cproof}{cor:realization:endpoint:bound}
In the following we distinguish between the case $\forall \, i \in \{1, 2, \ldots, \mathfrak{h}\} \colon \allowbreak I_i^\vartheta \allowbreak = \emptyset$ and the case $\exists \, i \in \{1, 2, \ldots, \mathfrak{h}\} \colon I_i^\vartheta \neq \emptyset$. First we prove \cref{cor:realization:endpoint:bound:eq1} in the case $\forall \, i \in \{1, 2, \ldots, \allowbreak \mathfrak{h}\} \allowbreak \colon \allowbreak I_i^\vartheta \allowbreak = \emptyset$. \Nobs that for all $x \in (\scra , \scrb )$ we have that $\realization{\vartheta} ( x ) = \realization{\vartheta} ( \scrb )$. This, \cref{prop:risk:gradient}, and the assumption that $( \nabla \mathcal{L} ) ( \vartheta ) = 0$ demonstrate that 
\begin{equation}
\textstyle \int_\scra^\scrb ( \realization{\vartheta} ( \scrb ) - f ( x ) ) \mathfrak{p} ( x ) \, \d x = 0.
\end{equation}
The integral mean value theorem therefore assures that there exists $x \in (\scra , \scrb)$ such that $f(x) = \realization{\vartheta} ( \scrb ) = \realization{\vartheta} ( \scra )$. Combining this with the fact that $f$ is non-decreasing shows that $f(\scra) \le \realization{\vartheta} ( \scra )$ and $f ( \scrb ) \ge \realization{\vartheta} ( \scrb )$. This proves \cref{cor:realization:endpoint:bound:eq1} in the case $\forall \, i \in \{1, 2, \ldots, \mathfrak{h}\} \colon I_i^\vartheta = \emptyset$.

 Next we prove \cref{cor:realization:endpoint:bound:eq1} in the case $\exists \, i \in \{1, 2, \ldots, \mathfrak{h}\} \colon I_i^\vartheta \neq \emptyset$. Let $\mathscr{z} \in \mathbb{R}$ satisfy $\mathscr{z} = \sup \rbr{\cup_{i \in \{1, 2, \ldots, \mathfrak{h}\}} I_i^\vartheta}$. \Nobs that $\realization{\vartheta} ( \scrb ) = \realization{\vartheta} (\mathscr{z})$. Combining this with \cref{lem:realization:pointwise:bound} demonstrates that $\realization{\vartheta} ( \scrb ) - f ( \scrb ) \le \realization{\vartheta} ( \mathscr{z} ) - f ( \mathscr{z} ) \le V$. Analogously, we obtain that $\realization{\vartheta} ( \scra ) - f ( \scra ) \ge -  V$. This proves \cref{cor:realization:endpoint:bound:eq1} in the case $\exists \, i \in \{1, 2, \ldots, \mathfrak{h}\} \colon I_i^\vartheta \neq \emptyset$.
\end{cproof}



\begin{cor}
\label{cor:endpoint:bound:2}
Assume \cref{setting:ann:clip}, assume that $f$ is non-decreasing and satisfies $\operatorname{Lip} ( f ) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$, let $\vartheta \in \mathbb{R}^{\mathfrak{h} + 1 }$ satisfy $( \nabla \mathcal{L} ) ( \vartheta ) = 0$, and let $V \in \mathbb{R}$ satisfy 
\begin{equation}
\textstyle V = \sup_{x \in ( \scra , \scrb ) } \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_i^\vartheta } v_i.
\end{equation}
Then
\begin{equation}
\max \cu[\big]{ \abs{ \realization{\vartheta} ( \scra ) - f ( \scra ) } , 
\abs{ \realization{\vartheta} ( \scrb ) - f ( \scrb ) } } \le \max \cu*{ f ( \scrb ) - f ( \scra ) , V } .
\end{equation}
\end{cor}

\begin{cproof}{cor:endpoint:bound:2}
\Nobs that \cref{cor:realization:endpoint:bound} ensures that $\realization{\vartheta} ( \scrb ) - f ( \scrb ) \le V$ and $f(\scra ) - \realization{\vartheta} ( \scra ) \le V$. Next, \cref{prop:risk:gradient}, the fact that $f$ and $\realization{\vartheta}$ are non-decreasing, and the fact that $\rbr[\big]{ \frac{\partial}{\partial \theta_{\mathfrak{h} + 1 } } \mathcal{L} } ( \vartheta ) = 0$ demonstrate that
\begin{equation}
\begin{split}
\textstyle \realization{\vartheta} ( \scrb ) \int_\scra^\scrb  \mathfrak{p} ( x ) \, \d x & \textstyle = \int_\scra^\scrb \realization{\vartheta} ( \scrb ) \mathfrak{p} ( x ) \, \d x \ge \int_\scra^\scrb \realization{\vartheta} ( x ) \mathfrak{p} ( x ) \, \d x = \int_\scra^\scrb f ( x ) \mathfrak{p} ( x ) \, \d x \\
& \textstyle \ge \int_\scra^\scrb f ( \scra ) \mathfrak{p} ( x ) \, \d x = f ( \scra ) \int_\scra^\scrb  \mathfrak{p} ( x ) \, \d x.
\end{split}
\end{equation}
Hence, we obtain that $f( \scrb ) - \realization{\vartheta} ( \scrb ) \le f ( \scrb ) - f ( \scra )$. In the same way we can show that $\realization{\vartheta} ( \scra ) - f ( \scra ) \le f ( \scrb ) - f ( \scra )$.
\end{cproof}




\subsection{Convergence analysis for GFs}

In this section we study the GF dynamics for the ANNs considered in \cref{setting:ann:clip}. In particular, we show that the GF trajectory always converges to a critical point, and we establish estimates for the risk at this limit point.

\subsubsection{Generalized Kurdyka-{\L}ojasiewicz inequalities 
	for the risk function}

Under the assumption that both the target function $f$ and the density $\mathfrak{p}$ are piecewise polynomial, we
use the results from the results from our previous article \cite{JentzenRiekert2021ExistenceGlobMin} to establish in \cref{cor:KL} a Kurdyka-\L ojasiewicz (KL) inequality for the risk function.

\begin{definition}[Piecewise polynomial functions]
\label{def:piecewise_polynomial}
Let $ f \colon \mathbb{R} \to \mathbb{R} $ be a function. 
Then we say that $ f $ is piecewise polynomial if and only if 
there exist
$ K \in \mathbb{N} $, $ \tau_0, \tau_1, \ldots, \tau_K \in [- \infty , \infty ] $
such that 
\begin{enumerate}[label=(\roman*)]

\item 
it holds that 
$ - \infty = \tau_0 < \tau_1 < \cdots < \tau_K = \infty $
and 

\item 
it holds for all 
$ i \in \cu{ 1, 2, \dots, K } $ that 
$ f|_{ ( \tau_{ i - 1 }, \tau_i ) } $ is a polynomial. 

\end{enumerate}
\end{definition}


\cfclear
\begin{prop}
\label{cor:KL}
Assume \cref{setting:ann:clip}, let $ \theta \in \mathbb{R}^{ \mathfrak{h} + 1 } $, and assume that $ f $ and $ \mathfrak{p} $ are piecewise polynomial \cfadd{def:piecewise_polynomial}\cfload. Then there exist $\varepsilon, c \in (0, \infty)$, $\mathfrak{A} \in (0, 1)$ such that\footnote{\Nobs that $\norm{\cdot} \colon (\cup_{d \in \mathbb{N}} \mathbb{R}^d) \to \mathbb{R}$ satisfies for all $d \in \mathbb{N}$, $x = (x_1, \ldots, x_d) \in \mathbb{R}^d$ that $\norm{x} = \big[\sum_{j = 1}^d \abs{x_j}^2\big]^{1/2}$.} for all $\vartheta \in \{ \psi \in \mathbb{R}^{\mathfrak{h} + 1} \colon \norm{ \psi - \theta} < \varepsilon\}$, $\mathfrak{a} \in (\mathfrak{A}, 1]$ it holds that
\begin{equation} \label{cor:KL:eq:claim}
\textstyle \abs{\mathcal{L}(\theta) - \mathcal{L}(\vartheta)}^{\mathfrak{a}} \le c \norm{(\nabla \mathcal{L})(\vartheta)}.
\end{equation}
\end{prop}
\begin{cproof}{cor:KL}
\Nobs that the assumption that $f$ and $\mathfrak{p}$ are piecewise polynomial assures that $(\mathbb{R}^{ \mathfrak{h} + 1 } \times \mathbb{R} \ni (\theta , x ) \mapsto (\realization{\theta} ( x ) - f ( x ) ) ^2 \mathfrak{p} ( x ) \in \mathbb{R})$ 
is an element of the set $ \mathscr{F}_{\mathfrak{h} + 1 , 1 , \infty} $ 
defined in \cite[Definition~5.1]{JentzenRiekert2021ExistenceGlobMin}.
Combining this with \cite[Proposition~5.4]{JentzenRiekert2021ExistenceGlobMin} demonstrates that $(\mathbb{R}^{\mathfrak{h} + 1} \allowbreak \ni \allowbreak \theta \allowbreak \mapsto \allowbreak \mathcal{L} ( \theta) = \int_\scra^\scrb (\realization{\theta} ( x ) - f ( x ) ) ^2 \mathfrak{p} ( x ) \, \d x \in \mathbb{R} ) $ is semi-algebraic. The fact that $\mathcal{L} \in C^1 ( \mathbb{R}^{ \mathfrak{h} + 1 }, \mathbb{R})$ and \cite[Theorem~3.1]{BolteDaniilidis2006} therefore establish \cref{cor:KL:eq:claim}.
\end{cproof}


\begin{prop} \label{prop:gf:chain:rule}
Assume \cref{setting:ann:clip} and let $\Theta \in C([0, \infty), \mathbb{R}^{\mathfrak{h} + 1})$ satisfy for all $t \in [0, \infty)$ that $\Theta_t = \Theta_0 - \int_0^t ( \nabla \mathcal{L} ) ( \Theta_s ) \, \d s$. Then it holds for all $t \in [0, \infty )$ that $\mathcal{L} ( \Theta_t ) = \mathcal{L} ( \Theta_0 ) - \int_0^t \norm{( \nabla \mathcal{L} ) ( \Theta_s ) } ^2 \, \d s$.
\end{prop}
\begin{cproof}{prop:gf:chain:rule}
This is a direct consequence of \cref{prop:risk:gradient}, the chain rule, and the fundamental theorem of calculus.
\end{cproof}


\subsubsection{Boundedness of the GF trajectory}

We next verify in \cref{cor:a_priori_bound} that the GF trajectory is bounded. This will be used to establish convergence in \cref{cor:limit_point}. 

\begin{lemma}[Upper bounds for realization functions]
\label{lem:realization_estimate}
Assume \cref{setting:ann:clip} and let $\theta = (\theta_1, \allowbreak \dots, \allowbreak \theta_{\mathfrak{h} + 1}) \in \mathbb{R}^{\mathfrak{h} + 1}$, $x \in [\scra, \scrb]$. Then
\begin{equation}\label{eqn:lem:realization_estimate}
\textstyle \abs{\realization{\theta} ( x ) - \theta_{\mathfrak{h} + 1 }} \leq \sum_{ i = 1 }^{\mathfrak{h}} v_i.
\end{equation} 
\end{lemma}
\begin{cproof}{lem:realization_estimate}
\Nobs that the fact that for all $x \in \mathbb{R}$ it holds that $0 \le \mathfrak{c}(x) \le 1$ establishes \cref{eqn:lem:realization_estimate}.
\end{cproof}



\begin{prop}[Upper bounds for outer biases]
\label{prop:a_priori_bound0}
Assume \cref{setting:ann:clip} and let $\theta = (\theta_1, \allowbreak \dots, \allowbreak \theta_{\mathfrak{h} + 1}) \in \mathbb{R}^{\mathfrak{h} + 1}$. Then
\begin{equation} \label{prop:a_priori_bound:eqclaim}
\textstyle \abs{\theta_{\mathfrak{h} + 1}} \le \br[\big]{\int_{\scra}^{\scrb} \mathfrak{p}(x) \, \d x}^{- 1/2} \rbr[\big]{[\mathcal{L}(\theta)]^{1/2} + \br[\big]{\int_{\scra}^{\scrb} \abs{f(x)}^2 \mathfrak{p}(x) \, \d x}^{1/2}} + \sum_{i = 1}^{\mathfrak{h}} v_i < \infty.
\end{equation}
\end{prop}
\begin{cproof}{prop:a_priori_bound0}
\Nobs that \cref{lem:realization_estimate} and Minkowski's inequality ensure that
\begin{equation}
\begin{split}
\textstyle [\mathcal{L}( \theta )]^{1/2} & \textstyle = \br[\big]{\int_{\scra}^{\scrb} \br[\big]{(\realization{\theta}(x) - \theta_{\mathfrak{h} + 1}) - f(x) + \theta_{\mathfrak{h} + 1}}^2 \mathfrak{p}(x) \, \d x}^{1/2} \\ 
& \textstyle \ge \br[\big]{\int_{\scra}^{\scrb} (\theta_{\mathfrak{h} + 1})^2 \mathfrak{p}(x) \, \d x}^{1/2} - \br[\big]{\int_{\scra}^{\scrb} \br[\big]{(\realization{\theta}(x) - \theta_{\mathfrak{h} + 1}) - f(x)}^2 \mathfrak{p}(x) \, \d x}^{1/2} \\ 
& \textstyle \ge \abs{\theta_{\mathfrak{h} + 1}} \br[\big]{\int_{\scra}^{\scrb} \mathfrak{p}(x) \, \d x}^{1/2} - \br[\big]{\int_{\scra}^{\scrb} \br[\big]{\abs{f(x)} + (\sum_{i = 1}^{\mathfrak{h}} v_i)}^2 \mathfrak{p}(x) \, \d x}^{1/2}.
\end{split}
\end{equation}
\Hence 
\begin{equation}
\begin{split}
\textstyle \abs{\theta_{\mathfrak{h} + 1}} \br[\big]{\int_{\scra}^{\scrb} \mathfrak{p}(x) \, \d x}^{1/2} & \textstyle \le [\mathcal{L}(\theta)]^{1/2} + \br[\big]{\int_{\scra}^{\scrb} \br[\big]{\abs{f(x)} + (\sum_{i = 1}^{\mathfrak{h}} v_i)}^2 \mathfrak{p}(x) \, \d x}^{1/2} \\
& \textstyle \le [\mathcal{L}(\theta)]^{1/2} + \br[\big]{\int_{\scra}^{\scrb} \abs{f(x)}^2 \mathfrak{p}(x) \, \d x}^{1/2} + \br[\big]{\sum_{i = 1}^{\mathfrak{h}} v_i} \br[\big]{\int_{\scra}^{\scrb} \mathfrak{p}(x) \, \d x}^{1/2}.
\end{split}
\end{equation}
This establishes \cref{prop:a_priori_bound:eqclaim}.
\end{cproof}


\begin{cor}[Upper bounds for GF trajectories]
\label{cor:a_priori_bound}
Assume \cref{setting:ann:clip} and let 
$\Theta = (\Theta^1, \dots, \allowbreak \Theta^{\mathfrak{h} + 1}) \in C([0,\infty), \mathbb{R}^{\mathfrak{h} + 1})$ 
satisfy for all $t \in [0, \infty)$ that $\Theta_t = \Theta_0 - \int_0^t (\nabla \mathcal{L})(\Theta_s) \, \d s$. Then
\begin{enumerate}[label=(\roman*)]
\item
\label{item1:cor:a_priori_bound} it holds for all $k \in \{1, 2, \ldots, \mathfrak{h}\}$ that
\begin{equation}
\textstyle \sup\nolimits_{t \in [0, \infty)} \abs{\Theta^k_t} \le \max\{ w_k \abs{ \scra - [w_k ] ^{-1} }, w_k \abs{ \scrb } , \abs{ \Theta^k_0 } \} < \infty,
\end{equation}

\item
\label{item2:cor:a_priori_bound} it holds that
\begin{equation}
\begin{split}
&
\textstyle 
  \sup_{ t \in [0,\infty) } \abs{\Theta^{\mathfrak{h} + 1}_t} 
\\ & 
\textstyle 
  \le \sum_{i = 1}^{\mathfrak{h}} v_i + \br[\big]{\int_{\scra}^{\scrb} \mathfrak{p}(x) \, \d x}^{- 1/2} \rbr[\big]{[\mathcal{L}( \Theta_0 )]^{1/2} + \br[\big]{\int_{\scra}^{\scrb} \abs{f(x)}^2 \mathfrak{p}(x) \, \d x}^{1/2}} < \infty ,
\end{split}
\end{equation}

and 
\item
\label{item3:cor:a_priori_bound} it holds that
\begin{equation}
\begin{split}
\textstyle \sup\nolimits_{t \in [0,\infty)} \norm{\Theta_t} & \textstyle \le \sum_{i = 1}^{\mathfrak{h}} v_i + \mathfrak{h} \max\{(\abs{\scra} + 1) \norm{w}, \abs{\scrb}\norm{w}, \norm{\Theta_0}\} \\
& \textstyle \quad + \br[\big]{\int_{\scra}^{\scrb} \mathfrak{p}(x) \, \d x}^{- 1/2}\rbr[\big]{[\mathcal{L}( \Theta_0 )]^{1/2} + \big[\int_{\scra}^{\scrb}
\abs{f(x)}^2 \mathfrak{p}(x) \, \d x \big]^{1/2}} < \infty.
\end{split}
\end{equation}
\end{enumerate}
\end{cor}
\begin{cproof}{cor:a_priori_bound}
\Nobs that the fact that for all $k \in \cu{1, 2, \ldots, \mathfrak{h} }$, $t \in [0 , \infty )$ with $\abs{\Theta_t^k } \allowbreak > \max \cu{w_k \abs{ \scra - [w_k ] ^{-1} }, w_k \abs{ \scrb } }$ it holds that $\frac{\d}{\d t } \Theta_t ^k = - ( \frac{\partial}{\partial \theta_k} \mathcal{L} ) ( \Theta_t ) = 0$ establishes \cref{item1:cor:a_priori_bound}. 
Combining \cref{item1:cor:a_priori_bound} with
 \cref{prop:gf:chain:rule} and \cref{prop:a_priori_bound0} 
 establishes \cref{item2:cor:a_priori_bound,item3:cor:a_priori_bound}.
\end{cproof}



\subsubsection{Existence of a limiting value}

\cfclear
\begin{cor}
\label{cor:limit_point}
Assume \cref{setting:ann:clip}, assume that $f$ and $\mathfrak{p}$ are piecewise polynomial, 
and let $\Theta \in C( [0,\infty), \mathbb{R}^{ \mathfrak{h} + 1 } ) $ satisfy for all $ t \in [0,\infty)$ that 
$\Theta_t = \Theta_0 - \int_0^t ( \nabla \mathcal{L} ) ( \Theta_s ) \, \d s$ \cfadd{def:piecewise_polynomial}\cfload. 
Then there exist $ \vartheta \in \mathbb{R}^{ \mathfrak{h} + 1 } $, $ \alpha, \mathscr{C} \in (0,\infty) $
such that for all $ t \in [0,\infty) $ it holds that
\begin{equation}
\textstyle \norm{\Theta_t - \vartheta} \le \mathscr{C} (1 + t)^{- \alpha }, \; 0 \le \mathcal{L}(\Theta_t) - \mathcal{L}(\vartheta) \le \mathscr{C} (1 + t)^{- 1}, \; \text{and}\; \limsup_{s \to \infty} \norm{\Theta_s - \vartheta} = 0.
\end{equation}
\end{cor}
\begin{cproof}{cor:limit_point}
Combining \cref{cor:KL}, \cref{cor:a_priori_bound}, and the fact that $\mathcal{L} \in C^1 ( \mathbb{R}^{ \mathfrak{h} + 1 } , \allowbreak \mathbb{R} )$ with, e.g., \cite[Corollary~8.4]{JentzenRiekert2021ExistenceGlobMin} and \cref{prop:gf:chain:rule} ensures that there exist $\vartheta \in \mathbb{R}^{\mathfrak{h} + 1}$, $\alpha , \mathscr{C} \in (0, \infty)$ such that for all $t \in [0,\infty)$ it holds that
\begin{equation}
\textstyle \norm{\Theta_t - \vartheta} \le \mathscr{C} ( 1 + t )^{- \alpha } \qquad \text{and} \qquad 0 \le \mathcal{L}(\Theta_t) - \mathcal{L}(\vartheta) \le \mathscr{C} (1 + t)^{- 1}.
\end{equation}
\end{cproof}


\subsubsection{Activity of neurons during training}

In this section we analyze the behavior of the activity intervals $I_k^{ \Theta_t }$ during training.
We first prove in \cref{lem:never:leave:int} that a neuron can never become inactive (in the sense that its activity interval $I_k^{ \Theta_t }$ is empty)
in finite time.

\begin{lemma} 
\label{lem:never:leave:int}
Assume \cref{setting:ann:clip}, let $\Theta \in C ( [ 0 , \infty ) , \mathbb{R} ^{ \mathfrak{h} + 1 } ) $ satisfy for all $t \in [0, \infty )$ that $\Theta_t = \Theta_0 - \int_0^t ( \nabla \mathcal{L} ) ( \Theta_s ) \, \d s$,
and let $k \in \{1, 2, \ldots, \mathfrak{h}\}$ satisfy $I_k^{ \Theta_0 } \neq \emptyset$. Then it holds for all $t \in [0, \infty )$ that $I_k^{ \Theta_t } \neq \emptyset$.
\end{lemma}

The proof of \cref{lem:never:leave:int} essentially relies on well-known arguments which are used to prove uniqueness for ODEs with locally Lipschitz continuous coefficients.

\begin{cproof}{lem:never:leave:int}
Throughout this proof let $\tau \in \mathbb{R} \cup \{\infty\}$ satisfy $\tau = \inf \cu{t \in [0, \infty ) \colon I_k^{ \Theta_t } = \emptyset}$. We assume for the sake of contradiction that $\tau  < \infty $. \Nobs that the fact that $[0, \infty ) \ni t \mapsto \psi_k ( \Theta_t ) \in \mathbb{R} $ is continuous and \cref{prop:active:intervals} ensure for all $t \in [0, \tau )$ that $\psi_k ( \Theta_t ) \in (\scra - \allowbreak [w_k]^{-1}, \allowbreak \scrb)$ and $\psi_k ( \Theta_\tau ) \in \cu{ \scra - [w_k]^{-1} , \scrb }$. Moreover, we have that $\tau > 0$. In the following we assume without loss of generality that $\psi_k (\Theta_\tau ) = \scrb $. Let $\mathfrak{C} \in ( 0, \infty )$ satisfy
\begin{equation} \label{lem:never:leave:int:eq1}
\textstyle \mathfrak{C} = 2 \rbr[\big]{\sum_{i=1}^\mathfrak{h} v_i + \br[\big]{\sup\nolimits_{t \in [ 0 , \tau ] } \abs{\Theta_t^{ \mathfrak{h} + 1 } } }
+ \br[\big]{ \sup\nolimits_{x \in [ \scra , \scrb ] } \abs { f ( x ) } } }  v_k [w_k]^{-1} \sup\nolimits_{x \in [ \scra , \scrb ] } \mathfrak{p} ( x ).
\end{equation}
\Nobs that \cref{prop:risk:gradient}, \cref{lem:realization_estimate}, and \cref{lem:never:leave:int:eq1} demonstrate for all $t \in [ 0 , \tau ]$ that
\begin{equation} \label{lem:never:leave:int:eq2}
\textstyle [w_k]^{-1} \abs[\big]{\rbr[\big]{ \tfrac{\partial}{\partial \theta_k } \mathcal{L} } ( \Theta_t)} \le 2 v_k [w_k]^{-1} \int_{I_k^{\Theta_t }} \abs{\realization{\Theta_t} ( x ) - f ( x ) } \mathfrak{p} ( x ) \, \d x \le \mathfrak{C} \abs{ \psi_k ( \Theta_t ) - \scrb }.
\end{equation}
Next let $\delta \in (0, \infty )$ satisfy $\delta = \min \cu{[2 \mathfrak{C}]^{-1}, \tau}$ and let $M \in \mathbb{R}$ satisfy $M = \sup_{t \in [\tau - \delta, \tau]} \abs{\psi_k (\Theta_t) - \scrb}$. \Nobs that the definition of $\tau$ proves that $M > 0$. Furthermore, \cref{lem:never:leave:int:eq2} shows for all $s \in [\tau - \delta, \tau]$ that
\begin{equation}
\begin{split}
\textstyle \abs{\psi_k ( \Theta_{s } ) - \scrb } & \textstyle = \abs{\psi_k ( \Theta_{s } ) - \psi_k ( \Theta_{\tau  } ) }
= \abs[\big]{\int_{s}^\tau  [w_k]^{-1} \rbr[\big]{ \tfrac{\partial}{\partial \theta_k } \mathcal{L} } ( \Theta_t ) \, \d t } \\
& \textstyle \le \int_{s}^\tau [w_k]^{-1} \abs[\big]{\rbr[\big]{\frac{\partial}{\partial \theta_k} \mathcal{L}} (\Theta_t)} \, \d t \le \mathfrak{C} ( \tau - s ) M \le \mathfrak{C} \delta M \le \tfrac{M}{2}.
\end{split}
\end{equation}
Hence, we obtain that $M \le \tfrac{M}{2}$, which contradicts the fact that $M > 0$.
\end{cproof}

Next, we show that if a neuron becomes inactive in the limit (in the sense that $I_k^\vartheta = \emptyset$) then a certain endpoint bound for the realization $\realization{\vartheta}$ must be satisfied. Roughly speaking, the activity interval must leave the interval $( \scra , \scrb )$ either on the right or on the left, which is why we formulate two separate results in \cref{lem:leave:interval:bound} and \cref{lem:leave:interval:bound:2}.


\begin{lemma}
\label{lem:leave:interval:bound}
Assume \cref{setting:ann:clip}, let $\Theta \in C([0, \infty ), \mathbb{R}^{\mathfrak{h} + 1})$ satisfy for all $t \in [0, \infty)$ that $\Theta_t = \Theta_0 - \int_0^t ( \nabla \mathcal{L} ) ( \Theta_s ) \, \d s$, let $\vartheta \in \mathbb{R}^{\mathfrak{h} + 1 }$ satisfy $\limsup_{t \to \infty} \norm{\Theta_t - \vartheta } = 0$, and let $k \in \{1, 2, \ldots, \mathfrak{h}\}$ satisfy $I_k^{ \Theta_0 } \neq \emptyset$ and $\psi_k (\vartheta) = \scrb$. Then $f(\scrb) \le \realization{\vartheta}(\scrb)$.
\end{lemma}
\begin{cproof}{lem:leave:interval:bound}
Throughout this proof let $\delta \in \mathbb{R}$ satisfy $\delta =  \frac{1}{2} (f(\scrb) - \realization{\vartheta}(\scrb))$ and assume for the sake of contradiction that $\delta > 0$. \Nobs that the assumption that $\limsup_{t \to \infty } \norm{\Theta_t - \vartheta } = 0$ implies that $\lim_{t \to \infty} \psi_k ( \Theta_t ) = \scrb$ and $\lim_{t \to \infty} \realization{\Theta_t} ( \psi_k ( \vartheta ) ) = \realization{\vartheta} ( \scrb )$. In addition, the assumption that $\delta > 0$, \cref{lem:never:leave:int}, and the fact that $f$ is continuous demonstrate that there exists $T \in [0, \infty )$ which satisfies for all $t \in [T, \infty )$ that
\begin{equation}
\textstyle f ( \psi_{k} ( \Theta_t ) ) > \realization{\Theta_t} ( \psi_k ( \Theta_t ) ) + \delta \quad \text{and} \quad \psi_k ( \Theta_t ) \in \rbr[\big]{ \scrb - \min \cu[\big]{[w_k]^{-1}, \delta \br[\big]{\sum_{i=1}^\mathfrak{h} v_i w_i}^{-1}}, \scrb}.
\end{equation}
Hence, we obtain for all $t \in [T, \infty)$, $x \in I_k^{\Theta_t}$ that $I_k^{\Theta_t} = (\psi_k (\Theta_t), \scrb)$ and
\begin{equation} \label{lem:leave:interval:bound:eq:mainest}
\begin{split}
\textstyle \realization{\Theta_t}(x) & \textstyle \le \realization{\Theta_t} (\psi_k ( \Theta_t ) ) + \rbr[\big]{\sum_{i = 1}^\mathfrak{h} v_i w_i } (x - \psi_k (\Theta_t)) \\
& \textstyle \le \realization{\Theta_t} ( \psi_k ( \Theta_t ) ) + \rbr[\big]{\sum_{i=1}^\mathfrak{h} v_i w_i } ( \scrb - \psi_k ( \Theta_t ) ) \\
& \textstyle \le \realization{\Theta_t} ( \psi_k ( \Theta_t ) ) + \delta < f ( \psi_k ( \Theta_t ) ) \le f ( x ) .
\end{split}
\end{equation}
This and \cref{prop:risk:gradient} prove for all $t \in [T , \infty )$
that $\rbr{\frac{\partial}{\partial \theta_k } \mathcal{L} } ( \Theta_t ) \le 0$. Combining this with the fact that for all $t \in [0, \infty )$
it holds that $\psi_k ( \Theta_t ) = \psi_k ( \Theta_0 ) + [w_k]^{-1} \int_0^t \rbr[\big]{ \frac{\partial}{\partial \theta_k } \mathcal{L} } ( \Theta_s ) \, \d s$ ensures that $[T , \infty ) \ni t \mapsto \psi_{k } ( \Theta_t ) \in \mathbb{R}$ is non-increasing. However, this contradicts the fact that $\lim_{t \to \infty} \psi_{ k } ( \Theta_t ) = \scrb > \psi_{k } ( \Theta_T)$.
\end{cproof}

Analogously we can prove the following result for the other endpoint.


\begin{lemma}
\label{lem:leave:interval:bound:2}
Assume \cref{setting:ann:clip}, let $\Theta \in C([0, \infty ), \mathbb{R}^{\mathfrak{h} + 1 } )$ satisfy for all $t \in [0, \infty )$ that $\Theta_t = \Theta_0 - \int_0^t ( \nabla \mathcal{L} ) ( \Theta_s ) \, \d s$, let $\vartheta \in \mathbb{R}^{\mathfrak{h} + 1 }$ satisfy $\limsup_{t \to \infty } \norm{\Theta_t - \vartheta } = 0$, and let $k \in \{1, 2, \ldots, \mathfrak{h}\}$ satisfy $I_k^{ \Theta_0 } \not= \emptyset$ and $\psi_k (\vartheta) = \scra - [w_k]^{-1}$. Then $f(\scra) \ge \realization{\vartheta}(\scra)$.
\end{lemma}

Next, we show in \cref{lem:neuron:leaves:interval} that under a suitable condition on the outer weights $v_i$, there necessarily exists a neuron which becomes inactive in the limit, and thus at least one of the previous endpoint bounds is applicable.

\begin{lemma} 
\label{lem:neuron:leaves:interval}
Assume \cref{setting:ann:clip}, assume that $f$ is non-decreasing
and satisfies $\operatorname{Lip} ( f ) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \allowbreak v_i w_i$, let $\Theta \in C([0, \infty ), \mathbb{R}^{\mathfrak{h} + 1 } )$ satisfy for all $t \in [0, \infty )$ that $\Theta_t = \allowbreak \Theta_0 - \allowbreak \int_0^t ( \nabla \mathcal{L} ) ( \Theta_s ) \allowbreak \, \d s$, let $\vartheta \in \mathbb{R}^{\mathfrak{h} + 1 }$ satisfy $\limsup_{t \to \infty } \norm{\Theta_t - \vartheta } = 0$, let $V \in \mathbb{R}$ satisfy 
\begin{equation}
\textstyle V = \sup_{x \in ( \scra , \scrb ) } \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_i^\vartheta } v_i,
\end{equation}
and assume
\begin{equation}  \label{lem:neuron:leaves:interval:eqass}
\textstyle \sum_{i=1}^\mathfrak{h}  v_i \indicator{(\scra - [w_i]^{-1} , \scrb)}(\psi_i (\Theta_0)) > f(\scrb) - f(\scra) + 4 V.
\end{equation}
Then there exists $i \in \{1, 2, \ldots, \mathfrak{h}\}$ such that $I_i^{\Theta_0} \neq \emptyset$ and $I_i^{\vartheta} = \emptyset$.
\end{lemma}
\begin{cproof}{lem:neuron:leaves:interval}
First, \cref{prop:risk:gradient} ensures that $( \nabla \mathcal{L} ) ( \vartheta ) = 0$. Furthermore, \nobs that
\begin{equation}
\textstyle \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, \psi_i ( \vartheta ) \in (\scrb - [w_i]^{-1}, \scrb )} v_i \le V \quad \text{and} \quad \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, \psi_i ( \vartheta ) \in (\scra - [w_i]^{-1}, \scra )} v_i \le V.
\end{equation}
Hence, we obtain that
\begin{equation}
\textstyle \realization{\vartheta} ( \scrb ) - \realization{\vartheta} ( \scra ) \ge \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, \psi_i ( \vartheta ) \in [ \scra , \scrb - [ w_i]^{-1} ] } v_i \ge \br[\big]{\sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, I_i^\vartheta \neq \emptyset} v_i} - 2 V.
\end{equation}
This, \cref{prop:active:intervals}, \cref{lem:neuron:leaves:interval:eqass}, and \cref{cor:realization:endpoint:bound} imply that
\begin{equation}
\begin{split}
& \textstyle \br[\Big]{\sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, I_i^{\Theta_0} \neq \emptyset} v_i} - (f(\scrb) - f(\scra)) \\
& \textstyle = \br[\big]{\sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, \psi_i (\Theta_0) \in (\scra - [w_i]^{-1}, \scrb)} v_i} - (f(\scrb) - f(\scra)) \\
& \textstyle > 4 V \ge (\realization{\vartheta}(\scrb) - f(\scrb)) + (f(\scra) - \realization{\vartheta}(\scra)) + 2 V \\
& \textstyle = (\realization{\vartheta}(\scrb) - \realization{\vartheta}(\scra) + 2 V) - (f(\scrb) - f(\scra)) \\
& \textstyle \ge \br[\big]{\sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, I_i^\vartheta \not= \emptyset} v_i} - ( f ( \scrb ) - f ( \scra ) ).
\end{split}
\end{equation}
Therefore, we obtain that $\cu{i \in \{1, 2, \ldots, \mathfrak{h}\} \colon I_i^\vartheta \neq \emptyset} \subsetneq \cu{i \in \{1, 2, \ldots, \mathfrak{h}\} \colon I_i^{\Theta_0 } \neq \emptyset}$.
\end{cproof}

In the next step we verify in \cref{lem:limit:not:const} that there must exist at least one active neuron for the limit point $\vartheta$.

\begin{lemma}\label{lem:limit:not:const}
Assume \cref{setting:ann:clip}, assume that $f$ is non-decreasing and satisfies $\operatorname{Lip} ( f ) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$, let $\Theta \in C([0, \infty ), \mathbb{R}^{\mathfrak{h} + 1 } )$ satisfy for all $t \in [0, \infty )$ that $\Theta_t = \allowbreak \Theta_0 - \allowbreak \int_0^t ( \nabla \mathcal{L} ) ( \Theta_s ) \allowbreak \, \d s$,
let $\vartheta \in \mathbb{R}^{\mathfrak{h} + 1 }$ satisfy $\limsup_{t \to \infty } \norm{\Theta_t - \vartheta } = 0$, assume
\begin{equation} \label{lem:limit:not:const:eq:ass1}
\textstyle \sup_{x \in (\scra , \scrb)} \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_i^\vartheta} v_i < \frac{\min\cu{\int_\scra^\scrb ( f(\scrb ) - f(x) ) \mathfrak{p} ( x ) \, \d x, \int_\scra^\scrb ( f( x ) - f ( \scra ) ) \mathfrak{p} ( x ) \, \d x} }{\int_\scra^\scrb \mathfrak{p} ( x ) \, \d x},
\end{equation}
and assume $\cu{i \in \{1, 2, \ldots, \mathfrak{h}\} \colon \psi_i ( \Theta_0 ) \in ( \scra - [w_i]^{-1}, \scrb)} \neq \emptyset$. Then $\cu{i \in \{1, 2, \ldots, \mathfrak{h}\} \colon \psi_i ( \vartheta ) \in ( \scra - [w_i]^{-1}, \scrb)} \neq \emptyset$.
\end{lemma}
\begin{cproof}{lem:limit:not:const}
Throughout this proof we assume for the sake of contradiction that
\begin{equation} \label{lem:limit:not:const:eq:cont}
\cu{i \in \{1, 2, \ldots, \mathfrak{h}\} \colon \psi_i ( \vartheta ) \in ( \scra - [w_i]^{-1}, \scrb)} = \emptyset.
\end{equation}
\Nobs that \cref{lem:limit:not:const:eq:cont} demonstrates for all $x \in (\scra, \scrb)$ that $\realization{\vartheta}(x) = \realization{\vartheta}(\scrb) = \realization{\vartheta}(\scra)$. In addition, \cref{lem:limit:not:const:eq:cont} implies that there exists $k \in \{1, 2, \ldots, \mathfrak{h}\}$ which satisfies $\psi_k ( \Theta_0 ) \in (\scra - [w_k]^{- 1}, \scrb)$ and $\psi_k ( \vartheta ) \allowbreak \in \allowbreak \cu{\scra \allowbreak - [w_k]^{-1}, \scrb}$. Assume without loss of generality that $\psi_k ( \vartheta ) = \scrb$. \Nobs that
the fact that $(\nabla \mathcal{L} ) ( \vartheta ) = 0$,
 \cref{cor:realization:endpoint:bound}, and \cref{lem:leave:interval:bound} assure that
\begin{equation} \label{lem:limit:not:const:eq2}
\textstyle \abs{\realization{\vartheta} ( \scrb ) - f ( \scrb ) } = \realization{\vartheta} ( \scrb ) - f ( \scrb ) \le \sup\nolimits_{x \in (\scra , \scrb ) } \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_i^\vartheta} v_i.
\end{equation}
Furthermore, the fact that $( \frac{ \partial}{\partial \theta_{\mathfrak{h} + 1 } } \mathcal{L} ) ( \vartheta ) = 0$ implies that
\begin{equation}
\textstyle \realization{\vartheta} ( \scrb ) = \br[\big]{\int_\scra^\scrb \mathfrak{p} ( x ) \, \d x }^{- 1} \rbr[\big]{\int_\scra^\scrb \realization{\vartheta} ( x ) \mathfrak{p} ( x ) \, \d x } = \br[\big]{\int_\scra^\scrb \mathfrak{p} ( x ) \, \d x}^{- 1} \rbr[\big]{\int_\scra^\scrb f(x) \mathfrak{p}(x) \, \d x}.
\end{equation}
Combining this with \cref{lem:limit:not:const:eq:ass1,lem:limit:not:const:eq2} yields a contradiction.
\end{cproof}

\subsubsection{Convergence with deterministic initial value}

Finally, we are in a position to establish in \cref{theo:conv:bias:deterministic} the main result about the convergence of GF with deterministic initialization, which relies on the previous lemmas from this section.

\begin{theorem}
\label{theo:conv:bias:deterministic}
Assume \cref{setting:ann:clip}, assume that $f$ is non-decreasing and satisfies $\operatorname{Lip} ( f ) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$, let $\Theta \in C([0, \infty ), \mathbb{R}^{\mathfrak{h} + 1 } )$ satisfy for all $t \in [0, \infty )$ that $\Theta_t = \allowbreak \Theta_0 - \allowbreak \int_0^t ( \nabla \mathcal{L} ) ( \Theta_s ) \allowbreak \, \d s$, let $\vartheta \in \mathbb{R}^{\mathfrak{h} + 1 }$ satisfy $\limsup_{t \to \infty } \norm{\Theta_t - \vartheta } = 0$, let $V \in \mathbb{R}$ satisfy 
\begin{equation}
\textstyle V = \sup_{x \in ( \scra , \scrb ) } \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_i^\vartheta } v_i,
\end{equation}
assume
\begin{equation}
\textstyle V < \br[\big]{\int_\scra^\scrb \mathfrak{p} ( x ) \, \d x } ^{-1} 
\min\cu[\big]{ \int_\scra^\scrb ( f(\scrb ) - f(x) ) \mathfrak{p} ( x ) \, \d x, \int_\scra^\scrb ( f( x ) - f(\scra)) \mathfrak{p} ( x ) \, \d x} ,
\end{equation}
and assume $\sum_{i = 1 }^ \mathfrak{h}  v_i \indicator{ ( \scra - [w_i]^{-1} , \scrb ) } ( \psi_i ( \Theta_0 ) ) > f ( \scrb ) - f ( \scra ) + 4 V$. Then 
\begin{equation}
\label{theo:conv:bias:deterministic:eqclaim}
\textstyle \mathcal{L}(\vartheta) \le \rbr{2 V ^2 + ( f(\scrb ) - f(\scra)) V} \int_\scra^\scrb \mathfrak{p}(x) \, \d x.
\end{equation}
\end{theorem}
\begin{cproof}{theo:conv:bias:deterministic}
First, \cref{prop:risk:gradient} implies that $(\nabla \mathcal{L} ) ( \vartheta ) = 0$. Combining this with \cref{lem:limit:not:const}
ensures that $\cup_{i \in \{1, 2, \ldots, \mathfrak{h}\}} I_i^\vartheta \not= \emptyset$. Let $q ,r \in [ \scra , \scrb ]$ satisfy $q = \inf (\cup_{i \in \{1, 2, \ldots, \mathfrak{h}\}} I_i^\vartheta)$ and $r = \sup (\cup_{i \in \{1, 2, \ldots, \mathfrak{h}\}} I_i^\vartheta)$. \Nobs that \cref{lem:realization:pointwise:bound} implies that
\begin{equation} \label{theo:conv:bias:deterministic:eq1}
\textstyle \sup_{x \in (q, r)} \abs{\realization{\vartheta}(x) - f(x)} \le V,
\end{equation}
\begin{equation} \label{theo:conv:bias:deterministic:eq3}
\textstyle \sup_{x \in (\scra, q)} \abs{\realization{\vartheta}(x) - f(x)} \le \max \cu{V, \abs{\realization{\vartheta}(\scra) - f(\scra)}},
\end{equation}
and
\begin{equation} \label{theo:conv:bias:deterministic:eq4}
\textstyle \sup_{x \in (r, \scrb)} \abs{\realization{\vartheta}(x) - f(x)} \le \max \cu{V, \abs{\realization{\vartheta}(\scrb) - f(\scrb)}}.
\end{equation}
We assume without loss of generality that there exists $k \in \{1, 2, \ldots, \mathfrak{h}\}$ which satisfies $\psi_k (\Theta_0) \allowbreak \in (\scra - [w_k]^{-1}, \scrb)$ and $\psi_k (\vartheta) = \scrb$ (cf.\ \cref{lem:neuron:leaves:interval}). Combining this with \cref{cor:realization:endpoint:bound} and \cref{lem:leave:interval:bound} assures that $\abs{\realization{\vartheta}(\scrb) - f(\scrb)} \le V$.

 In the following we distinguish between the case $\realization{\vartheta}(\scra) \le f(\scra)$ and the case $\realization{\vartheta}(\scra) > f(\scra)$. First we prove \cref{theo:conv:bias:deterministic:eqclaim} in the case $\realization{\vartheta}(\scra) \le f(\scra)$. \Nobs that \cref{cor:realization:endpoint:bound} assures that $\abs{\realization{\vartheta} ( \scra ) - f ( \scra ) } \le V$. Combining this with \cref{theo:conv:bias:deterministic:eq1,theo:conv:bias:deterministic:eq3,theo:conv:bias:deterministic:eq4} demonstrates that
\begin{equation}
\textstyle \mathcal{L}(\vartheta) = \int_\scra^\scrb (\realization{\vartheta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le V^2 \int_\scra^\scrb \mathfrak{p}(x) \, \d x.
\end{equation}
This proves \cref{theo:conv:bias:deterministic:eqclaim} in the case $\realization{\vartheta}(\scra) \le f(\scra)$. 

In the next step we prove \cref{theo:conv:bias:deterministic:eqclaim} in the case $\realization{\vartheta}(\scra) > f(\scra)$. Let $i \in \{1, 2, \ldots, \mathfrak{h}\}$ satisfy $q = \inf (I_i^\vartheta)$. \Nobs that \cref{prop:risk:gradient} and the integral mean value theorem imply that there exists $z \in I_i^\vartheta$ which satisfies $\realization{\vartheta}(z) = f(z)$. This and the assumption that $\operatorname{Lip}(f) < \min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i w_i$ ensure that $\realization{\vartheta}(q) < f(q)$. Combining this with the intermediate value theorem demonstrates that $\cu{u \in (\scra, q) \colon \realization{\vartheta}(u) = f(u)} \neq \emptyset$. Let $y = \inf \cu{u \in (\scra, q) \colon \realization{\vartheta}(u) = f(u)}$. \Nobs that the fact that $\forall \, x \in (\scra , q ) \colon \realization{\vartheta}(x) = \realization{\vartheta}(q)$ and \cref{theo:conv:bias:deterministic:eq1} prove that $\sup_{x \in (y, q ) } \abs { \realization{\vartheta} ( x ) - f ( x ) } \le V$. The fact that $( \frac{\partial}{\partial \theta_{\mathfrak{h} + 1 } } \mathcal{L} ) (\vartheta) = 0$, \cref{theo:conv:bias:deterministic:eq1}, and \cref{theo:conv:bias:deterministic:eq4} therefore show that
\begin{equation}
\begin{split}
\textstyle \int_\scra^y ( \realization{\vartheta} ( x ) - f ( x ) ) \mathfrak{p} ( x ) \, \d x & \textstyle = - \int_y^\scrb ( \realization{\vartheta} ( x ) - f ( x ) ) \mathfrak{p} ( x ) \, \d x \\
& \textstyle \le \int_y^\scrb \abs{\realization{\vartheta} ( x ) - f ( x ) } \mathfrak{p} ( x ) \, \d x \le V \int_y^\scrb \mathfrak{p} ( x ) \, \d x \le V \int_\scra^\scrb \mathfrak{p} ( x ) \, \d x.
\end{split}
\end{equation}
Combining this with the fact that for all $x \in (\scra , y )$ it holds that $\realization{\vartheta} ( x ) \ge f ( x )$ and \cref{theo:conv:bias:deterministic:eq3} demonstrates that
\begin{equation}
\begin{split}
\textstyle \int_\scra^y ( \realization{\vartheta} ( x ) - f ( x ) )^2 \mathfrak{p} ( x ) \, \d x & \textstyle \le \br[\big]{ \sup\nolimits_{x \in (\scra , y ) } \abs{\realization{\vartheta} ( x ) - f ( x ) } } \int_\scra^y ( \realization{\vartheta} ( x ) - f ( x ) ) \mathfrak{p} ( x ) \, \d x \\
& \textstyle \le V \max \cu{f(\scrb ) - f(\scra), V} \int_\scra ^\scrb \mathfrak{p} ( x ) \, \d x.
\end{split}
\end{equation}
This, \cref{theo:conv:bias:deterministic:eq3}, and \cref{theo:conv:bias:deterministic:eq4} show that
\begin{equation}
\textstyle \mathcal{L}(\vartheta) \le \rbr{V^2 + V \max \cu{f(\scrb) - f(\scra), V}} \int_\scra^\scrb \mathfrak{p} ( x ) \, \d x.
\end{equation}
This establishes \cref{theo:conv:bias:deterministic:eqclaim} in the case $\realization{\vartheta}(\scra) > f(\scra)$.
\end{cproof}

\subsection{Convergence results for GF with random initializations}
Our goal in this section is to verify that for our particular initialization the conditions of \cref{theo:conv:bias:deterministic} are satisfied with high probability and, thereby, prove our main convergence result for GF with a single random initialization.

\subsubsection{Products of half-normally distributed random variables}

In our main convergence result, \cref{theo:main:conv}, the weights $v_i, w_i$ will be initialized randomly. Since they are required to be positive, we will use a half-normal distribution instead of the popular normal distribution.
The first goal is to prove \cref{lem:random:init} below. For this we require several technical results on products of two independent half-normal random variables and sums of random variables of this type.


\begin{definition}[Modified Bessel function]
\label{def:bessel}
We denote by $K_0 \colon (0, \infty ) \to [ 0 , \infty ]$ the function which satisfies for all $x \in (0, \infty )$ that
\begin{equation}
\textstyle K_0 (x) = \int_0^\infty \exp(-x \cosh(w)) \, \d w.
\end{equation}
\end{definition}

The modified Bessel function of the first kind appears when considering the product of two independent normal random variables.
The next result is well-known in the physics literature; cf., e.g., Abramowitz \& Stegun~\cite[Chapter 9]{AbramowitzStegun1992}.

\cfclear
\begin{lemma}[Properties of the Modified Bessel function]
\label{lem:bessel:properties} \cfadd{def:bessel}
We have that
\begin{enumerate} [label = (\roman*)]
\item it holds that $K_0$ is strictly decreasing,

\item it holds for all $x \in (0, \infty)$ that $0 < K_0(x) < \infty$,

\item it holds that $\liminf_{x \searrow 0} K_0(x) = \infty$, and

\item it holds that $\limsup_{x \searrow 0} \Abs{\frac{K_0(x)}{\abs{\ln(x)}} - 1} = 0$
\end{enumerate}
\cfload.
\end{lemma}

As a consequence,
we obtain the following well-known result on products of standard normal random variables. The detailed proof is only included for completeness.

\cfclear
\begin{prop}
\label{prop:product:normal}
Let $(\Omega, \mathcal{F}, \P)$ be a probability space, let $X \colon \Omega \to (0, \infty )$ and $Y \colon \Omega \to (0, \infty )$ be i.i.d.\ random variables, and assume for all $x \in ( 0, \infty )$ that $\P ( X \le x ) = [\frac{2}{\pi}]^{1/2} \int_0^x \exp(- \frac{y^2}{2}) \, \d y$. Then 
\begin{enumerate} [label = (\roman*)]
\item
\label{prop:product:normal:item1} it holds for all $z \in (0, \infty)$ that $\P (XY \le z) = \frac{2}{\pi} \int_0^z K_0 ( y ) \, \d y$ and \cfadd{def:bessel}

\item
\label{prop:product:normal:item2} it holds that there exists $c \in (0, \infty )$ such that for all $z \in (0, \infty)$ it holds that $\P ( X Y \le z ) \le c z (1 + \abs{\ln(z)})$
\end{enumerate}
\cfload.
\end{prop}

The distribution of $X$ and $Y$ is sometimes called half-normal distribution or folded normal distribution.

\begin{cproof}{prop:product:normal}
	First, the fact that $X,Y$ are independent, the integral transformation theorem, and Fubini's theorem ensure for all $z \in ( 0 , \infty )$ that
	\begin{equation} \label{prop:product:normal:eq1}
	\begin{split}
	\textstyle \P ( X Y \le z ) & \textstyle = \frac{2}{\pi } \int_0^\infty \int_0^\infty \exp ( - \frac{x^2 + y^2 }{ 2 } ) \indicator{[0 , z ] } ( x y ) \, \d y \, \d x \\
	& \textstyle = \frac{2}{\pi } \int_{0}^{\infty} \int_{0}^{z/x} \exp ( - \frac{x^2 + y^2 }{ 2 } ) \, \d y \, \d x \\
	& \textstyle = \frac{2}{\pi } \int_{0}^{\infty} \int_{0}^{z} \exp ( - \frac{x^2 + (u/x)^2 }{2} ) \frac{1}{x} \, \d u \, \d x \\
	& \textstyle = \frac{2}{\pi} \int_{0}^{z} \rbr[\big]{\int_{0}^{\infty} \frac{1}{x} \exp(- \frac{x^2 + (u/x)^2}{2}) \, \d x} \, \d u.
	\end{split}
	\end{equation}
	Furthermore, the integral transformation theorem with the diffeomorphism $\mathbb{R} \ni w \mapsto \allowbreak u^{1/2} \allowbreak \exp(\frac{w}{2}) \allowbreak \in (0, \infty)$ and the fact that for all $w \in \mathbb{R}$ it holds that $\cosh(w) = \cosh(- w)$ show that for all $u \in (0, \infty )$ it holds that
	\begin{equation}
	\begin{split}
	& \textstyle \int_{ ( 0 , \infty ) } \frac{1}{x} \exp \rbr[\big]{ - \tfrac{x^2 + (u/x)^2 }{2} } \, \d x = \int_\mathbb{R}  \frac{1}{\sqrt{u}} \exp ( - \frac{w}{2} ) \exp \rbr[\big]{ - \frac{u e^w + u e^{-w} }{2} } \frac{ \sqrt{u }}{2} \exp ( \frac{w}{2} ) \, \d w \\
	& \textstyle = \frac{1}{2} \int_\mathbb{R}  \exp ( - u \cosh ( w ) ) \, \d w 
	= \int_0^\infty \exp ( - u \cosh ( w ) ) \, \d w = K_0 ( u ).
	\end{split}
	\end{equation}
	Combining this with \cref{prop:product:normal:eq1} establishes \cref{prop:product:normal:item1}. Next, \cref{lem:bessel:properties}
	implies that there exists $\varepsilon \in (0 , 1 )$ which satisfies for all $x \in (0, \varepsilon )$ that $K_0 ( x ) \le 2 \abs{\ln(x)}$. Combining this with \cref{prop:product:normal:item1} proves for all $z \in (0, \varepsilon]$ that
	\begin{equation}
	\textstyle \P ( X Y \le z ) \le \frac{4}{\pi } \int_0^z \abs{\ln(y)} \, \d y = \frac{4}{\pi } z ( 1 + \abs{\ln(z)}).
	\end{equation}
	%
	This, the fact that $\forall \, z \in ( 0, \infty ) \colon \P ( X Y \le z ) \le 1$, and the fact that $\forall \, z \in [ \varepsilon, \infty ) \colon z ( 1 + \abs{\ln(z)}) \allowbreak \ge \varepsilon$ demonstrate for all $z \in (0 , \infty )$ that $\P ( X Y \le z ) \le \max \cu{\varepsilon^{-1} , 4 \pi ^{-1} } z ( 1 + \abs{\ln(z)})$. This establishes \cref{prop:product:normal:item2}.
\end{cproof}

\subsubsection{Properties of order statistics}

\begin{lemma} \label{lem:log:estimate:help}
Let $c, \delta \in (0, \infty)$, let $g \colon (0,1) \to (0, \infty)$ be non-decreasing, and assume for all $u \in (0 , 1 )$ that $g ( u ) ( 1 + \abs{\ln(g(u))}) \ge c u$. Then $\inf_{u \in (0, 1)} u^{-1 - \delta} g(u) > 0$.
\end{lemma}
\begin{cproof}{lem:log:estimate:help}
Throughout this proof let $h \colon (0, 1) \to (0, \infty)$ satisfy $\forall \, u \in (0 , 1 ) \colon h ( u ) = u ^{-1 - \delta } g ( u )$. \Nobs that the assumption that $\forall \, u \in (0 , 1 ) \colon g ( u ) ( 1 + \abs{\ln(g(u))}) \ge c u$ demonstrates for all $u \in (0, 1)$ that
\begin{equation}\label{eqn:lem:log:estimate:help:lowerbound}
\textstyle u^{\delta} h ( u ) ( 1 + ( 1 + \delta ) \abs{\ln h ( u ) } + \abs{\ln u } ) \ge c > 0.
\end{equation}
Next we prove that there exists $\varepsilon \in (0, \infty)$ such that $\forall \, u \in (0 , \varepsilon ) \colon h ( u ) > \varepsilon$. For the sake of contradiction assume that there exists a sequence $(u_n) \downarrow 0$ which satisfies $\lim_{n \to \infty} h (u_n) = 0$. \Nobs that the fact that $\forall\, \eta > 0 \colon \lim_{x \downarrow 0 } x^\eta \abs{\ln(x)} = 0$ ensures that $\lim_{n \to \infty} ( u_n^{\delta} h ( u_n ) ( 1 + ( 1 + \delta ) \abs{\ln(h(u_n))} + \abs{\ln(u_n)})) = 0$. This contradicts to \cref{eqn:lem:log:estimate:help:lowerbound}. Hence, we obtain that there exists $\varepsilon > 0$ which satisfies $\forall \, u \in (0 , \varepsilon ) \colon h ( u ) > \varepsilon$. This implies that $\inf_{u \in (0 , \varepsilon ) } h ( u ) > 0$. Combining this with the fact that $\inf_{u \in [ \varepsilon , 1 ) } h ( u ) \ge g (\varepsilon) > 0$ shows that $\inf_{u \in (0, 1)} h(u) > 0$.
\end{cproof}


We next show some properties of the sums of the $k$ smallest among $N$ random variables with the distribution appearing in \cref{prop:product:normal}. For this we require a few known results about order statistics from \cite{Papadatos2001} and \cite{Bickel1967}.


\cfclear 
\begin{lemma}\label{lem:sum:order:stat}

Let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $N \in \mathbb{N}$ let $X_1^N, \ldots, X_N^N \colon \Omega \to (0, \infty)$ be i.i.d.\ random variables, assume for all $N \in \mathbb{N}$, $x \in ( 0, \infty )$ that $\P (X_1^N \le x) = \frac{2}{\pi} \allowbreak \int_0^x K_0(y) \, \d y$, and for every $N \in \mathbb{N}$ let $X^N_{(1)}, \ldots, X^N_{(N)} \colon \Omega \to (0, \infty)$ be the order statistics\footnote{The random permutation defined by sorting the values of $X^N_i$, $i \in \{1, 2, \ldots, N\}$, in non-decreasing order, i.e., $\forall\, N \in \mathbb{N} \colon X^N_{(1)} \le \cdots \le X^N_{(N)}$.} of $X^N_1, \ldots, X^N_N$ \cfadd{def:bessel}\cfload. Then
\begin{enumerate}[label = (\roman*)]
\item
\label{lem:sum:order:stat:item1} it holds for all $\delta \in (0, \infty)$ that there exists $c \in (0, \infty)$ such that for all $N \in \mathbb{N}$, $k \in \{1, 2, \ldots, N\}$ it holds that
\begin{equation}
\textstyle \mathbb{E} \br[\big]{\sum_{i=1}^k X_{(i)}^N } \ge \frac{c k^{2 + \delta}}{N^{1 + \delta}}
\end{equation}

and
\item
\label{lem:sum:order:stat:item2} it holds that for all $N \in \mathbb{N}$, $k \in \{1, 2, \ldots, N\}$ that
\begin{equation}
\textstyle \operatorname{Var} \rbr[\big]{\sum_{i=1}^k X^N_{(i)}} \le \operatorname{Var} \rbr[\big]{\sum_{i=1}^N X^N_i} = N \operatorname{Var} (X^N_1) \le N.
\end{equation}
\end{enumerate}
\end{lemma}
\begin{cproof}{lem:sum:order:stat}
Throughout this proof let $\delta \in (0, \infty)$ and let $g \colon (0, 1) \to (0, \infty)$ satisfy for all $u \in (0, 1)$ that $g(u) = \inf \cu{x \in (0, \infty) \colon \frac{2}{\pi} \int_0^x K_0(y) \, \d y \ge u}$. \Nobs that, e.g., Papadatos~\cite[Theorem~3.1]{Papadatos2001} implies for all $N \in \mathbb{N}$, $i \in \{1, 2, \ldots, N\}$ that 
\begin{equation} \label{lem:sum:order:stat:eq1}
\textstyle \mathbb{E} [X^N_{(i)}] \ge \frac{N}{i} \int_0^{i / N} g(u) \, \d u.
\end{equation}
Moreover, \cref{prop:product:normal} shows that there exists $c \in (0, \infty)$ which satisfies for all $N \in \mathbb{N}$, $x \in (0, \infty )$ that $\P(X_1^N \le x) \le c x (1 + \abs{\ln(x)})$. Hence, we obtain for all $u \in (0, 1)$ that $c g ( u ) ( 1 + \abs{ \ln(g(u))}) \ge u$. Combining this with \cref{lem:log:estimate:help} assures that there exists $\kappa \in (0, \infty)$ which satisfies $\forall \, u \in (0,1) \colon g ( u ) \ge \kappa u ^{ 1 + \delta }$. This and \cref{lem:sum:order:stat:eq1} imply for all $N \in \mathbb{N}$, $i \in \{1, 2, \ldots, N\}$ that
\begin{equation}
\textstyle \mathbb{E} [X^N_{(i)}] \ge \frac{\kappa N}{i} \int_0^{i/N} u^{1 + \delta} \, \d u = \frac{\kappa}{2 + \delta} \rbr[\big]{\frac{i}{N}}^{1 + \delta}.
\end{equation}
Combining this with the fact that for all $k \in \mathbb{N}$ it holds that $\sum_{i=1}^k i^{1 + \delta} \ge \int_0^k x ^{ 1 + \delta } \, \d x \allowbreak = \frac{1}{2 + \delta } k ^{ 2 + \delta }$ establishes \cref{lem:sum:order:stat:item1}. Next \nobs that for all $N \in \mathbb{N}$, $i, j \in \{1, 2, \ldots, N\}$ it holds that $\operatorname{Cov} ( X^N_{ ( i ) } , X^N_{ ( j ) } ) \ge 0$ (cf., e.g., Bickel~\cite[Theorem 2.1]{Bickel1967}). The assumption that $X^N_i$, $i \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak N\}$, are i.i.d.\ therefore assures for all $N \in \mathbb{N}$, $k \in \{1, 2, \ldots, N\}$ that
\begin{equation}
\begin{split}
\textstyle N \operatorname{Var} ( X_1^N ) & \textstyle =
\operatorname{Var} \rbr[\big]{\sum_{i=1}^N X^N_i} = \operatorname{Var} \rbr[\big]{\sum_{i=1}^N X^N_{(i)}} = \sum_{i, j=1}^N \operatorname{Cov} (X^N_{(i)}, X^N_{(j)}) \\
& \textstyle \ge \sum_{i,j=1}^k \operatorname{Cov} (X^N_{(i)}, X^N_{(j)}) = \operatorname{Var} \rbr[\big]{\sum_{i=1}^k X^N_{(i)}}.
\end{split}
\end{equation}
This and the fact that $\forall \, N \in \mathbb{N} \colon \operatorname{Var} ( X^N_1 ) \le \mathbb{E} [ (X_1 ^N )^2 ] = 1$ establish \cref{lem:sum:order:stat:item2}.
\end{cproof}


\cfclear 
\begin{cor}
\label{cor:sum:order:stat}
Let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $N \in \mathbb{N}$
let $X^N_1, \ldots, X^N_N \colon \Omega \to (0, \infty)$ be i.i.d.\ random variables, assume for all $N \in \mathbb{N}$, $x \in ( 0, \infty )$ that $\P(X^N_1 \le x) = \frac{2}{\pi} \allowbreak \int_0^x K_0(y) \, \d y $, for every $N \in \mathbb{N}$ let $X^N_{(1)}, \ldots, X^N_{(N)} \colon \Omega \to (0, \infty)$ be the order statistics of $X^N_1, \allowbreak \ldots, \allowbreak X^N_N$, and let $\mathfrak{C} \in \mathbb{R}$, $\varepsilon \in (0, 1)$, $\gamma \in (\frac{1 + \varepsilon}{2}, 1)$, $\delta \in (0, \frac{2 \gamma - \varepsilon - 1 }{ 1 - \gamma } ) $ \cfadd{def:bessel}\cfload. Then there exists $c \in \mathbb{R}$ such that for all $N \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P \rbr[\Big]{\sum_{i=1}^{ \lceil N ^\gamma \rceil } X_ { ( i ) } ^N \le \mathfrak{C} N ^{ \varepsilon } } \le c N ^{3 + 2 \delta - 4 \gamma - 2 \gamma \delta } .
	\end{equation}
\end{cor}
\begin{cproof}{cor:sum:order:stat}
First, \cref{lem:sum:order:stat} assures that there exists $\kappa \in (0, \infty )$ which satisfies for all $N \in \mathbb{N}$ that
\begin{equation}
\textstyle \mathbb{E} \br[\Big]{\sum_{i=1}^{\lceil N^\gamma \rceil } X_ { ( i ) } ^N } \ge 2 \kappa \frac{ (N ^\gamma ) ^{ 2 + \delta } }{ N ^{ 1 + \delta}} = 2 \kappa N^{\gamma (2 + \delta) - (1 + \delta)}.
\end{equation}
In addition, the fact that $\gamma (2 + \delta) - (1 + \delta) = (\gamma - 1) \delta + (2 \gamma - 1) > \varepsilon$ proves that there exists $\mathbf{n} \in \mathbb{N}$ which satisfies for all $N \in \mathbb{N} \cap [\mathbf{n}, \infty)$ that $\kappa N^{\gamma (2 + \delta) - (1 + \delta)} > \mathfrak{C} N^\varepsilon$. The Chebyshev's inequality and \cref{lem:sum:order:stat} hence prove that for all $N \in \mathbb{N} \cap [ \mathbf{n} , \infty )$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P \rbr[\Big]{\sum_{i=1}^{\lceil N^\gamma \rceil } X_ { ( i ) } ^N \le \mathfrak{C} N^{\varepsilon}} \le \P \rbr[\Big]{\sum_{i=1}^{\lceil N^\gamma \rceil} X_{(i)}^N \le \mathbb{E} \br[\Big]{\sum_{i=1}^{\lceil N^\gamma \rceil} X_{(i)}^N} - \kappa N^{\gamma (2 + \delta) - (1 + \delta)}} \\
& \textstyle \le \P \rbr[\Big]{\abs[\Big]{\sum_{i=1}^{\lceil N^\gamma \rceil} X_{(i)}^N - \mathbb{E} \br[\Big]{\sum_{i=1}^{\lceil N^\gamma \rceil} X_{(i)}^N}} \ge \kappa N^{\gamma (2 + \delta) - (1 + \delta)}} \\
& \textstyle \le (\kappa N^{\gamma (2 + \delta) - (1 + \delta)})^{-2}
\operatorname{Var} \rbr[\big]{\sum_{i=1}^{\lceil N^\gamma \rceil} X_{(i)}^N} \le (\kappa N^{\gamma (2 + \delta) - (1 + \delta)})^{-2} N
= \kappa^{-2} N^{3 + 2 \delta - 4 \gamma - 2 \gamma \delta}.
\end{split}
\end{equation}
This, the fact that $\forall \, N \in \{1, 2, \ldots, \mathbf{n}\} \colon N ^{3 + 2 \delta - 4 \gamma - 2 \gamma \delta} \ge \min \cu{ 1 , \mathbf{n}^{ 3 + 2 \delta - 4 \gamma - 2 \gamma \delta}}$, and the fact that $\forall \, N \in \mathbb{N} \colon  \P \rbr{\sum_{i=1}^{\lceil N^\gamma \rceil } X_{(i)}^N \le \mathfrak{C} N^{\varepsilon}} \le 1$ demonstrate for all $N \in \mathbb{N}$ that
\begin{equation}
\textstyle \P \rbr[\Big]{\sum_{i=1}^{\lceil N^\gamma \rceil} X_{(i)}^N \le \mathfrak{C} N^{\varepsilon}} \le \max \cu[\big]{1, \mathbf{n}^{- (3 + 2 \delta - 4 \gamma - 2 \gamma \delta)}, \kappa^{-2}} N^{3 + 2 \delta - 4 \gamma - 2 \gamma \delta}.
\end{equation}
\end{cproof}

\begin{lemma}
\label{lem:size:estimate}
Let $(\Omega, \mathcal{F}, \P)$ be a probability space, let $N \in \mathbb{N}$, $\alpha \in \mathbb{R}$, $\varepsilon \in (0, 1)$, $\delta \in (0, \varepsilon)$, let $X_1, \ldots, X_N \colon \Omega \to ( 0 , \infty )$ be random variables, and assume for all $i \in \{1, 2, \ldots, N\}$, $x \in (0, \infty)$ that $\P(N^\alpha X_i \le x) = [\frac{2}{\pi}]^{1/2} \int_0^x \exp(- \frac{y^2}{2}) \, \d y$. Then
\begin{equation}
\textstyle \P \rbr[\big]{\max \cu{X_1, \ldots, X_N} \ge N^{-\alpha + \varepsilon}} \le (\varepsilon - \delta)^{-1} N^{- \delta}.
\end{equation}
\end{lemma}
\begin{cproof}{lem:size:estimate}
Throughout this proof let $\Gamma \colon (0, \infty) \to (0, \infty)$ satisfy for all $x \in (0, \infty)$ that $\Gamma(x) = \int_0^{\infty} t^{x - 1} e^{-t} \, \d t$. First, the well-known formula for absolute moments of the normal distribution (cf., e.g., (18) in Winkelbauer~\cite{Winkelbauer2012}) and the fact that $\forall \, x \in (0 , \infty ) \colon \Gamma (x + 1) \le \max \cu{1 , x^x}$ (cf., e.g., \cite[Proposition~5.3]{JentzenWelti2020}) assure for all $p \in [1, \infty)$ that
\begin{equation}
\begin{split}
\textstyle (\mathbb{E}[(X_1)^p])^{1/p} & \textstyle = N^{-\alpha} (\mathbb{E}[(N^\alpha X_1)^p])^{1/p} = N^{-\alpha} \rbr[\big]{\pi^{- 1/2} 2^{p/2} \Gamma(\frac{p+1}{2})}^{1/p} \\
& \textstyle \le N^{-\alpha} \pi^{- 1/(2 p)} 2^{1/2} \max \cu[\big]{1, (\tfrac{p-1}{2})^{(p-1)/(2 p)}} \\
& \textstyle \le N^{-\alpha} \rbr[\big]{2^{1/2} (\frac{p}{2})^{1/2} \indicator{[3, \infty)} (p) + 2^{1/2} \indicator{[2^{1/2}, 3)} (p) + \pi^{- 2^{- 3/2}} 2^{1/2} \indicator{[1, 2^{1/2})} (p)} \\
& \textstyle \le N^{-\alpha} p. 
\end{split}
\end{equation}
This, Jensen's inequality, and the fact that $(\varepsilon - \delta)^{-1} \in [1 , \infty )$
demonstrate that
\begin{equation}
\begin{split}
& \textstyle \P \rbr[\big]{\max \cu{X_1, \ldots, X_N} \ge N^{- \alpha + \varepsilon}} = \P \rbr[\big]{N^{\alpha - \varepsilon} \max \cu{X_1, \ldots, X_N} \ge 1} \\
& \textstyle \le \mathbb{E} \br[\big]{N^{\alpha - \varepsilon} \max \cu{X_1, \ldots, X_N}} \le N^{\alpha - \varepsilon} \rbr[\big]{\mathbb{E} \br[\big]{(\max \cu{X_1, \ldots, X_N})^{(\varepsilon - \delta)^{-1}}}}^{\varepsilon - \delta} \\
& \textstyle \le N^{\alpha - \varepsilon} \rbr[\big]{\mathbb{E} \br[\big]{(X_1)^{(\varepsilon - \delta)^{-1}} + \cdots + (X_N)^{(\varepsilon - \delta)^{-1}}}}^{\varepsilon - \delta} = N^{\alpha - \varepsilon} \rbr[\big]{N \mathbb{E} \br[\big]{( X_1)^{(\varepsilon - \delta)^{-1}}}}^{\varepsilon - \delta} \\
& \textstyle \le N^{\alpha - \varepsilon} N^{\varepsilon - \delta} N^{- \alpha}(\varepsilon - \delta)^{-1} = (\varepsilon - \delta)^{-1} N^{- \delta}.
\end{split}
\end{equation}
\end{cproof}


\begin{cor}
\label{cor:product:bias:sum}
Let $(\Omega, \mathcal{F}, \P)$ be a probability space, let $\alpha, \beta \in \mathbb{R}$, for every $N \in \mathbb{N}$ let $V_1^N, \ldots, \allowbreak V_N^N, \allowbreak W_1^N, \allowbreak \ldots, W_N^N \colon \Omega \to (0, \infty)$ be random variables, assume for every $N \in \mathbb{N}$ that $V_i^N$, $i \in \{1, 2, \ldots, N\}$, are i.i.d.\ and that $W_i^N$, $i \in \{1, 2, \ldots, N\}$, are i.i.d., assume for all $N \in \mathbb{N}$, $x \in (0, \infty)$ that $\P (N^\alpha V_1^N \le x) = \P (N^\beta W_1^N \le x) = [\frac{2}{\pi}]^{1/2} \int_0^x \exp (- \frac{y^2}{2}) \, \d y$, and let $\mathfrak{C} \in \mathbb{R}$, $\gamma \in (\nicefrac{3}{4}, 1)$, $\eta \in (0 , \min \cu{4 \gamma - 3, \gamma - \frac{1}{2}})$. Then there exists $c \in \mathbb{R}$ which satisfies for all $N \in \mathbb{N}$ that
\begin{equation}
\label{cor:product:bias:sum:eqclaim}
\textstyle \P \rbr[\big]{\exists\, J \subseteq \{1, 2, \ldots, N\} \colon \br[\big]{\# J \ge N^\gamma, \, \sum_{i \in J} V_i^N W_i^N \le \mathfrak{C} \max_{i \in \{1, 2, \ldots, N\}} V_i^N W_i^N}} \le c N^{- \eta}.
\end{equation}
\end{cor}

\cfclear
\begin{cproof}{cor:product:bias:sum}
Throughout this proof for every $N \in \mathbb{N}$, $i \in \{1, 2, \ldots, N\}$ let $X_i^N = V_i^N W_i^N \colon \Omega \to (0, \infty)$ be a random variable, and for every $N \in \mathbb{N}$ let $X_{(1)}^N, \ldots, X_{(N)}^N \colon \Omega \to (0, \infty)$ be the order statistics of $X_1^N, \ldots, X_N^N$. \Nobs that by homogeneity we may assume without loss of generality that $\alpha = \beta = 0$. Moreover, we have for every $N \in \mathbb{N}$ that $X_1^N, \ldots, X_N^N$ are i.i.d.\ random variables. Next, \cref{prop:product:normal} assures for all $N \in \mathbb{N}$, $i \in \{1, 2, \ldots, N\}$, $x \in [0, \infty)$ that $\P(X_i^N \le x) = \frac{2}{\pi} \int_0^x K_0(y) \, \d y$\cfadd{def:bessel} \cfload. Furthermore, the fact that $\gamma > \nicefrac{3}{4}$ and the fact that $\eta < \min \cu{4 \gamma - 3, \gamma - \frac{1}{2}}$ demonstrate that there exist $\varepsilon, \delta \in (0, \infty)$ which satisfy
\begin{equation}
\label{cor:product:bias:sum:eq:defdelta}
\textstyle \varepsilon < 2 \gamma - 1, \quad \varepsilon > 2 \eta, \quad \delta < \frac{2 \gamma - \varepsilon - 1}{1 - \gamma}, \quad \text{and} \quad 3 + 2 \delta - 4 \gamma - 2 \gamma \delta < - \eta.
\end{equation}
In the next step \nobs that the fact that for all $N \in \mathbb{N}$, $J \subseteq \{1, 2, \ldots, N\}$ with $\# J \ge N^\gamma$ it holds that $\sum_{i \in J} V_i^N W_i^N = \sum_{i \in J } X_i^N \ge \sum_{i=1}^{ \lceil N^\gamma \rceil} X_{(i)}^N$ shows that
\begin{equation} \label{cor:product:bias:sum:eq1}
\begin{split}
& \textstyle \P \rbr[\big]{\exists \, J \subseteq \{1, 2, \ldots, N\} \colon \br*{\# J \ge N^\gamma, \, \sum_{i \in J} V_i^N W_i^N \le \mathfrak{C} \max_{i \in \{1, 2, \ldots, N\}} V_i^N W_i^N}} \\
& \textstyle \le \P \rbr*{\sum_{i=1}^{ \lceil N^\gamma \rceil} X_{(i)}^N \le \mathfrak{C} X_{(N)}^N} \le \P \rbr*{\sum_{i=1}^{\lceil N^\gamma \rceil} X_{(i)}^N \le \mathfrak{C} N^\varepsilon} + \P \rbr[\big]{X_{(N)}^N \ge N^\varepsilon}.
\end{split}
\end{equation}
Next, \cref{lem:size:estimate} (applied with $\alpha \with 0 $, $\varepsilon \with \frac{\varepsilon}{2}$, $\delta \with \eta$ in the notation of \cref{lem:size:estimate}) and \cref{cor:product:bias:sum:eq:defdelta} ensure that there exists $c \in \mathbb{R}$ which satisfies for all $N \in \mathbb{N}$ that
\begin{equation}
\begin{split}
& \textstyle \P \rbr[\big]{X_{(N)}^N \ge N^{\varepsilon}} \\
& \textstyle \le \P \rbr[\big]{\max_{i \in \{1, 2, \ldots, N\}} V_i^N \ge N^{\nicefrac{\varepsilon}{2}}} + \P \rbr[\big]{\max_{i \in \{1, 2, \ldots, N\}} W_i^N \ge N^{\nicefrac{\varepsilon}{2}}} \le c N^{- \eta}.
\end{split}
\end{equation}
Combining this with \cref{cor:sum:order:stat} assures that there exists $\kappa \in \mathbb{R}$ which satisfies for all $N \in \mathbb{N}$ that
\begin{equation}
\textstyle \P \rbr[\Big]{\textstyle\sum_{i = 1}^{\lceil N^\gamma \rceil} X_{(i)}^N \le \mathfrak{C} N^\varepsilon} + \P \rbr[\big]{X_{(N)}^N \ge N^\varepsilon} \le \kappa N^{3 + 2 \delta - 4 \gamma - 2 \gamma \delta} + c N^{- \eta}.
\end{equation}
This, \cref{cor:product:bias:sum:eq:defdelta}, and \cref{cor:product:bias:sum:eq1} establish \cref{cor:product:bias:sum:eqclaim}.
\end{cproof}

\subsubsection{Properties of random ANN initializations}

Now we are in a position to establish in \cref{lem:random:init} some important properties of the random initializations we consider. In the notation of \cref{lem:random:init}
the random variables $\Theta^{\mathfrak{h} , i }$, $1 \le i \le \mathfrak{h}$, describe the inner biases,
the random variables $\Theta^{\mathfrak{h} , \mathfrak{h} + i }$, $1 \le i \le \mathfrak{h}$, describe the inner weights,
and $\Theta^{\mathfrak{h} , 2 \mathfrak{h} + i }$, $1 \le i \le \mathfrak{h}$,
describe the outer weights.
While the biases are normally distributed, the weights are half-normally distributed with appropriate scaling in the width $\mathfrak{h}$.


\begin{lemma}
\label{lem:random:init}
Let $\scra \in \mathbb{R}$, $\scrb \in (\scra, \infty)$, $\alpha \in (\nicefrac{3}{4}, 1)$, $\beta \in ( \alpha + 2 , \infty )$, $\gamma \in (\nicefrac{3}{4} , \alpha )$, for every $\mathfrak{h} \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$, $\theta = ( \theta_1, \ldots, \theta_{3 \mathfrak{h} } ) \in \mathbb{R}^{ 3 \mathfrak{h} }$ let $I_i^\theta \subseteq \mathbb{R}$ satisfy $I_i^\theta = \cu{x \in ( \scra , \scrb ) \colon \theta_{\mathfrak{h} + i } x + \theta_{ i } \in (0, 1 )}$, let $\Theta^\mathfrak{h} = (\Theta^{\mathfrak{h}, 1}, \ldots, \Theta^{ \mathfrak{h} , 3 \mathfrak{h} } ) \colon \Omega \to \mathbb{R}^{ 3 \mathfrak{h} }$, $\mathfrak{h} \in \mathbb{N}$, be random variables, assume for all $\mathfrak{h} \in \mathbb{N}$ that $\Theta^{ \mathfrak{h} , 1 } , \ldots, \Theta^{ \mathfrak{h} , 3 \mathfrak{h} }$ are independent, assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \{1, 2, \ldots, \mathfrak{h}\}$ that $\mathfrak{h}^{- \beta} \Theta^{ \mathfrak{h} , k }$ is standard normal, and assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \{1, 2, \ldots, \mathfrak{h}\}$, $x \in (0, \infty)$ that $\P(\mathfrak{h}^{- \beta} \Theta^{\mathfrak{h}, \mathfrak{h} + k} \leq x) = \P(\mathfrak{h}^{\alpha} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + k} \le x) = [\frac{2}{\pi}]^{1/2} \int_0^x \exp(- \frac{y^2}{2}) \, \d y$. Then
\begin{enumerate} [ label = (\roman*)]
\item
\label{lem:random:init:item1} there exist $c_1 \in (0, 1)$, $c_2 \in (c_1, 1)$ such that for all $\mathfrak{h} \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$ it holds that $c_1 \le \P(I_i^{\Theta^\mathfrak{h}} \neq \emptyset) \le c_2$,

\item
\label{lem:random:init:item2} for all $\delta \in (0, \frac{\alpha - \gamma}{2})$ there exists $c_3 \in \mathbb{R}$ such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P \rbr[\big]{\max_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{ \mathfrak{h} , 2 \mathfrak{h} + i } \le \mathfrak{h} ^{ - \gamma - \delta } } > 1 - c_3 \mathfrak{h} ^{ - \delta },
\end{equation}

\item
\label{lem:random:init:itemnew} for all $\mathfrak{C} \in \mathbb{R}$, $\delta \in (0, \frac{\alpha - \gamma }{2} )$, $\eta \in (0 , \min \cu{ 4 \gamma - 3 , \gamma - \tfrac{1}{2} } )$ there exists $c_4 \in \mathbb{R}$ such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{multline}
\textstyle \P\big(\exists\, J \subseteq \{1, 2, \ldots, \mathfrak{h}\} \colon
\big[\rbr[\big]{\sum_{i \in J} \Theta^{\mathfrak{h}, \mathfrak{h} + i} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \le \mathfrak{C} \max_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{\mathfrak{h}, \mathfrak{h} + i} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i}} \\
\textstyle \wedge \rbr[\big]{\sum_{i \in J} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \ge \mathfrak{h}^{- \delta}}\big]\big) \le c_4 \mathfrak{h}^{- \min \cu{\delta, \eta}},
\end{multline}

\item
\label{lem:random:init:item3} for all $q \in [\alpha^{-1} , \infty)$ there exists $c_5 \in \mathbb{R}$ such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \rbr[\Big]{\mathbb{E} \br[\Big]{\abs[\big]{\sum_{i=1}^\mathfrak{h} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i}}^{2 q}}}^{\nicefrac{1}{q}} \le c_5 \mathfrak{h}^{2 - 2 \alpha},
\end{equation}

\item
\label{lem:random:init:item4} for all $\kappa \in (0, \infty)$ there exists $c_6 \in (0, \infty )$ such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P \rbr*{\sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, I_i^{\Theta ^\mathfrak{h} } \neq \emptyset} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \le \kappa } \le \frac{c_6}{\mathfrak{h}},
\end{equation}

and
\item 
\label{lem:random:init:item5} for all $\rho \in (0, \infty)$ there exists $c_7 \in \mathbb{R}$ such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $\P \rbr{\min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \allowbreak \Theta^{\mathfrak{h}, \mathfrak{h} + i} \allowbreak \ge \allowbreak \rho} \ge 1 - c_7 \mathfrak{h}^{1 - \beta}$ and $\P \rbr{\min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{\mathfrak{h}, \mathfrak{h} + i} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \ge \rho} \ge 1 - c_7 \mathfrak{h}^{(2 - \beta + \alpha)/2}$.
\end{enumerate}
\end{lemma}
\begin{cproof}{lem:random:init}
First, the fact that the distribution of $(\mathfrak{h}^{- \beta} \Theta^{\mathfrak{h}, i}, \mathfrak{h}^{- \beta} \Theta^{\mathfrak{h}, \mathfrak{h} + i}) \colon \Omega \to \allowbreak [0, \infty) \allowbreak \times \mathbb{R}$, $\mathfrak{h} \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$, is absolutely continuous with a positive density and independent of $\mathfrak{h} \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$ and the fact that for all $\mathfrak{h} \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$ it holds that
\begin{equation}
\begin{split}
 \textstyle 0 & < \P \rbr[\big]{ \mathfrak{h} ^{ - \beta } ( \scra \Theta^{ \mathfrak{h} , \mathfrak{h} + i } + \Theta^{ \mathfrak{h} , i } ) < 0 < \mathfrak{h} ^{ - \beta } ( \scrb \Theta^{ \mathfrak{h} , \mathfrak{h} + i } + \Theta^{ \mathfrak{h} , i } ) } \\
 & \le \textstyle \P (I_i^{ \Theta^\mathfrak{h}} \neq \emptyset) \le 1 -  \P \rbr[\big]{ \mathfrak{h} ^{ - \beta } ( \scrb \Theta^{ \mathfrak{h} , \mathfrak{h} + i } + \Theta^{ \mathfrak{h} , i } < 0)} < 1
\end{split}
\end{equation}
establish \cref{lem:random:init:item1}. In the next step \nobs that \cref{lem:size:estimate} (applied for every $\mathfrak{h} \in \mathbb{N}$, $\delta \in (0, \frac{\alpha - \gamma }{2} )$ with $N \with \mathfrak{h}$, $\scra \with \alpha$, $\varepsilon \with \alpha - \gamma - \delta$, $\delta \with \delta$ in the notation of \cref{lem:size:estimate}) shows for all $\delta \in (0, \frac{\alpha - \gamma}{2})$, $\mathfrak{h} \in \mathbb{N}$ that
\begin{equation}
\textstyle \P \rbr[\big]{ \max\nolimits_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{ \mathfrak{h} , 2 \mathfrak{h} + i } \ge \mathfrak{h} ^{- \gamma - \delta } } \le (\alpha - \gamma - 2 \delta ) ^{-1} \mathfrak{h}^{- \delta}.
\end{equation}
This establishes \cref{lem:random:init:item2}. Next \nobs that for all $\mathfrak{h} \in \mathbb{N}$, $J \subseteq \{1, 2, \ldots, \mathfrak{h}\}$, $\omega \in \Omega$ it holds that $\# J \ge \mathfrak{h}^\gamma$ or $\sum_{j \in J } \Theta^{ \mathfrak{h} , 2 \mathfrak{h} + j } ( \omega ) \le \mathfrak{h} ^\gamma \max_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{ \mathfrak{h} , 2 \mathfrak{h} + i } ( \omega)$. Hence, we obtain for all $\mathfrak{h} \in \mathbb{N}$, $\mathfrak{C} \in \mathbb{R}$, $\delta \in (0, \frac{\alpha - \gamma}{2})$ that
\begin{equation}
\begin{split}
& \textstyle \P \rbr[\big]{\exists\, J \subseteq \{1, 2, \ldots, \mathfrak{h}\} \colon \textstyle \br[\big]{\sum_{i \in J } \Theta^{\mathfrak{h}, \mathfrak{h} + i} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i } \le \mathfrak{C} \max_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{ \mathfrak{h} , \mathfrak{h} + i } \Theta^{\mathfrak{h} , 2 \mathfrak{h} + i } } \\
& \textstyle \qquad  \wedge \br[\big]{\sum_{i \in J} \Theta^{\mathfrak{h} , 2 \mathfrak{h} + i } \ge \mathfrak{h} ^{ - \delta } } } \\
& \textstyle \le \P \rbr[\big]{\exists\, J \subseteq \{1, 2, \ldots, \mathfrak{h}\} \colon \br[\big]{\# J \ge \mathfrak{h}^\gamma} \wedge \\
& \textstyle \qquad  \br[\big]{\sum_{i \in J} \Theta^{ \mathfrak{h} , \mathfrak{h} + i } \Theta^{ \mathfrak{h} , 2 \mathfrak{h} + i} \le \mathfrak{C} \max_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{\mathfrak{h}, \mathfrak{h} + i} \Theta^{ \mathfrak{h} , 2 \mathfrak{h} + i } } }  \\
& \textstyle + \P \rbr[\big]{\max_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \ge \mathfrak{h}^{- \gamma - \delta }}.
\end{split}
\end{equation}
Combining this with \cref{lem:random:init:item2} and \cref{cor:product:bias:sum} demonstrates that for all $\mathfrak{C} \in \mathbb{R}$, $\delta \in (0 , \frac{ \alpha - \gamma}{2} )$, $\eta \in (0, \min \cu{4 \gamma - 3, \gamma - \tfrac{1}{2}, \delta})$ there exist $c_3, c_4 \in \mathbb{R}$ which satisfy for all $\mathfrak{h} \in \mathbb{N}$ that
\begin{multline}
\textstyle \P \big(\exists\, J \subseteq \{1, 2, \ldots, \mathfrak{h}\} \colon \br[\big]{\sum_{i \in J} \Theta^{ \mathfrak{h} , \mathfrak{h} + i } \Theta^{ \mathfrak{h} , 2 \mathfrak{h} + i } \le \mathfrak{C} \max\nolimits_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{ \mathfrak{h} , \mathfrak{h} + i } \Theta^{ \mathfrak{h} , 2 \mathfrak{h} + i }  } \\ 
\textstyle \wedge \br[\big]{\sum_{i \in J} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \ge \mathfrak{h} ^{ - \delta}}\big) \le c_4 \mathfrak{h}^{- \eta} + c_3 \mathfrak{h}^{- \delta} \le (c_3 + c_4) \mathfrak{h}^{- \min \cu{\delta , \eta}}.
\end{multline}
This establishes \cref{lem:random:init:itemnew}. Next, by the well-known formula for absolute moments of the normal distribution (cf., e.g., (18) in Winkelbauer~\cite{Winkelbauer2012}), for every $q \in [1, \infty )$ there exists $\mathfrak{C}_q \in (0, \infty )$ which satisfies for all $\mathfrak{h} \in \mathbb{N}$ that $\mathbb{E} [ ( \Theta^{ \mathfrak{h} , 2 \mathfrak{h} + 1 } ) ^{ 2 q } ] = \mathfrak{C}_q \mathfrak{h}^{- 2 q \alpha}$. Combining this with the fact that for all $q \in [1, \infty)$ it holds that $[0, \infty) \ni x \mapsto x^{2q} \in [0, \infty)$ is convex and Jensen's inequality assures that for all $q \in [1, \infty)$, $\mathfrak{h} \in \mathbb{N}$ we have that
\begin{equation}
\textstyle \mathbb{E}\br*{\abs[\big]{\sum_{i=1}^\mathfrak{h} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i}}^{2 q}} \le \mathbb{E}\br*{\mathfrak{h}^{2 q - 1} \sum_{i = 1}^\mathfrak{h} (\Theta^{\mathfrak{h}, 2 \mathfrak{h} + i})^{2 q}} = \mathfrak{h}^{2 q} \mathbb{E}\br[\big]{(\Theta^{\mathfrak{h}, 2 \mathfrak{h} + 1})^{2 q}} = \mathfrak{C}_q \mathfrak{h}^{2 q (1 - \alpha)}.
\end{equation}
This establishes \cref{lem:random:init:item3}. For every $\mathfrak{h} \in \mathbb{N}$ let $X_i^\mathfrak{h} \colon \Omega \to \mathbb{R}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$, and $Z^\mathfrak{h} \colon \Omega \to \mathbb{R}$ be random variables which satisfy for all $i \in \{1, 2, \ldots, \mathfrak{h}\}$ that \begin{equation}
\textstyle X_i^\mathfrak{h} (\omega) =
\begin{cases}
\Theta^{\mathfrak{h}, 2 \mathfrak{h} + i}(\omega) & \colon I_i^{\Theta^\mathfrak{h}} \neq \emptyset  \\
0 & \colon \text{else}
\end{cases}
\end{equation}
and $Z^\mathfrak{h} = \sum_{i=1}^\mathfrak{h} X_i$. \Nobs that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $X_i^\mathfrak{h}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$, are i.i.d.\ random variables. Furthermore, \cref{lem:random:init:item1} and the fact that for all $\mathfrak{h} \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$ it holds that there exists a standard normal random variable $U$ such that $\Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \sim \mathfrak{h}^{- \alpha} \abs{U}$ imply that for all $\mathfrak{h} \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$ we have that
\begin{equation}
\textstyle \mathbb{E}[X_i^\mathfrak{h}] = \mathbb{E}[\Theta^{\mathfrak{h}, 2 \mathfrak{h} + i}] \, \P(I_i^{\Theta^\mathfrak{h}} \neq \emptyset) \ge c_1 \mathfrak{h}^{- \alpha} [\frac{2}{\pi}]^{1/2} \qquad \text{and} \qquad \operatorname{Var}(X_i^\mathfrak{h}) \le \mathbb{E}[(X_i^{\mathfrak{h}})^2] \le \mathfrak{h}^{-2 \alpha}.
\end{equation}
Hence, we obtain that there exists $\lambda \in (0, \infty)$ which satisfy for all $\mathfrak{h} \in \mathbb{N}$ that $\mathbb{E}[Z^\mathfrak{h}] \ge \lambda \mathfrak{h}^{1 - \alpha}$ and $\operatorname{Var}(Z^\mathfrak{h}) \le \mathfrak{h}^{1 - 2 \alpha}$. Combining this with Chebyshev's inequality assures for all $\kappa \in (0, \infty)$, $\mathfrak{h} \in \mathbb{N} \cap ((2 \kappa)^{(1 - \alpha)^{-1}} \lambda^{- (1 - \alpha)^{-1}}, \infty)$ that $\lambda \mathfrak{h}^{1 - \alpha} - \kappa \ge \frac{\lambda \mathfrak{h}^{- \alpha}}{2}$ and, therefore,
\begin{equation}
\begin{split}
\textstyle \P(Z^\mathfrak{h} \le \kappa) & \textstyle = \P\rbr[\big]{\lambda \mathfrak{h}^{1 - \alpha} - Z^\mathfrak{h} \ge \lambda \mathfrak{h}^{1 - \alpha} - \kappa} \le \P\rbr[\big]{\mathbb{E}[Z^\mathfrak{h}] - Z^\mathfrak{h} \ge \lambda \mathfrak{h}^{1 - \alpha} - \kappa} \\
& \textstyle \le \P\rbr[\big]{\abs{Z^\mathfrak{h} - \mathbb{E}[Z^\mathfrak{h}]} \ge \frac{\lambda \mathfrak{h}^{- \alpha}}{2}} \le \frac{4 \mathfrak{h}^{1 - 2 \alpha}}{\lambda^2 \mathfrak{h}^{- 2 \alpha}} = \frac{4}{\lambda^2 \mathfrak{h}}.
\end{split}
\end{equation}
This and the fact that for all $\kappa \in (0, \infty)$, $\mathfrak{h} \in \mathbb{N} \cap [1, (2 \kappa)^{(1 - \alpha)^{- 1}} \lambda^{- (1 - \alpha)^{-1}}]$ it holds that $\P(Z^\mathfrak{h} \le \kappa) \le 1 \le (\frac{2 \kappa}{\lambda})^{(1 - \alpha)^{- 1}} \mathfrak{h}^{- 1}$ demonstrate that there exists $c_5 \in \mathbb{R}$ which satisfies for all $\kappa \in (0, \infty )$, $\mathfrak{h} \in \mathbb{N}$ that $\P(Z^\mathfrak{h} \le \kappa ) \le c_5 \mathfrak{h}^{- 1}$. This establishes \cref{lem:random:init:item4}.

Next, the fact that for all $\mathfrak{h} \in \mathbb{N}$, $x \in [0, \infty)$ it holds that $\P(\Theta^{\mathfrak{h}, \mathfrak{h} + 1} \le x) = \P(\mathfrak{h}^{- \beta} \Theta^{\mathfrak{h}, 1} \le \mathfrak{h}^{- \beta} x) \allowbreak = \allowbreak [\frac{2}{\pi}]^{1/2} \int_{0}^{\mathfrak{h}^{- \beta} x} \exp(- \frac{y^2}{2}) \, \d y \le \mathfrak{h}^{- \beta} x$ and $\P(\Theta^{\mathfrak{h}, 2 \mathfrak{h} + 1} \le x) \le \mathfrak{h}^{\alpha} x$, the fact that for all $\rho \in (0, \allowbreak \infty), \allowbreak \mathfrak{h} \allowbreak \in \allowbreak \mathbb{N} \allowbreak \cap [\rho^{(\beta - \alpha)^{-1}}, \infty)$ it holds that $\rho^{1/2} \mathfrak{h}^{- \beta} \mathfrak{h}^{(\alpha + \beta)/2} \le 1$, and Bernoulli's inequality establish that for all $\rho \allowbreak \in \allowbreak (0, \allowbreak \infty)$, $\mathfrak{h} \in \mathbb{N} \cap [\rho^{(\beta - \alpha)^{- 1}}, \infty)$ we have that
\begin{equation}
\begin{split}
& \textstyle \P \rbr[\big]{\min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{\mathfrak{h}, \mathfrak{h} + i} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \ge \rho} \\
& \textstyle = \P \rbr[\big]{\forall\, i \in \{1, 2, \ldots, \mathfrak{h}\} \colon \Theta^{\mathfrak{h}, \mathfrak{h} + i} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \ge \rho} = \br[\big]{\P\rbr[\big]{\Theta^{\mathfrak{h}, \mathfrak{h} +  1} \Theta^{\mathfrak{h}, 2 \mathfrak{h} + 1} \ge \rho}}^\mathfrak{h} \\
& \textstyle \ge \br[\big]{\P \rbr[\big]{\Theta^{\mathfrak{h}, \mathfrak{h} + 1} \ge \rho^{1/2} \mathfrak{h}^{(\alpha + \beta)/2}} \P \rbr[\big]{\Theta^{\mathfrak{h}, 2 \mathfrak{h} + 1} \ge \rho^{1/2} \mathfrak{h}^{- (\alpha + \beta)/2}}}^\mathfrak{h} \\
& \textstyle \ge \rbr[\big]{1 - \rho^{1/2} \mathfrak{h}^{- \beta} \mathfrak{h}^{(\alpha + \beta)/2}}^\mathfrak{h} \rbr[\big]{1 - \rho^{1/2} \mathfrak{h}^{\alpha} \mathfrak{h}^{- (\alpha + \beta)/2}}^\mathfrak{h} \\
& \textstyle = \rbr[\big]{1 - \rho^{1/2} \mathfrak{h}^{(\alpha - \beta)/2}}^{2 \mathfrak{h}} \ge \rbr[\big]{1 - \rho^{1/2} \mathfrak{h}^{(2 + \alpha - \beta)/2}}^2 \ge 1 - 2 \rho^{1/2} \mathfrak{h}^{(2 + \alpha - \beta)/2}.
\end{split}
\end{equation}
This and the fact that for all $\rho \in (0, \infty)$ there exists $c_\rho \in \mathbb{R}$ which satisfies for all $\mathfrak{h} \in \mathbb{N} \cap (0, \allowbreak \rho^{(\beta - \alpha)^{- 1}}]$ that $c_\rho \mathfrak{h}^{(2 + \alpha - \beta)/2} \ge 1$ demonstrate for all $\rho \in (0, \infty)$, $\mathfrak{h} \in \mathbb{N}$ that $\P \rbr[\big]{\min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \allowbreak \Theta^{\mathfrak{h}, i} \allowbreak \Theta^{\mathfrak{h}, 2 \mathfrak{h} + i} \allowbreak \ge \rho} \ge 1 - \max \cu{2 \rho^{1/2}, c_\rho} \mathfrak{h}^{(2 + \alpha - \beta)/2}$. In the same way, we can use Bernoulli's inequality to show that for every $\rho \in (0, \infty)$ there exists $\bfc \in \mathbb{R}$ which satisfies for all $\mathfrak{h} \in \mathbb{N}$ that $\P \rbr{\min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} \Theta^{\mathfrak{h}, \mathfrak{h} + i} \ge \rho} \ge 1 - \bfc \mathfrak{h}^{ 1 - \beta}$. This establishes \cref{lem:random:init:item5}.
\end{cproof}

\subsubsection{Almost sure non-convergence of GF to descending critical points}

Our final building block is the non-convergence of randomly initialized gradient flows to descending critical points in the sense of \cref{def:strict_saddle}.
For this we use \cref{prop:non-convergence_to_strict_saddle} below, which is well-known in the scientific literature.

\cfclear
\begin{proposition}\label{prop:non-convergence_to_strict_saddle}
	Let $\mathfrak{h} \in \mathbb{N}$, $f \in C^2 (\mathbb{R}^{\mathfrak{h}}, [0, \infty))$, for every $x \in \mathbb{R}^{\mathfrak{h}}$ let $X^x = (X_t^x)_{t \in [0, \infty)} \in C([0, \infty), \mathbb{R}^{\mathfrak{h}})$ satisfy for all $t \in [0, \infty)$ that 
	\begin{equation}\label{eqn:prop:non-convergence_to_strict_saddle:gf}
	\textstyle X_0^x = x \qquad \text{and} \qquad X_t^x = X_0^x - \int_0^{t} (\nabla f)(X_s^x) \, \d s,
	\end{equation}
	%
	and let $\lambda \colon \mathcal{B}(\mathbb{R}^{\mathfrak{h}}) \to [0, \infty]$ be the Lebesgue--Borel measure on $\mathbb{R}^{\mathfrak{h}}$. Then
	\begin{equation}\label{eqn:prop:non-convergence_to_strict_sadd:zero_set}
	\textstyle \lambda \bigl(\bigl\{x \in \mathbb{R}^{\mathfrak{h}} \colon \bigl[\exists\, y, v \in \mathbb{R}^{\mathfrak{h}} \colon (\limsup_{t \to \infty} \norm{X_t^x - y} = 0  > \scalar{v, ((\operatorname{Hess} f)(y)) v})\bigr]\bigr\}\bigr) = 0.
	\end{equation}
\end{proposition}
\begin{cproof}{prop:non-convergence_to_strict_saddle}
	\Nobs that Bah et al.~\cite[Theorem~28]{BahRauhutTerstiege2021} (applied with $\mathcal{M} \with \mathbb{R}^{\mathfrak{h}}$, $L \with f$ in the notation of \cite[Theorem~28]{BahRauhutTerstiege2021}) establishes \cref{eqn:prop:non-convergence_to_strict_sadd:zero_set}.
\end{cproof}

We also refer to Lee et al.~\cite{MR3960812} for a similar result with discrete-time GD.
We now formulate a corollary for the case where only some parameters are trained and the remaining ones are initialized randomly and independently, which is suitable for our ANN setting with untrained weights. In combination with \cref{cor:non:strict:saddle} this will allow us to estimate the risk of the critical limit point.


\cfclear 
\begin{cor}
\label{cor:strict:saddle:conv:random}
Let $\mathfrak{n} , \mathfrak{h} \in \mathbb{N}$,
$f \in C ( \mathbb{R}^{ \mathfrak{n} } \times \mathbb{R}^{ \mathfrak{h} } , [ 0 , \infty ) )$,
let $\mathcal{G} \colon \mathbb{R}^\mathfrak{n} \times \mathbb{R}^\mathfrak{h} \to \mathbb{R}^\mathfrak{h}$ be measurable and satisfy for all $x \in \mathbb{R}^\mathfrak{n}$,
$y \in \cu{z \in \mathbb{R}^\mathfrak{h} \colon f ( x , \cdot ) \text{ is differentiable at } z}$
that $\mathcal{G} ( x , y ) = (\nabla _y f) ( x , y )$,
let $\lambda \colon \mathcal{B}(\mathbb{R}^{\mathfrak{h}}) \to [0, \infty]$ be the Lebesgue--Borel measure on $\mathbb{R}^{\mathfrak{n}}$,
let $A \subseteq \mathbb{R}^\mathfrak{n}$ be measurable and satisfy $\lambda ( \mathbb{R}^\mathfrak{n} \backslash A ) = 0$,
assume for all $x \in A$ that $f ( x , \cdot ) \in C^2 ( \mathbb{R}^\mathfrak{h} , \mathbb{R} )$,
let $\mathcal{S}_x \subseteq \mathbb{R}^\mathfrak{h}$, $x \in \mathbb{R}^\mathfrak{n}$,
satisfy
for every $x \in A$
that $\mathcal{S}_x = \cu{y \in \mathbb{R}^\mathfrak{h} \colon y \text{ is a \cfadd{def:strict_saddle}\text{descending } critical point of } f(x, \cdot ) }$\cfadd{def:strict_saddle}, let $(\Omega , \mathcal{F} , \P)$ be a probability space,
let $X \colon [0 , \infty ) \times \Omega \to \mathbb{R}^\mathfrak{n} \times \mathbb{R}^\mathfrak{h}$ and $Y \colon [0 , \infty ) \times \Omega \to \mathbb{R}^\mathfrak{n} \times \mathbb{R}^\mathfrak{h}$ be stochastic processes with continuous sample paths,
assume that $X_0 \colon \Omega \to \mathbb{R}^\mathfrak{n}$ and $Y_0 \colon \Omega \to \mathbb{R}^\mathfrak{h}$ are independent random variables,
assume that the distribution of $Y_0$ is absolutely continuous with respect to the Lebesgue--Borel measure on $\mathbb{R}^\mathfrak{h}$,
assume that $\P ( X_0 \in A  ) = 1$,
and assume for all $t \in [0 , \infty )$, $\omega \in \Omega $
that
\begin{equation}
\label{cor:strict:saddle:conv:random:eq1}
\textstyle
X_t ( \omega ) = X_0 ( \omega ) \qquad \text{and} \qquad Y_t ( \omega ) = Y_0 ( \omega ) - \int_0^t \mathcal{G} ( X_s ( \omega ) , Y_s ( \omega ) ) \, \d s\ifnocf.
\end{equation}
\cfload. Then
\begin{equation}
\label{cor:strict:saddle:conv:random:eqclaim}
\textstyle
\P \rbr*{ \exists \, ( x , y ) \in \mathbb{R}^\mathfrak{n} \times \mathbb{R}^\mathfrak{h} \colon \br*{ \lim_{t \to \infty} X_t = x }
\wedge \br*{ \lim_{t \to \infty} Y_t = y }
\wedge \br*{ y \in \mathcal{S}_x } } = 0 .
\end{equation}
\end{cor}
\begin{cproof}{cor:strict:saddle:conv:random}
First \nobs that \cref{cor:strict:saddle:conv:random:eq1} and the fact that $\P ( X_0 \in A ) = 1$ ensure that
\begin{equation}
\begin{split}
& \textstyle \P \rbr*{ \exists \, ( x , y ) \in \mathbb{R}^\mathfrak{n} \times \mathbb{R}^\mathfrak{h} \colon \br*{ \lim_{t \to \infty} X_t = x }
\wedge \br*{ \lim_{t \to \infty} Y_t = y }
\wedge \br*{ y \in \mathcal{S}_x } } \\
& \textstyle = \P \rbr*{ \br*{ X_0 \in A } \wedge \br*{\exists \, y \in \mathcal{S}_{ X_0 } \colon \lim_{t \to \infty} Y_t = y  } } .
\end{split}
\end{equation}
Furthermore, the fact that for all $x \in A$ it holds that $f ( x , \cdot ) \in C^2 ( \mathbb{R}^\mathfrak{h} , \mathbb{R} )$,
the fact that for all $\omega \in \Omega$,
$t \in [0 , \infty )$ with $X_0 ( \omega ) \in A$
it holds that
$Y_t ( \omega ) = Y_0 ( \omega ) - \int_0^t \nabla_y f ( X_0 ( \omega ) , Y_s ( \omega ) ) \, \d s$,
and the Picard-Lindelöf uniqueness theorem for ODEs demonstrate that 
there exists a measurable function $\Phi \colon \mathbb{R}^\mathfrak{n} \times \mathbb{R}^\mathfrak{h} \to C( [ 0 , \infty ) , \mathbb{R}^\mathfrak{h} )$
which satisfies for all $\omega \in \Omega$, $t \in [0 , \infty )$ with $X_0 ( \omega ) \in A$
that $Y_t ( \omega ) = \Phi_t ( X_0 ( \omega ) , Y_0 ( \omega ) )$.
In particular, there exists a measurable function $F \colon \mathbb{R}^\mathfrak{n} \times \mathbb{R}^\mathfrak{h} \to \mathbb{R}$ which satisfies
\begin{equation}
F ( X_0, Y_0 ) = \begin{cases}
1 & \colon \br*{ X_0 \in A } \wedge \br*{\exists \, y \in \mathcal{S}_{ X_0 } \colon \lim_{t \to \infty} Y_t = y  } \\
0 & \colon \text{else.}
\end{cases}
\end{equation}
In addition, \cref{prop:non-convergence_to_strict_saddle} and the assumption that the distribution of $Y_0$ is absolutely continuous with respect to the Lebesgue measure on $\mathbb{R}^\mathfrak{h}$ show for all $x \in A$ that
\begin{equation}
\textstyle
\mathbb{E} [ F ( x , Y_0 ) ] = \P \rbr*{ \exists \, y \in \mathcal{S}_x \colon \lim_{t \to \infty} \Phi_t ( x , Y_0 ) = y } = 0.
\end{equation}
Combining this with the fact that $X_0$ and $Y_0$ are independent and Lemma 2.8 in the \href{https://arxiv.org/abs/1801.09324}{arXiv-version} of \cite{JentzenKuckuckNeufeldVonWurstemberger2021}
implies that
\begin{equation}
\textstyle \P \rbr*{ \br*{ X_0 \in A } \wedge \br*{\exists \, y \in \mathcal{S}_{ X_0 } \colon \lim_{t \to \infty} Y_t = y  } } = \mathbb{E} [ F ( X_0 , Y_0 ) ] = \mathbb{E} \br*{ \indicator{A} ( X_0 ) F ( X_0 , Y_0 ) } = 0.
\end{equation}
\end{cproof}





\subsubsection{Convergence of GF with random initialization}
\label{sec:convergence_of_GF_with_random_init_clipping}


In \cref{theo:main:conv} we establish the convergence of GF with random initializations, which is one of the main results of this article. \cref{theorem_intro} in the introduction is a direct consequence of \cref{theo:main:conv}.
 For convenience of the reader, \cref{theo:main:conv} is formulated in a self-contained way.

\cfclear
\begin{theorem} \label{theo:main:conv}
Let $\scra \in \mathbb{R}$, $\scrb \in (\scra , \infty)$, let $f, \mathfrak{p} \in C(\mathbb{R} , \mathbb{R})$ be piecewise polynomial\cfadd{def:piecewise_polynomial}, assume for all $x \in \mathbb{R}$ that $\mathfrak{p}(x) \ge 0$ and $\mathfrak{p}^{- 1}(\mathbb{R} \backslash \{0\}) = (\scra, \scrb)$, assume that $f$ is strictly increasing, let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $\mathfrak{h} \in \mathbb{N}$, $\theta = ( \theta_1, \ldots, \theta_{\mathfrak{d}_\mathfrak{h} } ) \in \mathbb{R}^{\mathfrak{d}_\mathfrak{h}}$ let $\mathfrak{d} _\mathfrak{h} = 3 \mathfrak{h} + 1$, let $\mathfrak{N}^{ \theta}_\mathfrak{h} \colon \mathbb{R} \to \mathbb{R}$
satisfy for all $x \in \mathbb{R}$ that
\begin{equation}
\textstyle \mathfrak{N}^{ \theta }_\mathfrak{h} (x) = \theta_{\mathfrak{d}_\mathfrak{h}} + \sum_{i = 1}^{\mathfrak{h}} \theta_{2 \mathfrak{h} + i} \max \cu{\min\cu{\theta_{\mathfrak{h} + i } x + \theta_{ i }, 1}, 0},
\end{equation}
let $\mathcal{L}_\mathfrak{h} \colon \mathbb{R}^{ \mathfrak{d}_\mathfrak{h} } \to \mathbb{R}$ satisfy
\begin{equation} \label{theo:main:conv:eq:risk}
\textstyle \mathcal{L}_{\mathfrak{h}}(\theta) = \int_{\mathbb{R}}
\rbr[\big]{\mathfrak{N}^{\theta}_\mathfrak{h} (x) - f(x)}^2 \mathfrak{p}(x) \, \d x,
\end{equation}
let $\mathcal{G}_\mathfrak{h} = (\mathcal{G}_\mathfrak{h}^1 , \ldots, \mathcal{G}_\mathfrak{h}^{\mathfrak{d}_\mathfrak{h}}) \colon \mathbb{R}^{\mathfrak{d}_\mathfrak{h}} \to \mathbb{R}^{\mathfrak{d}_\mathfrak{h}}$ satisfy for all $k \in \{1, 2, \ldots, \mathfrak{d}_\mathfrak{h}\}$ that $\mathcal{G}_\mathfrak{h}^k (\theta) \allowbreak = \allowbreak (\frac{\partial^-}{\partial \theta_k} \mathcal{L}_\mathfrak{h}) (\theta) \allowbreak \indicator{\mathbb{N} \backslash (\mathfrak{h}, 3 \mathfrak{h}]} (k)$, and let $\Theta^\mathfrak{h} = (\Theta^{\mathfrak{h}, 1}, \ldots, \Theta^{\mathfrak{h}, \mathfrak{d}_\mathfrak{h}}) \colon [0, \infty) \times \Omega \to \mathbb{R}^{\mathfrak{d}_\mathfrak{h}}$
be a stochastic process with continuous sample paths, assume for all $\mathfrak{h} \in \mathbb{N}$, $t \in [0 , \infty )$ that 
\begin{equation} \label{theo:main:conv:eq:assgf}
\textstyle \P \rbr[\big]{\Theta_t^\mathfrak{h} = \Theta_0^\mathfrak{h} - \int_0^t \mathcal{G}_\mathfrak{h} (\Theta_s^\mathfrak{h}) \, \d s} = 1,
\end{equation}
assume for all $\mathfrak{h} \in \mathbb{N}$ that $\Theta_0^{\mathfrak{h}, 1}, \ldots, \Theta_0^{\mathfrak{h}, \mathfrak{d}_\mathfrak{h}}$ are independent, let $\alpha \in (\nicefrac{3}{4}, 1)$, $\beta \in (\alpha + 2, \infty)$, assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \{1, 2, \ldots, \mathfrak{h}\}$ that $\mathfrak{h}^{- \beta} \Theta_0^{\mathfrak{h}, k}$ is standard normal,
assume for all $\mathfrak{h} \in \mathbb{N} $, $k \in \{1, 2, \ldots, \mathfrak{h}\}$, $x \in (0, \infty)$ that
\begin{equation}\label{eqn:theo:main:conv:weights}
\textstyle  \P\rbr[\big]{\mathfrak{h}^{- \beta } \Theta_0^{ \mathfrak{h}, \mathfrak{h} + k } \leq x} = \P\rbr[\big]{\mathfrak{h}^{\alpha} \Theta_0^{\mathfrak{h}, 2 \mathfrak{h} + k} \le x} = [\frac{2}{\pi}]^{1/2} \int_0^x \exp (- \frac{y^2}{2}) \, \d y,
\end{equation}
assume for all $\mathfrak{h} \in \mathbb{N}$
that $\mathbb{E} [ \abs{ \Theta_0^{ \mathfrak{h} , \mathfrak{d}_\mathfrak{h} } } ^2 ] < \infty $,
and let $\delta \in (0, \min \cu{\frac{\beta - \alpha - 2 }{2} , \frac{ 4 \alpha - 3 }{ 9 } } ) $ \cfload. Then
\begin{enumerate} [label = (\roman*)]
\item
\label{theo:main:conv:item1}
it holds for all $\mathfrak{h} \in \mathbb{N}$ that $ \sup_{t \in [0, \infty ) } \mathbb{E} [ \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) ]
= \mathbb{E} [ 
\sup_{t \in [0, \infty ) } \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) ] < \infty $,
\item
\label{theo:main:conv:item2}
there exists $c \in (0, \infty )$ such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P \rbr[\big]{ \limsup\nolimits_{t \to \infty} \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) \ge c}
\ge 
\P \rbr[\big]{ \inf\nolimits_{t \in [0 , \infty ) } \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) \ge c}
\ge c^\mathfrak{h} > 0,
\end{equation}
\item
\label{theo:main:conv:item3}
there exist $\bfc \in (0, \infty )$
and random variables $\mathfrak{C}_\mathfrak{h} \colon \Omega \to \mathbb{R}$,
$\mathfrak{h} \in \mathbb{N}$,
which satisfy for all
$\mathfrak{h} \in \mathbb{N}$
that
\begin{equation}
\textstyle \P \rbr[\big]{ \forall \, t \in (0, \infty ) \colon \mathcal{L}_\mathfrak{h} ( \Theta_t ^\mathfrak{h} ) \le \bfc \mathfrak{h} ^{ - \delta } + \mathfrak{C}_\mathfrak{h} t^{-1} } \ge 1 - \bfc \mathfrak{h} ^{ - \delta },
\end{equation}
and
\item
\label{theo:main:conv:item4}
for every $\varepsilon \in ( - \infty , \delta - 2 ( 1 - \alpha ) )$ there exists $\mathfrak{C} \in (0 , \infty )$ such that for all
$\mathfrak{h} \in \mathbb{N}$
we have that
\begin{equation}
\textstyle \limsup\nolimits_{ t \to \infty }
\rbr[\big]{ \mathbb{E}[ 
\mathcal{L}_{ \mathfrak{h} }( 
\Theta^{ \mathfrak{h} }_t ) ] }
\le \mathbb{E} \br[\big]{ \limsup\nolimits_{t \to \infty} \mathcal{L}_{ \mathfrak{h} }( 
\Theta^{ \mathfrak{h} }_t) }
\le \mathfrak{C} \mathfrak{h} ^{ - \varepsilon }.
\end{equation}
\end{enumerate}
\end{theorem}

Note that \cref{theo:main:conv:item2} shows for every width $\mathfrak{h} \in \mathbb{N}$ that, with a positive probability, the risk $\mathcal{L} _ \mathfrak{h} ( \Theta_t ^\mathfrak{h} )$ does not converge to zero.
Nevertheless, \cref{theo:main:conv:item3} reveals that the risk converges with probability close to $1$ at rate $\mathcal{O} ( t^{-1} )$ to a limiting value of at most $\mathbf{c} \mathfrak{h} ^{ - \delta }$.
Additionally, \cref{theo:main:conv:item4} establishes convergence of the risk in the $L^1$-norm with respect to the underlying probability distribution.

\begin{cproof}{theo:main:conv}
Throughout this proof let $\operatorname{Lip} \colon C(\mathbb{R}, \mathbb{R}) \to [0, \infty]$ satisfy for all $F \in C(\mathbb{R}, \allowbreak \mathbb{R})$ that $\operatorname{Lip}(F) = \sup_{x, y \in [\scra, \scrb], x \neq y}\frac{\abs{F(x) - F(y)}}{\abs{x - y}}$, let $\mathfrak{D} , \nu \in (0 , \infty )$ satisfy $\mathfrak{D} = \sup_{x \in [ \scra , \scrb ]} \abs{ f(x) }$ and $\nu = \int_\mathbb{R} \mathfrak{p} ( x ) \, \d x$,
for every $\mathfrak{h} \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$, $\theta = (\theta_1, \ldots, \theta_{\mathfrak{d}_\mathfrak{h} } ) \in \mathbb{R}^{ \mathfrak{d}_\mathfrak{h}}$ let $I_i^\theta \subseteq \mathbb{R}$ satisfy $I_i^\theta = \cu{ x \in ( \scra , \scrb ) \colon \theta_{\mathfrak{h} + i} x + \theta_{ i } \in (0 , 1 ) } $, and let $G \in \mathcal{F}$ satisfy
\begin{equation} \label{theo:main:conv:eq:def:g}
\textstyle G = \cu[\big]{\omega \in \Omega \colon \forall \, t \in [0, \infty ) \colon \Theta_t^\mathfrak{h}  (\omega)
= \Theta_0^\mathfrak{h} ( \omega )  - \textstyle\int_0^t \mathcal{G}_\mathfrak{h} ( \Theta_s ^\mathfrak{h} ( \omega ) ) \, \d s } .
\end{equation}
First, \cref{theo:main:conv:eq:def:g} and \cref{prop:gf:chain:rule} assure for all $\mathfrak{h} \in \mathbb{N}$, $t \in [0, \infty )$, $\omega \in G$
that $\mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ( \omega )) \le \mathcal{L}_\mathfrak{h} ( \Theta_0^\mathfrak{h} ( \omega ))$. This and the fact that $\P ( G ) = 1$ (cf.~\cref{theo:main:conv:eq:assgf}) show for all $\mathfrak{h} \in \mathbb{N}$ that 
\begin{equation} \label{theo:main:conv:eq:supt}
\sup\nolimits_{t \in [0, \infty )} \mathbb{E} \br[\big]{ \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) } = \mathbb{E} \br[\big]{ \sup\nolimits_{t \in [0, \infty) } \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) } = \mathbb{E} \br[\big]{ \mathcal{L}_\mathfrak{h} ( \Theta_0 ^\mathfrak{h} )  }.
\end{equation}
Furthermore, the assumption that $\mathfrak{p}^{-1}(\mathbb{R} \backslash \cu{0}) = (\scra, \scrb)$ and the Cauchy-Schwarz inequality imply for all $\mathfrak{h} \in \mathbb{N}$, $\theta = ( \theta_1, \ldots, \theta_{\mathfrak{d}_\mathfrak{h} } ) \in \mathbb{R}^{ \mathfrak{d}_\mathfrak{h} }$ that
\begin{equation}
\begin{split}
\textstyle \mathcal{L}_\mathfrak{h} ( \theta )
& \textstyle \le ( \mathfrak{h} + 2 ) \rbr[\big]{
\int_\scra^\scrb f(x) ^2 \mathfrak{p} ( x ) \, \d x + \int_\scra^\scrb \abs{\theta_{\mathfrak{d}_\mathfrak{h} } } ^2 \mathfrak{p} ( x ) \, \d x 
+ \sum_{i=1}^\mathfrak{h} \int_\scra ^\scrb \abs{ \theta_{2 \mathfrak{h} + i } } ^2 \mathfrak{p} ( x ) \, \d x
}
\\
& \textstyle \le (\mathfrak{h} + 2 ) \nu
\rbr[\big]{ \mathfrak{D} ^2 + \abs{\theta_{\mathfrak{d}_\mathfrak{h} } } ^2 + \textstyle\sum_{i=1}^\mathfrak{h}  \abs{ \theta_{ 2 \mathfrak{h} + i } } ^2 }.
\end{split}
\end{equation}
Combining this with the fact that for all $\mathfrak{h} \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{h}\}$ it holds that $\mathbb{E} [ \abs{ \Theta^{\mathfrak{h} , \mathfrak{d}_\mathfrak{h} }_0 } ^2 ] < \infty$ and $\mathbb{E} [ \abs{ \Theta^{\mathfrak{h} , 2 \mathfrak{h} + i } _0 } ^2 ] = \mathfrak{h}^{-2 \alpha }$ demonstrates that for all $\mathfrak{h} \in \mathbb{N}$ we have that
\begin{equation}
\mathbb{E} [ \mathcal{L}_\mathfrak{h} ( \Theta^\mathfrak{h}_0 ) ]
\le ( \mathfrak{h} + 2 ) \nu \rbr[\big]{ \mathfrak{D} ^2 + \mathfrak{h} ^{ 1 - 2 \alpha } + \mathbb{E} [ \abs{ \Theta^{\mathfrak{h} , \mathfrak{d}_\mathfrak{h} }_0 } ^2 ] } < \infty .
\end{equation}
This and \cref{theo:main:conv:eq:supt} establish \cref{theo:main:conv:item1}. Next, let $\xi \in (0, \infty )$ satisfy
\begin{equation}
\textstyle \xi = \inf_{\mu \in \mathbb{R} } \int_\scra^\scrb (f(x) - \mu )^2 \mathfrak{p} ( x ) \, \d x
\end{equation}
and for every $\mathfrak{h} \in \mathbb{N}$ let $B_\mathfrak{h} \in \mathcal{F}$ satisfy 
\begin{equation}
\textstyle B_\mathfrak{h} = \cu[\big]{\omega \in G \colon \big[\forall \, i \in \{1, 2, \ldots, \mathfrak{h}\} \colon I_i^{\Theta_0^\mathfrak{h}} = \emptyset\big]}.
\end{equation}
\Nobs that \cref{lem:random:init}, \cref{eqn:theo:main:conv:weights}, and the fact that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $\Theta_0^{ \mathfrak{h} , i }$, $i \in \{1, 2, \ldots, \mathfrak{d}_\mathfrak{h} \}$, are independent demonstrate that there exists $c \in (0,1)$ which satisfies for all $\mathfrak{h} \in \mathbb{N}$ that $\P ( B_\mathfrak{h} ) \ge c^\mathfrak{h} $. Moreover, the fact that $\mathfrak{p}^{-1 } ( \mathbb{R} \backslash \cu{0} ) = ( \scra , \scrb )$, \cref{theo:main:conv:eq:def:g}, and \cref{prop:risk:gradient} prove for all $\mathfrak{h} \in \mathbb{N}$, $\omega \in B_\mathfrak{h}$, $t \in [0, \infty )$ that $ \forall \, i \in \{1, 2, \ldots, \mathfrak{h}\} \colon I_i^{ \Theta_t^\mathfrak{h} } = \emptyset $. Hence, we obtain for all $\mathfrak{h} \in \mathbb{N}$, $\omega \in B_\mathfrak{h}$, $t \in [0, \infty )$ that $(\scra , \scrb ) \ni x \mapsto \mathfrak{N}^ {\Theta_t^\mathfrak{h}  }_\mathfrak{h} ( x ) \in \mathbb{R} $ is constant. This implies for all $\mathfrak{h} \in \mathbb{N}$, $\omega \in B_\mathfrak{h}$ that
\begin{equation}
\textstyle \forall\, t \in [0, \infty ) \colon \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ( \omega ) ) \ge \xi > 0.
\end{equation}
Therefore, we obtain for all $\mathfrak{h} \in \mathbb{N}$ that
\begin{equation}
\textstyle \P \rbr[\big]{\limsup\nolimits_{t \to \infty} \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) \ge \xi} \ge \P \rbr[\big]{ \inf\nolimits_{t \in [0 , \infty ) } \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) \ge \xi} \ge \P(B_\mathfrak{h})
\ge c^\mathfrak{h} > 0.
\end{equation}
This establishes \cref{theo:main:conv:item2}. Next, the assumption that $\mathfrak{p}^{-1} (\mathbb{R} \backslash \{0\}) = (\scra, \scrb)$ and the assumption that $\mathfrak{p}$ is piecewise polynomial imply that there exist $\varepsilon_1, \varepsilon_2  \in (0 , \infty )$, $\varepsilon_3 \in (0 , \varepsilon_1 )$ which satisfy that $\mathfrak{p} _ { [ \scra , \scra + \varepsilon_1 ] }$ is strictly increasing, $\mathfrak{p}_{[\scrb - \varepsilon_1, \scrb ]}$ is strictly decreasing, and $\inf_{x \in [ \scra + \varepsilon_3 , \scrb - \varepsilon_3 ] } > \varepsilon_2 \operatorname{Lip} ( \mathfrak{p} )$. In addition, the fact that $0 < \delta < (4 \alpha - 3) / 9$ and the fact that $\nicefrac{3}{4} < \alpha < 1$ assure that there exists $\gamma \in (\nicefrac{3}{4} , \alpha )$ which satisfies
\begin{equation} \label{theo:main:conv:eq:gamma}
\textstyle \delta < \min \cu*{ 4 \gamma - 3 , \gamma - \tfrac{1}{2} , \tfrac{ \alpha - \gamma}{2} }.
\end{equation}
Moreover, \cref{cor:limit_point} proves that for all $\mathfrak{h} \in \mathbb{N}$, $\omega \in G$ there exist $\vartheta^\mathfrak{h} ( \omega ) \in \mathbb{R}^{\mathfrak{d}_\mathfrak{h} }$, $\mathfrak{C}_\mathfrak{h} ( \omega ) \in \mathbb{R}$ which satisfy for all $t \in [0, \infty )$ that $\limsup_{s \to \infty} \norm{\Theta^\mathfrak{h}_s ( \omega ) - \vartheta^\mathfrak{h} ( \omega ) } = 0$ and $0 \le \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ( \omega ) ) - \mathcal{L}_\mathfrak{h} ( \vartheta^\mathfrak{h} ( \omega ) ) \le \mathfrak{C}_\mathfrak{h} ( \omega ) ( 1+t)^{-1}$. 
In the following let $\mathbf{m} \in (0 , \infty )$
satisfy
\begin{equation}
\textstyle \mathbf{m} =  \nu ^{ - 1 }
\min \cu[\big]{ \int_\scra^\scrb ( f(\scrb ) - f(x) ) \mathfrak{p} ( x ) \, \d x ,
	\int_\scra^\scrb ( f( x ) - f ( \scra ) ) \mathfrak{p} ( x ) \, \d x} 
\end{equation}
and for every $\mathfrak{h} \in \mathbb{N}$ let $A_i^\mathfrak{h} \in \mathcal{F}$, $i \in \{1, 2, \ldots, 5\}$, satisfy
\begin{equation}
\textstyle A_1^\mathfrak{h} = \cu*{ \omega \in G \colon \sup\nolimits_{x \in ( \scra , \scrb ) } \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in  I_i^{\vartheta^\mathfrak{h} ( \omega ) } } \vartheta^{ \mathfrak{h} , 2 \mathfrak{h} + i } ( \omega ) < \min \cu{ \mathbf{m} , \mathfrak{h} ^{ - \delta } } },
\end{equation}
\begin{equation}
\textstyle A_2^\mathfrak{h} = \cu*{ \omega \in \Omega \colon
\sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, I_i^{ \Theta_0 ( \omega ) } \not= \emptyset }
\Theta_0^{ \mathfrak{h} , 2 \mathfrak{h} + i } ( \omega ) 
> f ( \scrb ) - f ( \scra ) + 4 \mathbf{m}  },
\end{equation}
\begin{multline}
\textstyle A_3^\mathfrak{h} = \Big\{ \omega \in G \colon 
\sup\nolimits_{x \in ( \scra , \scrb ) }  \sum_{i \in \{1, 2, \ldots, \mathfrak{h}\}, \, x \in I_i^{ \vartheta^\mathfrak{h} ( \omega ) } } \vartheta^{ \mathfrak{h} , \mathfrak{h} + i } ( \omega ) \vartheta^{ \mathfrak{h} , 2 \mathfrak{h} + i } ( \omega ) \\
 > 4 \max \nolimits_{i \in \{1, 2, \ldots, \mathfrak{h}\} }  \vartheta^{ \mathfrak{h} , \mathfrak{h} + i } ( \omega ) \vartheta^{ \mathfrak{h} , 2 \mathfrak{h} + i } ( \omega ) \Big\},
\end{multline}
\begin{equation}
A_4^\mathfrak{h} = \cu[\big]{ \omega \in G \colon \min \nolimits_{i \in \{1, 2, \ldots, \mathfrak{h}\} } \Theta_0^{ \mathfrak{h} , \mathfrak{h} + i } ( \omega ) \Theta_0^{ \mathfrak{h} , 2 \mathfrak{h} + i } ( \omega ) > \operatorname{Lip} ( f ) },
\end{equation}
and
\begin{equation}
\textstyle A_5^\mathfrak{h} = \cu[\big]{ \omega \in G \colon \min\nolimits_{i \in \{1, 2, \ldots, \mathfrak{h}\} } \Theta_0^{ \mathfrak{h} , \mathfrak{h} + i } ( \omega ) > \max \cu{ ( \varepsilon_2 ) ^{-1} , 2 ( \varepsilon_1 - \varepsilon_3 ) ^{-1} } }.
\end{equation}
\Nobs that the fact that $\forall \, \mathfrak{h} \in \mathbb{N} , \, \omega \in  G \colon \mathcal{G}_\mathfrak{h} ( \vartheta^\mathfrak{h} ( \omega ) ) = 0 $ and \cref{cor:non:strict:saddle} demonstrate for all $\mathfrak{h} \in \mathbb{N}$, $\omega \in A_3^\mathfrak{h} \cap A_4^\mathfrak{h} \cap A_5^\mathfrak{h}$ that the Hessian of $\mathcal{L}_ \mathfrak{h} ( \vartheta^\mathfrak{h} ( \omega ) )$ restricted to the coordinates $\vartheta_i^\mathfrak{h}$, $i \in ( \mathbb{N} \cap [1  , \mathfrak{h} ] ) \cup \cu{ \mathfrak{d}_\mathfrak{h} }$, has a negative eigenvalue. Combining this with \cref{cor:strict:saddle:conv:random} shows for all $\mathfrak{h} \in \mathbb{N} $ that $ \P ( A_3^\mathfrak{h} \cap A_4^\mathfrak{h} \cap A_5^\mathfrak{h} ) = 0$. \Nobs that \cref{theo:conv:bias:deterministic} proves that there exists $\mathscr{C} \in \mathbb{R}$ which satisfies for all $\mathfrak{h} \in \mathbb{N}$, $\omega \in A_1^\mathfrak{h} \cap A_2^\mathfrak{h} \cap A_4^\mathfrak{h}$ that
\begin{equation} \label{theo:main:conv:eq:limit:risk}
\textstyle \mathcal{L}_\mathfrak{h} ( \vartheta^\mathfrak{h} ( \omega ) )
\le \mathscr{C} ( \mathfrak{h} ^{ - \delta } + \mathfrak{h} ^{ - 2 \delta } )
\le 2 \mathscr{C} \mathfrak{h} ^{ - \delta }.
\end{equation}
Hence, we obtain for all $\mathfrak{h} \in \mathbb{N}$ that
\begin{equation} \label{theo:conv:random:eq:risk:prob}
\begin{split}
\textstyle \P \rbr[\big]{ \forall \, t \in [0, \infty ) \colon \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) \le 2 \mathscr{C} \mathfrak{h} ^{ - \delta } + \mathfrak{C}_\mathfrak{h} t^{-1}} & \textstyle \ge \P \rbr[\big]{ \mathcal{L}_\mathfrak{h} ( \vartheta^\mathfrak{h} ( \omega ) ) \le 2 \mathscr{C} \mathfrak{h} ^{ - \delta } } \\
& \textstyle \ge  \P \rbr[\big]{ A_1^\mathfrak{h} \cap A_2^\mathfrak{h} \cap A_4^\mathfrak{h}}.
\end{split}
\end{equation}
Furthermore, \cref{lem:random:init} proves that there exist $c_1, c_2 \in (0 , \infty)$ which satisfy for all $\mathfrak{h} \in \mathbb{N}$ that $\P ( A_2^\mathfrak{h} ) \ge 1 - c_1 \mathfrak{h} ^{-1} \ge 1 - c_2 \mathfrak{h} ^{ - \delta }$, $\P ( A_4^\mathfrak{h} ) \ge 1 - c_2 \mathfrak{h} ^{1 - \nicefrac{( \beta - \alpha )}{2} } \ge 1 - c_2 \mathfrak{h} ^{ - \delta }$, and $\P ( A_5^\mathfrak{h} ) \ge 1 - c_2 \mathfrak{h}^{ 1 - \nicefrac{ \beta}{2} } \ge 1 - c_2 \mathfrak{h} ^{ - \delta }$. Hence, we obtain for all $\mathfrak{h} \in \mathbb{N}$ that
\begin{equation} \label{theo:conv:random:eq:prop:lower1}
\begin{split}
\textstyle \P \rbr[\big]{ A_1^\mathfrak{h} \cap A_2^\mathfrak{h} \cap A_4^\mathfrak{h} } & \textstyle \ge 1 - \P ( \Omega \backslash A_1^\mathfrak{h} ) - \P (\Omega \backslash A_2^\mathfrak{h}) - \P ( \Omega \backslash A_4^\mathfrak{h} ) \\
& \textstyle \ge 1 - \P \rbr[\big]{ ( \Omega \backslash A_1^\mathfrak{h} ) \cap A_3^\mathfrak{h} } - \P \rbr[\big]{ ( \Omega \backslash A_1^\mathfrak{h} ) \cap (\Omega \backslash A_3^\mathfrak{h}) } - (c_1 + c_2) \mathfrak{h}^{- \delta}.
\end{split}
\end{equation}
Moreover, the fact that $\forall \, \mathfrak{h} \in \mathbb{N}$, $i \in \mathbb{N} \cap ( \mathfrak{h} , 3 \mathfrak{h} ] \colon \vartheta^{ \mathfrak{h} , i } = \Theta_0^{ \mathfrak{h} , i }$, \cref{theo:main:conv:eq:gamma}, and \cref{lem:random:init:itemnew} in \cref{lem:random:init} (applied with $\eta \with \delta$ in the notation of \cref{lem:random:init:itemnew} in \cref{lem:random:init}) establish that there exists $c_3 \in (0 , \infty )$ which satisfies for all $\mathfrak{h} \in \mathbb{N} \cap ( \mathbf{m}^{ - \nicefrac{1}{\delta } } , \infty )$ that 
\begin{equation} 
\begin{split}
& \textstyle \P \rbr[\big]{ ( \Omega \backslash A_1^\mathfrak{h} ) \cap ( \Omega \backslash A_3^\mathfrak{h} ) } \le \P \rbr[\big]{ \exists \,  J \subseteq \{1, 2, \ldots, \mathfrak{h}\} \colon \\
& \textstyle \br[\big]{ \sum_{i \in J } \Theta_0^{ \mathfrak{h} , \mathfrak{h} + i } \Theta_0^{ \mathfrak{h} , 2 \mathfrak{h} + i } \le 4 \max\nolimits_{i \in \{1, 2, \ldots, \mathfrak{h}\} } \Theta^{ \mathfrak{h} , \mathfrak{h} + i } \Theta_0^{ \mathfrak{h} , 2 \mathfrak{h} + i } \wedge  \textstyle\sum_{j \in J } \Theta_0^{ \mathfrak{h} , 2 \mathfrak{h} + j } \ge  \mathfrak{h} ^{ - \delta } } } \le c_3 \mathfrak{h} ^{ - \delta }.
\end{split}
\end{equation}
Hence, we obtain for all $\mathfrak{h} \in \mathbb{N}$ that
\begin{equation} \label{theo:conv:random:eq:prop:lower2}
\textstyle \P \rbr[\big]{ ( \Omega \backslash A_1^\mathfrak{h} ) \cap ( \Omega \backslash A_3^\mathfrak{h} ) }
\le \max \cu{ c_3 , \mathbf{m}^{-1} } \mathfrak{h} ^{ - \delta }.
\end{equation}
In addition, the fact that $ \P ( A_3^\mathfrak{h} \cap A_4^\mathfrak{h} \cap A_5^\mathfrak{h} ) = 0$ proves for all $\mathfrak{h} \in \mathbb{N}$ that $\P ( ( \Omega \backslash A_1^\mathfrak{h} ) \cap A_3^\mathfrak{h} ) \le \P ( A_3^\mathfrak{h} ) \le \P ( \Omega \backslash A_4^\mathfrak{h} ) + \P ( \Omega \backslash A_5^\mathfrak{h} ) \le 2 c_2 \mathfrak{h} ^{ - \delta }$. This, \cref{theo:conv:random:eq:prop:lower1}, and \cref{theo:conv:random:eq:prop:lower2} ensure for all $\mathfrak{h} \in \mathbb{N}$ that
\begin{equation} \label{theo:conv:random:eq:prop:lower3}
\textstyle \P \rbr[\big]{ A_1^\mathfrak{h} \cap A_2^\mathfrak{h} \cap A_4^\mathfrak{h} } \ge 1 - ( c_1 + 3c_2 + \max \cu{ c_3 , \mathbf{m}^{-1} } ) \mathfrak{h} ^{ - \delta }.
\end{equation}
Combining this with \cref{theo:conv:random:eq:risk:prob} establishes \cref{theo:main:conv:item3}. Next \nobs that for all $\mathfrak{h} \in \mathbb{N}$,
$\omega \in G$
it holds that $\limsup_{t \to \infty} \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ( \omega ) ) = \mathcal{L}_\mathfrak{h} ( \vartheta^\mathfrak{h} ( \omega ) )$. Combining this with \cref{theo:main:conv:eq:limit:risk},
the fact that $\P ( G ) = 1$, and Hölder's inequality establishes for all $\mathfrak{h} \in \mathbb{N}$, $p , q\in (1 , \infty )$ with $\frac{1}{p} + \frac{1}{q} = 1$ that
\begin{equation} \label{theo:main:conv:eq:limsup:est}
\begin{split}
\textstyle \mathbb{E} \br[\big]{ \limsup\nolimits_{t \to \infty} \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h})} & \textstyle = \mathbb{E} \br[\big]{\mathcal{L}_\mathfrak{h} (\vartheta^\mathfrak{h})} = \mathbb{E} \br[\big]{\mathcal{L}_\mathfrak{h} ( \vartheta^\mathfrak{h} ) \indicator{ A_1^\mathfrak{h} \cap A_2^\mathfrak{h} \cap A_4^\mathfrak{h} } } + 
\mathbb{E} \br[\big]{\mathcal{L}_\mathfrak{h} ( \vartheta^\mathfrak{h} ) \indicator{\Omega \backslash( A_1^\mathfrak{h} \cap A_2^\mathfrak{h} \cap A_4^\mathfrak{h} ) } } \\
& \textstyle \le 2 \mathscr{C} \mathfrak{h}^{- \delta} + \rbr*{ \mathbb{E} \br[\big]{\indicator{\Omega  \backslash( A_1^\mathfrak{h} \cap A_2^\mathfrak{h} \cap A_4^\mathfrak{h} ) } } }^{ \nicefrac{1}{p}} \rbr[\big]{ \mathbb{E} \br[\big]{ ( \mathcal{L} _ \mathfrak{h} ( \vartheta ^\mathfrak{h} ) )^{ q } } }^{ \nicefrac{1}{q} } \\
& \textstyle =  2 \mathscr{C} \mathfrak{h}^{- \delta} + \rbr[\big]{1 - \P  \rbr[\big]{ A_1^\mathfrak{h} \cap A_2^\mathfrak{h} \cap A_4^\mathfrak{h}}}^{\nicefrac{1}{p}} \rbr[\big]{ \mathbb{E} \br[\big]{ ( \mathcal{L} _ \mathfrak{h} ( \vartheta ^\mathfrak{h} ) )^{q}}}^{\nicefrac{1}{q}}.
\end{split}
\end{equation}
In addition, the fact that $\forall \, \mathfrak{h} \in \mathbb{N} , \omega \in G \colon \mathcal{G}_\mathfrak{h} ( \vartheta^\mathfrak{h} ( \omega ) ) = 0 $ and \cref{lem:realization:pointwise:bound} assure for all $\mathfrak{h} \in \mathbb{N}$,
$\omega \in G$ that
\begin{equation}
\begin{split}
\textstyle \mathcal{L}_\mathfrak{h} ( \vartheta^{ \mathfrak{h} } ( \omega ))
& \textstyle \le \nu \rbr[\big]{ (f(\scrb ) - f(\scra) ) ^2 + \abs[\big]{ \sum_{i=1}^\mathfrak{h} \vartheta^{ \mathfrak{h} , 2 \mathfrak{h} + i } ( \omega) }^2 } \\
& \textstyle =  \nu \rbr[\big]{ ( f ( \scrb ) - f ( \scra ) )^2 + \abs[\big]{\sum_{i=1}^\mathfrak{h}  \Theta_0^{ \mathfrak{h} , 2 \mathfrak{h} + i}(\omega)}^2}.
\end{split}
\end{equation}
This, the fact that $\P ( G ) = 1$, and the fact that $ \forall \, x,y \in [0, \infty ) , q \in [1 , \infty ) \colon   ( x + y )^q \le 2^{ q - 1 } ( x^q + y^q )$ demonstrate for all $\mathfrak{h} \in \mathbb{N}$, $q \in [1 , \infty )$ that
\begin{equation}
\textstyle \mathbb{E} \br[\big]{ ( \mathcal{L} _ \mathfrak{h} ( \vartheta ^\mathfrak{h} ) )^{ q } } \le 2^{q-1} \nu^q \rbr*{ (f(\scrb ) - f(\scra ) ) ^{ 2 q } + \abs[\big]{ \sum_{i = 1}^\mathfrak{h} \Theta_0^{\mathfrak{h}, 2 \mathfrak{h} + i}}^{2 q}}.
\end{equation}
Combining this with \cref{lem:random:init:item3} in \cref{lem:random:init} implies that for all $q \in [  1 , \infty )$
there exists $\mathfrak{C} \in \mathbb{R}$ such that for all $\mathfrak{h} \in \mathbb{N}$ it holds
that
\begin{equation}
\textstyle \rbr[\big]{\mathbb{E} \br[\big]{ ( \mathcal{L} _ \mathfrak{h} ( \vartheta ^\mathfrak{h} ))^{q}}}^{\nicefrac{1}{q}} \le \mathfrak{C} ( 1 + \mathfrak{h} ^{ 2 - 2 \alpha } ) \le 2 \mathfrak{C} \mathfrak{h}^{2 - 2 \alpha}.
\end{equation}
This, \cref{theo:conv:random:eq:prop:lower3}, and \cref{theo:main:conv:eq:limsup:est} show that for all $p \in (1, \infty )$ there exists $\mathfrak{C} \in \mathbb{R}$ which satisfies for all $\mathfrak{h} \in \mathbb{N}$ that $ \mathbb{E} \br{ \limsup\nolimits_{t \to \infty} \mathcal{L}_\mathfrak{h} ( \Theta_t^\mathfrak{h} ) } \le 2 \mathscr{C} \mathfrak{h} ^{ - \delta } + \mathfrak{C}  \mathfrak{h} ^{ 2 - 2 \alpha - \nicefrac{\delta }{p}} $. The reverse Fatou's lemma and the fact that $\forall \, p \in (1, \infty ), \varepsilon \in ( - \infty ,  \nicefrac{\delta }{p} - 2 ( 1 - \alpha ) ] , \mathfrak{h} \in \mathbb{N} \colon  \mathfrak{h} ^{ 2 - 2 \alpha - \nicefrac{\delta }{p}} \le \mathfrak{h}^{ - \varepsilon}$ therefore establish \cref{theo:main:conv:item4}.
\end{cproof}

As a consequence, we obtain in \cref{cor:main:conv} a convergence rate of $\mathfrak{h} ^{- \varepsilon}$ for $\varepsilon \in (0, \infty)$ arbitrarily close to $\frac{1}{9}$, if the scaling exponents $\alpha, \beta$ are chosen appropriately. Thus the rate is independent of any specific features of the target function $f$ or the density $\mathfrak{p}$. For the proof, we simply need to choose the scaling parameters $\alpha, \beta$ in \cref{theo:main:conv} in such a way that the conditions on $\varepsilon$ and $\delta$ are satisfied, which is possible iff $\varepsilon < \frac{1}{9} $.


\cfclear
\begin{cor} \label{cor:main:conv}
Let $\scra \in \mathbb{R}$, $\scrb \in (\scra , \infty )$, let $f, \mathfrak{p} \in C ( \mathbb{R} , \mathbb{R})$ be piecewise polynomial\cfadd{def:piecewise_polynomial}, assume for all $x \in \mathbb{R}$ that $\mathfrak{p} ( x ) \ge 0$ and $\mathfrak{p}^{- 1} (\mathbb{R} \backslash \{0\}) = (\scra, \scrb)$, assume that $f$ is strictly increasing, let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $\mathfrak{h} \in \mathbb{N}$, $\theta = ( \theta_1, \ldots, \theta_{\mathfrak{d}_\mathfrak{h} } ) \in \mathbb{R}^{ \mathfrak{d}_\mathfrak{h} }$ let $\mathfrak{d} _\mathfrak{h} = 3 \mathfrak{h} + 1 $, let $\mathfrak{N}^{ \theta}_\mathfrak{h} \colon \mathbb{R} \to \mathbb{R}$ satisfy for all $x \in \mathbb{R}$ that
\begin{equation}
\textstyle \mathfrak{N}^{ \theta }_\mathfrak{h} ( x ) = \theta_{ \mathfrak{d}_\mathfrak{h} } +
\sum_{ i = 1 }^{ \mathfrak{h} } \theta_{2 \mathfrak{h} + i } \max \cu{ \min \cu{ \theta_{\mathfrak{h} + i } x + \theta_{ i } , 1 } , 0 } ,
\end{equation}
let $\mathcal{L}_\mathfrak{h} \colon \mathbb{R}^{ \mathfrak{d}_\mathfrak{h} } \to \mathbb{R}$ satisfy
\begin{equation} 
\textstyle \mathcal{L}_{ \mathfrak{h} }( \theta ) 
= \int_{ \mathbb{R} }
\rbr[\big]{  \mathfrak{N}^{\theta}_\mathfrak{h} ( x ) - f ( x ) }^2 \mathfrak{p}( x ) \, \d x,
\end{equation}
and let $\mathcal{G}_\mathfrak{h} = ( \mathcal{G}_\mathfrak{h} ^1 , \ldots, \mathcal{G}_\mathfrak{h}^{\mathfrak{d}_\mathfrak{h}}) \colon \mathbb{R}^{\mathfrak{d}_\mathfrak{h}} \to \mathbb{R}^{\mathfrak{d}_\mathfrak{h}}$ satisfy for all $k \in \{1, 2, \ldots, \mathfrak{d}_\mathfrak{h}\}$ that $\mathcal{G}_\mathfrak{h}^k (\theta) = (\frac{\partial^-}{\partial \theta_k} \mathcal{L}_\mathfrak{h}) (\theta) \allowbreak \indicator{\mathbb{N} \backslash ( \mathfrak{h} , 3 \mathfrak{h} ] } ( k )$, for every $\alpha, \beta \in \mathbb{R}$ let $\Theta^{\mathfrak{h}, \alpha, \beta} = (\Theta^{\mathfrak{h}, \alpha, \beta, 1}, \ldots, \Theta^{ \mathfrak{h} , \alpha , \beta , \mathfrak{d}_\mathfrak{h} } ) \colon [0, \infty ) \times \Omega \to \mathbb{R}^{ \mathfrak{d}_\mathfrak{h} }$, $\mathfrak{h} \in \mathbb{N}$, be stochastic processes with continuous sample paths, assume for all $\mathfrak{h} \in \mathbb{N}$, $\alpha , \beta \in \mathbb{R}$, $t \in [0 , \infty ) $ that 
\begin{equation}
\P \bigl(\Theta_t^{ \mathfrak{h} , \alpha , \beta }  
= \Theta_0^{ \mathfrak{h} , \alpha , \beta }   - \textstyle\int_0^t \mathcal{G}_\mathfrak{h} ( \Theta_s ^{\mathfrak{h} , \alpha , \beta }  ) \, \d s \bigr) = 1 ,
\end{equation}
assume for all $\mathfrak{h} \in \mathbb{N}$, $\alpha , \beta \in \mathbb{R}$ that $\Theta_0^{ \mathfrak{h} , \alpha , \beta , 1 } , \ldots, \Theta_0^{ \mathfrak{h} , \alpha , \beta , \mathfrak{d}_\mathfrak{h} }$ are independent, assume for all $\mathfrak{h} \in \mathbb{N}$, $\alpha , \beta \in \mathbb{R}$, $k \in \{1, 2, \ldots, \mathfrak{h}\}$ that $\mathfrak{h}^{- \beta } \Theta_0^{ \mathfrak{h} , \alpha , \beta , k }$ is standard normal, assume for all $\mathfrak{h} \in \mathbb{N}$, $\alpha, \beta \in \mathbb{R}$, $ k \in \{1, 2, \ldots, \mathfrak{h}\}$, $x \in (0, \infty)$ that
\begin{equation}
\textstyle \P(\mathfrak{h}^{- \beta} \Theta_0^{\mathfrak{h}, \alpha, \beta, \mathfrak{h} + k} \leq x) = \P (\mathfrak{h}^{\alpha} \Theta_0^{\mathfrak{h}, \alpha, \beta, 2 \mathfrak{h} + k} \le x) = [\frac{2}{\pi}]^{1/2} \int_0^x \exp(-  \frac{y^2}{2} ) \, \d y,
\end{equation}
assume for all $\mathfrak{h} \in \mathbb{N}$, $\alpha , \beta \in \mathbb{R}$ that $\mathbb{E} [ \abs{ \Theta_0^{ \mathfrak{h} , \alpha , \beta , \mathfrak{d}_\mathfrak{h} } } ^2 ] < \infty $, and let $\varepsilon \in (0, \nicefrac{1}{9})$ \cfload. Then there exist $\alpha \in (\nicefrac{3}{4}, 1)$, $\beta \in (\alpha + 2, \infty)$ such that
\begin{enumerate} [label = (\roman*)]
\item
\label{cor:main:conv:item1} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $ \sup_{t \in [0, \infty ) } \mathbb{E} [ \mathcal{L}_\mathfrak{h} (\Theta_t^{\mathfrak{h}, \alpha, \beta})] = \mathbb{E} [\sup_{t \in [0, \infty ) } \mathcal{L}_\mathfrak{h} ( \Theta_t^{ \mathfrak{h} , \alpha , \beta }  ) ] < \infty $,

\item
\label{cor:main:conv:item2} there exist $\bfc \in (0, \infty )$
and random variables $\mathfrak{C}_\mathfrak{h} \colon \Omega \to \mathbb{R}$, $\mathfrak{h} \in \mathbb{N}$, such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P \rbr[\big]{ \forall \, t \in (0, \infty ) \colon \mathcal{L}_\mathfrak{h} ( \Theta_t ^{ \mathfrak{h} , \alpha , \beta }  ) \le \bfc \mathfrak{h} ^{ - \varepsilon } + \mathfrak{C}_\mathfrak{h} t^{-1} } \ge 1 - \bfc \mathfrak{h} ^{ - \varepsilon },
\end{equation}
and

\item
\label{cor:main:conv:item3} there exists $\mathscr{C} \in (0 , \infty )$ such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \limsup\nolimits_{ t \to \infty } \rbr[\big]{\mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Theta^{\mathfrak{h}, \alpha, \beta}_t)]} \le \mathbb{E} \br[\big]{ \limsup\nolimits_{t \to \infty} \mathcal{L}_{ \mathfrak{h} }(\Theta^{\mathfrak{h} , \alpha , \beta } _t) } \le \mathscr{C} \mathfrak{h} ^{ - \varepsilon }.
\end{equation}
\end{enumerate}
\end{cor}
\begin{cproof}{cor:main:conv}
Throughout this proof let $\delta \in (\varepsilon, \nicefrac{1}{9})$. \Nobs that $\frac{9 \delta + 3 }{4} < 1$. This ensures that there exist $\alpha \in (\nicefrac{3}{4}, 1)$, $\beta \in (\alpha + 2, \infty )$ which satisfy
\begin{equation}
\delta < \min \cu*{ \tfrac{ 4 \alpha - 3 }{9} , \tfrac{ \beta - \alpha - 2 }{2} } \qquad \text{and} \qquad \varepsilon <  \delta - 2 ( 1 - \alpha ).
\end{equation}
\cref{theo:main:conv} and the fact that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $\mathfrak{h}^{- \delta} \le \mathfrak{h}^{- \varepsilon}$ hence establish \cref{cor:main:conv:item1,cor:main:conv:item2,cor:main:conv:item3}.
\end{cproof}









\section{ANNs with rectified linear unit (ReLU) activation}
\label{sec:anns_with_relu}

The main result of this section is \cref{cor:main} below, which in particular implies \cref{theorem_intro} from the introduction.

\subsection{Differentiability of the risk function}

\subsubsection{ANNs with density}

We first introduce in \cref{setting1} the framework for shallow ANNs with ReLU activation which will be employed throughout this section.


\begin{setting}\label{setting1}
Let $\scra, c \in \mathbb{R}$, $\scrb \in (\max\{\scra, 0\}, \infty)$, $\mathfrak{h} \in \mathbb{N}$, $v = (v_1, \ldots, v_{\mathfrak{h}}) \in \mathbb{R}^{\mathfrak{h}}$, $f \in C^1(\mathbb{R}, \allowbreak \mathbb{R})$, $\mathfrak{p}, \mathfrak{c} \in C(\mathbb{R}, \mathbb{R})$ satisfy for all $x \in \mathbb{R}$ that 
\begin{equation}
\textstyle f'(x) > 0 = f(\scra), \qquad \mathfrak{p}^{-1}(\mathbb{R} \backslash \{0\}) = (\scra, \scrb), \qquad \text{and} \qquad \mathfrak{c}(x) = \min\{\max\{x, \scra\}, \scrb\},
\end{equation}
for every $\theta = (\theta_1, \ldots, \theta_{\mathfrak{h}}) \in \mathbb{R}^{\mathfrak{h}}$ let $\mathcal{N}^{\theta} \in C(\mathbb{R}, \mathbb{R})$ satisfy for all $x \in \mathbb{R}$ that $\mathcal{N}^{\theta}(x) = c + \sum_{i = 1}^{\mathfrak{h}} v_j \max\{x - \theta_{j}, 0\}$, let $\mathcal{L} \colon \mathbb{R}^{\mathfrak{h}} \to \mathbb{R}$ satisfy for all $\theta \in \mathbb{R}^{\mathfrak{h}}$ that
\begin{equation}
\textstyle \mathcal{L}(\theta) = \int_{\mathbb{R}} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x,
\end{equation}
and let $\mathcal{G} = (\mathcal{G}_1, \ldots, \mathcal{G}_{\mathfrak{h}}) \colon \mathbb{R}^{\mathfrak{h}} \to \mathbb{R}^{\mathfrak{h}}$ satisfy for all $\theta \in \{\vartheta \in \mathbb{R}^{\mathfrak{h}} \colon \mathcal{L} \text{ is differentiable at } \vartheta\}$ that
\begin{equation}
\textstyle \mathcal{G}(\theta) = - (\nabla \mathcal{L})(\theta). 
\end{equation}
\end{setting}

In \cref{setting1} the outer weight parameters $v_i \in \mathbb{R}$ and the outer bias $c \in \mathbb{R}$ are fixed real numbers, and the inner biases $\theta_i \in \mathbb{R}$ are the variable parameters.
The inner weights are for the moment assumed to be $1$; however, by homogeneity of the ReLU activation function $\mathbb{R} \ni x \mapsto \max \cu{x , 0 } \in \mathbb{R}$ this is not a restrictive assumption.
For every parameter vector $\theta \in \mathbb{R}^\mathfrak{h}$ we again denote by $\mathcal{N} ^\theta \in C ( \mathbb{R} , \mathbb{R} )$ the realization function of the considered ANN with ReLU activation and by $\mathcal{L} ( \theta ) \in [0 , \infty )$ the corresponding $L^2$-risk.



\subsubsection{Derivatives of the risk function}

As before, the assumption that $\mathfrak{p}^{-1} ( \mathbb{R} \backslash \cu{0}) = (\scra, \scrb)$ can be used to show in \cref{lemma:derivatives_of_the_risk} that the risk function $\mathcal{L}$ is twice continuously differentiable.

\cfclear
\begin{lemma}\label{lemma:derivatives_of_the_risk}
Assume \cref{setting1}. Then
\begin{enumerate}[label=(\roman*)]
\item
\label{item1:lemma:derivatives_of_the_risk} it holds that $\mathcal{L} \in C^2 (\mathbb{R}^{\mathfrak{h}}, \mathbb{R})$,

\item
\label{item2:lemma:derivatives_of_the_risk} it holds for all $\theta = (\theta_1, \ldots, \theta_{\mathfrak{h}}) \in \mathbb{R}^{\mathfrak{h}}$, $j \in \{1, 2, \ldots, \mathfrak{h}\}$ that
\begin{equation}\label{eqn:1st_derivative}
\textstyle \bigl(\frac{\partial}{\partial \theta_{j}} \mathcal{L}\bigr)(\theta) = - 2 v_j \int_{\mathfrak{c}(\theta_j)}^{\scrb} (\mathcal{N}^{\theta}(x) - f(x)) \mathfrak{p}(x) \d x,
\end{equation}
and
\item
\label{item3:lemma:derivatives_of_the_risk} it holds for all $\theta = (\theta_1, \ldots, \theta_{\mathfrak{h}}) \in \mathbb{R}^{\mathfrak{h}}$, $i, j \in \{1, 2, \ldots, \mathfrak{h}\}$ that
\begin{equation}\label{eqn:2nd_derivative}
\begin{split}
\textstyle \bigl(\frac{\partial^2}{\partial \theta_i \partial \theta_j} \mathcal{L}\bigr)(\theta) & \textstyle = 2 v_i v_j \int_{\mathfrak{c}(\max\{\theta_i, \theta_j\})}^{\scrb} \mathfrak{p}(x) \, \d x + 2 v_j (\mathcal{N}^{\theta}(\theta_j) - f(\theta_j)) \mathfrak{p}(\theta_j) \mathbbm{1}_{\{j\}}(i) \mathbbm{1}_{[\scra, \scrb]}(\theta_j).
\end{split}
\end{equation}
\end{enumerate}
\end{lemma}
\begin{cproof}{lemma:derivatives_of_the_risk}
\Nobs that \cite[Lemma~14]{JentzenRiekert2022Piecewise}, the assumption that $\mathfrak{p}^{-1}(\mathbb{R} \backslash \{0\}) \allowbreak = (\scra, \allowbreak \scrb)$, and the Leibniz integral rule establish \cref{item1:lemma:derivatives_of_the_risk,item2:lemma:derivatives_of_the_risk,item3:lemma:derivatives_of_the_risk}.
\end{cproof}


\cfclear
\begin{lemma}[Critical points]\label{lemma:critical_points}
Assume \cref{setting1}, let $\theta = (\theta_1, \ldots, \theta_{\mathfrak{h}}) \in \mathcal{G}^{-1} (\{0\})$, $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathfrak{h}\}$, and let $s \colon \{1, 2, \ldots, k\} \to \{1, 2, \ldots, k\}$ satisfy for all $j \in \{1, 2, \ldots, k\}$ that $\abs{v_{s(j)}} > 0$ and $\scra \allowbreak \le \allowbreak \theta_{s(1)} \allowbreak < \theta_{s(2)} < \ldots < \theta_{s(k)} \le \scrb$. Then
\begin{equation}\label{eqn:lemma:critical_points}
\begin{split}
0 & \textstyle = \bigl[\int_{\scra}^{\theta_{s(1)}} (\mathcal{N}^{\theta}(x) - f(x))\mathfrak{p}(x) \, \d x \bigr] \mathbbm{1}_{(-\infty, \scra]}(\min_{j \in \{1, 2, \ldots, \mathfrak{h}\}}(\theta_j)) \\
& \textstyle = \int_{\theta_{s(k)}}^{\scrb} (\mathcal{N}^{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x = \sum_{j = 1}^{k - 1} \Abs{\int_{\theta_{s(j)}}^{\theta_{s(j + 1)}} (\mathcal{N}^{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x}.
\end{split}
\end{equation}
\end{lemma}
\begin{cproof}{lemma:critical_points}
\Nobs that \cref{item2:lemma:derivatives_of_the_risk} in \cref{lemma:derivatives_of_the_risk} and the assumption that $\theta = (\theta_1, \allowbreak \ldots, \allowbreak \theta_{\mathfrak{h}}) \allowbreak \in \allowbreak \mathcal{G}^{-1}(\{0\})$ establish \cref{eqn:lemma:critical_points}.
\end{cproof}



\subsection{Convergence of the gradient flow (GF) to suitable critical points}

In this section we establish convergence of the considered GF process to a critical point, and we also analyze the critical points of the risk. The main upper bound for the risk of critical points is given in \cref{cor:suitable_critical_upper_bound} below.

\subsubsection{Distinctness property of the kinks of active neurons of critical points}


\cfclear
\begin{lemma}[Distinctness of the active kinks]\label{lemma:distinctness_active_kinks}
Assume \cref{setting1}, assume for all $x \in \mathbb{R}$ that $\mathfrak{p}(x) \ge 0$, let $\mathfrak{H} \subseteq \mathbb{N}$ satisfy $\mathfrak{H} = \{1, 2, \ldots, \mathfrak{h}\}$, assume $\min_{j \in \mathfrak{H}} v_j > 0$, let $\theta = (\theta_1, \ldots, \theta_{\mathfrak{h}}) \in \mathcal{G}^{-1}(\{0\})$, and assume that $\theta$ is not a \cfadd{def:strict_saddle}\text{descending } critical point of $\mathcal{L}$ \cfload. Then it holds for all $j \in \mathfrak{H}$ with $\scra < \theta_j < \scrb$ and $\mathcal{N}^{\theta}(\theta_j) < f(\theta_j)$ that
\begin{equation}\label{eqn:lemma:distinctness_active_kinks}
\textstyle \{i \in \mathfrak{H} \backslash \{j\} \colon \theta_i = \theta_j\} = \emptyset.
\end{equation}
\end{lemma}
\begin{cproof}{lemma:distinctness_active_kinks}
\Nobs that the assumption that $\theta \in \mathcal{G}^{-1}(\{0\})$ and the assumption that $\theta$ is not a \cfadd{def:strict_saddle}\text{descending } critical point of $\mathcal{L}$ ensure that for all $v \in \mathbb{R}^{\mathfrak{h}}$ it holds that
\begin{equation}
\textstyle \scalar{v, ((\operatorname{Hess} \mathcal{L})(\theta))v} = v ((\operatorname{Hess} \mathcal{L})(\theta)) v^T \ge 0\ifnocf.
\end{equation}
\cfload[.]This implies that $(\operatorname{Hess} \mathcal{L})(\theta)$ is positive-semidefinite. The Sylvester's criterion therefore shows that\footnote{\Nobs that the Sylvester's criterion ensures that a Hermitian matrix is positive-semidefinite if and only if all of its principal minors are non-negative.} for all $i, j \in \mathfrak{H}$ with $i \neq j$ it holds that $\frac{\partial^2}{\partial \theta_j^2} \mathcal{L}(\theta) \ge 0 $ and
\begin{equation}\label{eqn:lemma:distinctness_active_kinks:minors}
\textstyle \det\!
\begin{pmatrix}
\bigl(\frac{\partial^2}{\partial \theta_i^2} \mathcal{L}\bigr)(\theta) & \bigl(\frac{\partial^2}{\partial \theta_i \partial \theta_j} \mathcal{L}\bigr)(\theta) \\[1ex]
\bigl(\frac{\partial^2}{\partial \theta_j \partial \theta_i} \mathcal{L}\bigr)(\theta) & \bigl(\frac{\partial^2}{\partial \theta_j^2} \mathcal{L}\bigr)(\theta)
\end{pmatrix}\! = \bigl[\bigl(\frac{\partial^2}{\partial \theta_i^2} \mathcal{L}\bigr)(\theta)\bigr] \bigl[\bigl(\frac{\partial^2}{\partial \theta_j^2} \mathcal{L}\bigr)(\theta)\bigr] - \bigl[\bigl(\frac{\partial^2}{\partial \theta_i \partial \theta_j} \mathcal{L}\bigr)(\theta)\bigr]^2 \ge 0.
\end{equation}
Assume that there exist $i ,j \in \mathfrak{H}$ which satisfy
\begin{equation}
\textstyle i \neq j, \qquad \scra < \theta_i = \theta_j < \scrb, \qquad \text{and} \qquad \mathcal{N}^{\theta}(\theta_j) < f(\theta_j).
\end{equation}
\Nobs that the assumption that $\min_{\mathfrak{j} \in \mathfrak{H}} v_{\mathfrak{j}} > 0$, \cref{eqn:lemma:distinctness_active_kinks:minors}, and \cref{lemma:derivatives_of_the_risk} demonstrate that
\begin{equation}
\begin{split}
\textstyle \frac{\partial^2}{\partial \theta_j^2} \mathcal{L}(\theta) & \textstyle = 2 [v_j]^2 \int_{\theta_j}^{\scrb} \mathfrak{p}(x) \, \d x + 2 v_j (\mathcal{N}^{\theta}(\theta_j) - f(\theta_j)) \mathfrak{p}(\theta_j) \\
& \textstyle = 2 v_j \bigl[v_j \int_{\theta_j}^{\scrb} \mathfrak{p}(x) \, \d x + (\mathcal{N}^{\theta}(\theta_j) - f(\theta_j)) \mathfrak{p}(\theta_j)\bigr] \ge 0
\end{split}
\end{equation}
and
\begin{equation}
\begin{split}
& \textstyle \bigl[\frac{\partial^2}{\partial \theta_i^2} \mathcal{L}(\theta)\bigr] \bigl[\frac{\partial^2}{\partial \theta_j^2} \mathcal{L}(\theta)\bigr] - \bigl[\frac{\partial^2}{\partial \theta_i \partial \theta_j} \mathcal{L}(\theta)\bigr] \bigl[\frac{\partial^2}{\partial \theta_j \partial \theta_i} \mathcal{L}(\theta)\bigr] \\
& \textstyle = 4 v_i v_j \bigl[v_j \int_{\theta_j}^{\scrb} \mathfrak{p}(x) \, \d x + (\mathcal{N}^{\theta}(\theta_j) - f(\theta_j)) \mathfrak{p}(\theta_j)\bigr] \bigl[v_i \int_{\theta_j}^{\scrb} \mathfrak{p}(x) \, \d x + (\mathcal{N}^{\theta}(\theta_j) - f(\theta_j)) \mathfrak{p}(\theta_j)\bigr] \\
& \textstyle \quad - 4 [v_i v_j]^2 \bigl[\int_{\theta_j}^{\scrb} \mathfrak{p}(x) \, \d x\bigr]^2 \\
& \textstyle = 4 v_i v_j (\mathcal{N}^{\theta}(\theta_j) - f(\theta_j)) \mathfrak{p}(\theta_j) \bigl[(v_i + v_j) \int_{\theta_j}^{\scrb} \mathfrak{p}(x) \, \d x + (\mathcal{N}^{\theta}(\theta_j) - f(\theta_j)) \mathfrak{p}(\theta_j)\bigr] \ge 0.
\end{split}
\end{equation}
This contradicts to the assumption that $\mathcal{N}^{\theta}(\theta_j) < f(\theta_j)$. Hence, we obtain that for all $l \in \mathfrak{H}$ with $\scra < \theta_l < \scrb$ and $\mathcal{N}^{\theta}(\theta_l) < f(\theta_l)$ it holds that
\begin{equation}
\textstyle \{k \in \mathfrak{H} \backslash \{l\} \colon \theta_k = \theta_l\} = \emptyset.
\end{equation}
\end{cproof}


\subsubsection{Upper bound for the risk of suitable critical points}

Since the weight parameters $v_i$ are assumed to be positive, the realization function $\mathcal{N}^\theta$ is always convex. Thus we need the target function $f$ to be convex as well in \cref{lemma:general_L2_risk_bound} and subsequent results in order for the approximation error to converge to zero.

\cfclear
\begin{lemma}\label{lemma:nonemptyset_endpoints}
Let $\mathscr{a} \in \mathbb{R}$, $\mathscr{b} \in (\mathscr{a}, \infty)$, $f \in C^1([\mathscr{a}, \mathscr{b}], \mathbb{R})$, $\mathfrak{p} \in C([\mathscr{a}, \mathscr{b}], \mathbb{R})$ satisfy for all $x \in [\mathscr{a}, \mathscr{b}]$ that $f'(x) > 0 \le \mathfrak{p}(x)$ and $\int_{\scra}^{\scrb} \abs{\mathfrak{p}(y)} \, \d y > 0$, assume that $f$ is strictly convex\footnote{\Nobs that for all $\scra \in \mathbb{R}$, $\scrb \in (\scra, \infty)$ and all functions $f \colon [\scra, \scrb] \to \mathbb{R}$ it holds that $f$ is strictly convex if and only if it holds for all $x, y \in [\scra, \scrb]$, $\lambda \in (0, 1)$ with $x \neq y$ that $f(\lambda x + (1 - \lambda) y) < \lambda f(x) + (1 - \lambda) f(y)$.}, let $g \colon [\mathscr{a}, \mathscr{b}] \to \mathbb{R}$ be affine linear, and assume $\int_{\mathscr{a}}^{\mathscr{b}} (f(x) - g(x)) \mathfrak{p}(x) \, \d x = 0$. Then
\begin{equation}
\textstyle \{x \in \{\mathscr{a}, \mathscr{b}\} \colon g(x) < f(x)\} \neq \emptyset.
\end{equation}
\end{lemma}
\begin{cproof}{lemma:nonemptyset_endpoints}
For the sake of contradiction we assume that 
\begin{equation}\label{eqn:lemma:nonemptyset_endpoints:contradict}
\textstyle g(\scra) \ge f(\scra) \qquad \text{and} \qquad g(\scrb) \ge f(\scrb).
\end{equation}
\Nobs that \cref{eqn:lemma:nonemptyset_endpoints:contradict}, the assumption that $f$ is strictly convex, and the assumption that $g$ is affine linear ensure that for all $x \in (\scra, \scrb)$ it holds that $g(x) > f(x)$. Combining this with the assumption that $\forall\, x \in [\scra, \scrb] \colon \mathfrak{p}(x) \ge 0$ and the assumption that $\int_{\scra}^{\scrb} \abs{\mathfrak{p}(y)} \, \d y > 0$ shows that $\int_{\mathscr{a}}^{\mathscr{b}} (f(x) - g(x)) \mathfrak{p}(x) \, \d x < 0$. This contradicts to the assumption that $\int_{\mathscr{a}}^{\mathscr{b}} (f(x) - g(x)) \mathfrak{p}(x) \, \d x = 0$. Therefore, we obtain that $\{x \in \{\mathscr{a}, \mathscr{b}\} \colon g(x) < f(x)\} \neq \emptyset$.
\end{cproof}


\cfclear
\begin{lemma}\label{lemma:general_L2_risk_bound}
Let $\mathscr{a} \in \mathbb{R}$, $\mathscr{b} \in (\mathscr{a}, \infty)$, $f \in C^1([\mathscr{a}, \mathscr{b}], \mathbb{R})$, $\mathfrak{p} \in C([\mathscr{a}, \mathscr{b}], \mathbb{R})$ satisfy for all $x \in [\mathscr{a}, \mathscr{b}]$ that $f'(x) > 0 \le \mathfrak{p}(x)$ and $\int_{\scra}^{\scrb} \abs{\mathfrak{p}(y)} \, \d y > 0$, assume that $f$ is strictly convex, let $g \colon [\mathscr{a}, \mathscr{b}] \to \mathbb{R}$ be affine linear, assume that $\int_{\mathscr{a}}^{\mathscr{b}} (f(x) - g(x)) \mathfrak{p}(x) \, \d x = 0$, and let $\lambda \in \mathbb{R}$ satisfy $\lambda = \max_{x \in \{y \in \{\mathscr{a}, \mathscr{b}\} \colon g(y) < f(y)\}} \allowbreak \abs{f(x) - g(x)}$ (cf.\ \cref{lemma:nonemptyset_endpoints}). Then
\begin{equation}\label{eqn:lemma:general_L2_risk_bound}
\textstyle \int_{\mathscr{a}}^{\mathscr{b}} (f(x) - g(x))^2 \mathfrak{p}(x) \, \d x \le \lambda [\lambda + \sup_{x \in [\mathscr{a}, \mathscr{b}]} f'(x)][\sup_{x \in [\mathscr{a}, \mathscr{b}]} \mathfrak{p}(x)] (\mathscr{b} - \mathscr{a}).
\end{equation}
\end{lemma}
\begin{cproof}{lemma:general_L2_risk_bound}
Throughout this proof let $L \in (0, \infty)$ satisfy $L = \sup_{x \in [\mathscr{a}, \mathscr{b}]} f'(x)$, let $\sigma \in \mathbb{R}$ satisfy $\sigma = \sup_{x \in [\mathscr{a}, \mathscr{b}]} \mathfrak{p}(x)$, and let $A, B \in \mathbb{R}$ satisfy for all $x \in [\mathscr{a}, \mathscr{b}]$ that $g(x) = A x + B$. \Nobs that the assumption that $f$ is strictly convex ensures that for all $x, y \in [\scra, \scrb]$ it holds that
\begin{equation}\label{eqn:convex_f-n}
\textstyle f(y) \ge f'(x) (y - x) + f(x).
\end{equation}
In the following we distinguish between the case $(g(\mathscr{a}) < f(\mathscr{a})) \wedge (g(\mathscr{b}) \ge f(\mathscr{b}))$, the case $(g(\mathscr{a}) \ge f(\mathscr{a})) \wedge (g(\mathscr{b}) < f(\mathscr{b}))$, and the case $(g(\mathscr{a}) < f(\mathscr{a})) \wedge (g(\mathscr{b}) < f(\mathscr{b}))$. First we prove \cref{eqn:lemma:general_L2_risk_bound} in the case 
\begin{equation}\label{eqn:lemma:general_L2_risk_bound:case_1}
\textstyle (g(\mathscr{a}) < f(\mathscr{a})) \wedge (g(\mathscr{b}) \ge f(\mathscr{b})).
\end{equation}
Let $q \in [\mathscr{a}, \mathscr{b})$ satisfy $g(q) = f(q)$. \Nobs that \cref{eqn:lemma:general_L2_risk_bound:case_1} and the assumption that $f$ is strictly convex prove that
\begin{equation}
\textstyle \forall\, x \in [\mathscr{a}, q) \colon g(x) < f(x) \qquad \text{and} \qquad \forall\, x \in (q, \mathscr{b}) \colon g(x) > f(x).
\end{equation}
Combining this with \cref{eqn:convex_f-n} shows that
\begin{equation}
\begin{split}
& \textstyle \sup_{x \in [\mathscr{a}, q]} \abs{g(x) - f(x)} \\
& \textstyle = f(\mathscr{a}) - g(\mathscr{a}) \ge  f'(q)(\mathscr{a} - q) + f(q) - g(\mathscr{a}) \\
& \textstyle = f'(q)(\mathscr{a} - q) + g(q) - g(\mathscr{a}) = (A - f'(q)) (q - \mathscr{a}) \ge 0
\end{split}
\end{equation}
and
\begin{equation}
\begin{split}
& \textstyle \sup_{x \in [q, \mathscr{b}]} \abs{g(x) - f(x)} \\
& \textstyle \le g(\mathscr{b}) - [f'(q)(\mathscr{b} - q) + f(q)] = g(\mathscr{b}) - g(q) - f'(q)(\mathscr{b} - q) \\
& \textstyle = (A - f'(q)) (\mathscr{b} - q).
\end{split}
\end{equation}
The assumption that $\int_{\mathscr{a}}^{\mathscr{b}} (f(x) - g(x)) \mathfrak{p}(x) \, \d x = 0$ therefore ensures that
\begin{equation}
\textstyle \int_{\mathscr{a}}^{q} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le \abs{g(\mathscr{a}) - f(\mathscr{a})}^2 (q - \mathscr{a}) \sigma \le \lambda^2 \sigma (q - \mathscr{a})
\end{equation}
and
\begin{equation}
\begin{split}
& \textstyle \int_{q}^{\mathscr{b}} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \\
& \textstyle \le (A - f'(q)) (\mathscr{b} - q) \int_{q}^{\mathscr{b}} (g(x) - f(x)) \mathfrak{p}(x) \, \d x \\
& \textstyle = (A - f'(q)) (\mathscr{b} - q) \int_{\mathscr{a}}^{q} (f(x) - g(x)) \mathfrak{p}(x) \, \d x \\
& \textstyle \le (A - f'(q)) (\mathscr{b} - q) (f(\mathscr{a}) - g(\mathscr{a})) (q - \mathscr{a}) \sigma \\
& \textstyle \le \abs{f(\mathscr{a}) - g(\mathscr{a})}^2 (\mathscr{b} - q) \sigma \le \lambda^2 \sigma (\mathscr{b} - q).
\end{split}
\end{equation}
Hence, we obtain that
\begin{equation}
\begin{split}
& \textstyle \int_{\mathscr{a}}^{\mathscr{b}} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \\
& \textstyle = \int_{\mathscr{a}}^{q} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x + \int_{q}^{\mathscr{b}} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \\
& \le \lambda^2 \sigma (q - \mathscr{a}) + \lambda^2 \sigma (\mathscr{b} - q) = \lambda^2 \sigma (\mathscr{b} - \mathscr{a}).
\end{split}
\end{equation}
This proves \cref{eqn:lemma:general_L2_risk_bound} in the case $(g(\mathscr{a}) < f(\mathscr{a})) \wedge (g(\mathscr{b}) \ge f(\mathscr{b}))$. Next we prove \cref{eqn:lemma:general_L2_risk_bound} in the case 
\begin{equation}\label{eqn:lemma:general_L2_risk_bound:case_2}
\textstyle (g(\mathscr{a}) \ge f(\mathscr{a})) \wedge (g(\mathscr{b}) < f(\mathscr{b})).
\end{equation}
Let $q \in (\mathscr{a}, \mathscr{b}]$ satisfy $g(q) = f(q)$. \Nobs that \cref{eqn:lemma:general_L2_risk_bound:case_2} and the assumption that $f$ is strictly convex assure that
\begin{equation}
\textstyle \forall\, x \in (\mathscr{a}, q) \colon g(x) > f(x) \qquad \text{and} \qquad \forall\, x \in (q, \mathscr{b}] \colon g(x) < f(x).
\end{equation}
Combining this with \cref{eqn:convex_f-n} proves that
\begin{equation}
\begin{split}
& \textstyle \sup_{x \in [\mathscr{a}, q]} \abs{g(x) - f(x)} \\
& \textstyle \le g(\mathscr{a}) - f'(q)(\mathscr{a} - q) - f(q) = g(\mathscr{a}) - g(q) - f'(q)(\mathscr{a} - q) \\
& \textstyle = (f'(q) - A) (q - \mathscr{a})
\end{split}
\end{equation}
and
\begin{equation}
\begin{split}
& \textstyle \sup_{x \in [q, \mathscr{b}]} \abs{g(x) - f(x)} \\
& \textstyle = f(\mathscr{b}) - g(\mathscr{b}) \ge f'(q)(\mathscr{b} - q) + f(q) - g(\mathscr{b}) \\
& = f'(q)(\mathscr{b} - q) + g(q) - g(\mathscr{b}) \\
& \textstyle = (f'(q) - A) (\mathscr{b} - q) \ge 0.
\end{split}
\end{equation}
The assumption that $\int_{\mathscr{a}}^{\mathscr{b}} (f(x) - g(x)) \mathfrak{p}(x) \, \d x = 0$ hence demonstrates that
\begin{equation}
\begin{split}
& \textstyle \int_{\mathscr{a}}^{q} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \\
& \textstyle \le (f'(q) - A) (q - \mathscr{a}) \int_{\mathscr{a}}^{q} (g(x) - f(x)) \mathfrak{p}(x) \, \d x \\
& \textstyle = (f'(q) - A) (q - \mathscr{a}) \int_{q}^{\mathscr{b}} (f(x) - g(x)) \mathfrak{p}(x) \, \d x \\
& \textstyle \le (f'(q) - A) (q - \mathscr{a}) (f(\mathscr{b}) - g(\mathscr{b})) (\mathscr{b} - q) \sigma \\
& \textstyle \le \abs{f(\mathscr{b}) - g(\mathscr{b})}^2 (q - \mathscr{a}) \sigma \le \lambda^2 \sigma (q - \mathscr{a})
\end{split}
\end{equation}
and
\begin{equation}
\textstyle \int_{q}^{\mathscr{b}} (g(x) - f(x))^2 \, \d x \le \abs{g(\mathscr{b}) - f(\mathscr{b})}^2 (\mathscr{b} - q) \sigma \le \lambda^2 \sigma (\mathscr{b} - q).
\end{equation}
Therefore, we obtain that
\begin{equation}
\begin{split}
& \textstyle \int_{\mathscr{a}}^{\mathscr{b}} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \\
& \textstyle = \int_{\mathscr{a}}^{q} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x + \int_{q}^{\mathscr{b}} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \\
& \le \lambda^2 \sigma (q - \mathscr{a}) + \lambda^2 \sigma (\mathscr{b} - q) = \lambda^2 \sigma (\mathscr{b} - \mathscr{a}).
\end{split}
\end{equation}
This proves \cref{eqn:lemma:general_L2_risk_bound} in the case $(g(\mathscr{a}) \ge f(\mathscr{a})) \wedge (g(\mathscr{b}) < f(\mathscr{b}))$. In the next step we prove \cref{eqn:lemma:general_L2_risk_bound} in the case
\begin{equation}\label{eqn:lemma:general_L2_risk_bound:case_3}
\textstyle (g(\mathscr{a}) < f(\mathscr{a})) \wedge (g(\mathscr{b}) < f(\mathscr{b})).
\end{equation}

Let $q_1, q_2 \in (\mathscr{a}, \mathscr{b})$ satisfy $q_1 < q_2$ and $\abs{g(q_1) - f(q_1)} + \abs{g(q_2) - f(q_2)} = 0$. \Nobs that \cref{eqn:lemma:general_L2_risk_bound:case_3} and the assumption that $f$ is strictly convex show that
\begin{equation}
\textstyle \forall\, x \in [\mathscr{a}, q_1) \cup (q_2, \mathscr{b}] \colon g(x) < f(x) \qquad \text{and} \qquad \forall\, x \in (q_1, q_2) \colon g(x) > f(x).
\end{equation}
Combining this with \cref{eqn:convex_f-n} ensures that
\begin{equation}
\begin{split}
& \textstyle \sup_{x \in [\mathscr{a}, q_1]} \abs{g(x) - f(x)} \\
& \textstyle = f(\mathscr{a}) - g(\mathscr{a}) \ge f'(q_1)(\mathscr{a} - q_1) + f(q_1) - g(\mathscr{a}) \\
& \textstyle = f'(q_1)(\mathscr{a} - q_1) + g(q_1) - g(\mathscr{a}) = (A - f'(q_1)) (q_1 - \mathscr{a}) \ge 0,
\end{split}
\end{equation}
\begin{equation}
\begin{split}
& \textstyle \sup_{x \in [q_1, q_2]} \abs{g(x) - f(x)} \\
& \textstyle \le g(q_1) - f'(q_2) (q_1 - q_2) - f(q_2) = g(q_1) - g(q_2) - f'(q_2) (q_1 - q_2) \\
& \textstyle = (f'(q_2) - A)(q_2 - q_1),
\end{split}
\end{equation}
\begin{equation}
\begin{split}
& \textstyle \sup_{x \in [q_1, q_2]} \abs{g(x) - f(x)} \\
& \textstyle \le g(q_2) - f'(q_1) (q_2 - q_1) - f(q_1) = g(q_2) - g(q_1) - f'(q_1) (q_2 - q_1) \\
& \textstyle = (A - f'(q_1))(q_2 - q_1),
\end{split}
\end{equation}
and
\begin{equation}
\begin{split}
& \textstyle \sup_{x \in [q_2, \mathscr{b}]} \abs{g(x) - f(x)} \\
& \textstyle = f(\mathscr{b}) - g(\mathscr{b}) \ge f'(q_2)(\mathscr{b} - q_2) + f(q_2) - g(\mathscr{b}) \\
& \textstyle = f'(q_2)(\mathscr{b} - q_2) + g(q_2) - g(\mathscr{b}) = (f'(q_2) - A) (\mathscr{b} - q_2) \ge 0.
\end{split}
\end{equation}
The assumption that $\int_{\mathscr{a}}^{\mathscr{b}} (f(x) - g(x)) \mathfrak{p}(x) \, \d x = 0$ therefore proves that
\begin{equation}
\textstyle \int_{\mathscr{a}}^{q_1} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le (f(\mathscr{a}) - g(\mathscr{a}))^2 (q_1 - \mathscr{a}) \sigma \le \lambda^2 \sigma (q_1 - \mathscr{a}),
\end{equation}
\begin{equation}
\begin{split}
& \textstyle \int_{q_1}^{q_2} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le (f'(q_2) - A) (q_2 - q_1) \int_{q_1}^{q_2} (g(x) - f(x)) \mathfrak{p}(x) \, \d x \\
& \textstyle = (f'(q_2) - A) (q_2 - q_1) \bigl[\int_{\mathscr{a}}^{q_1} (f(x) - g(x)) \mathfrak{p}(x) \, \d x + \int_{q_2}^{\mathscr{b}} (f(x) - g(x)) \mathfrak{p}(x) \, \d x\bigr] \\
& \textstyle \le (f'(q_2) - A) (q_2 - q_1) [(f(\mathscr{a}) - g(\mathscr{a})) (q_1 - \mathscr{a}) \sigma \\
& \textstyle \quad + (f(\mathscr{b}) - g(\mathscr{b})) (\mathscr{b} - q_2) \sigma] \\
& \textstyle \le L (q_2 - q_1) (\lambda (q_1 - \mathscr{a}) \sigma + \lambda (\mathscr{b} - q_2) \sigma) \le L \lambda \sigma (q_2 - q_1),
\end{split}
\end{equation}
and
\begin{equation}
\textstyle \int_{q_2}^{\mathscr{b}} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le (f(\mathscr{b}) - g(\mathscr{b}))^2 (\mathscr{b} - q_2) \sigma \le \lambda^2 \sigma (\mathscr{b} - q_2).
\end{equation}
Hence, we obtain that
\begin{equation}
\begin{split}
& \textstyle \int_{\mathscr{a}}^{\mathscr{b}} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \\
& \textstyle = \int_{\mathscr{a}}^{q_1} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x  + \int_{q_1}^{q_2} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x + \int_{q_2}^{\mathscr{b}} (g(x) - f(x))^2 \mathfrak{p}(x) \, \d x \\
& \textstyle \le \lambda^2 \sigma (q_1 - \mathscr{a}) + L \lambda \sigma (q_2 - q_1) + \lambda^2 \sigma (\mathscr{b} - q_2) \le \lambda \sigma (L + \lambda) (\mathscr{b} - \mathscr{a}).
\end{split}
\end{equation}
This proves \cref{eqn:lemma:general_L2_risk_bound} in the case $(g(\mathscr{a}) < f(\mathscr{a})) \wedge (g(\mathscr{b}) < f(\mathscr{b}))$.
\end{cproof}

In \cref{lemma:critical_upper_bound} we use the fact that $f$ is strictly convex and the realization is piecewise linear, and thus it is essentially sufficient to estimate the error at the breakpoints $\theta_i$.

\cfclear
\begin{lemma}[Upper bound for the risk of critical points]\label{lemma:critical_upper_bound}
Assume \cref{setting1}, assume for all $x \in \mathbb{R}$ that $\mathfrak{p}(x) \ge 0$, assume that $f$ is strictly convex, let $L \in (0, \infty)$ satisfy $L = \allowbreak \sup_{x \in [\scra, \scrb]} f'(x)$, let $\mathfrak{H} \subseteq \mathbb{N}$, $\theta = (\theta_1, \ldots, \theta_{\mathfrak{h}}) \in \mathcal{G}^{-1}(\{0\})$, $\lambda \in \mathbb{R}$ satisfy $\mathfrak{H} = \{1, 2, \ldots, \mathfrak{h}\}$ and 
\begin{equation}
\textstyle \lambda = \max_{j \in \{i \in \mathfrak{H} \colon [\scra < \theta_{i} < \scrb \text{ and } \mathcal{N}^{\theta}(\theta_i) < f(\theta_i)]\}} \abs{\mathcal{N}^{\theta}(\theta_j) - f(\theta_j)},
\end{equation}
assume $\min_{j \in \mathfrak{H}} v_j > 0$, and assume $\mathcal{N}^{\theta}(\scrb) \ge f(\scrb)$. Then it holds that
\begin{equation}\label{eqn:lemma:critical_upper_bound}
\textstyle \mathcal{L}(\theta) \le \max\{\lambda, \abs{c}\} (L + \max\{\lambda, \abs{c}\}) [\sup_{x \in [\scra, \scrb]} \mathfrak{p}(x)] (\scrb - \scra).
\end{equation}
\end{lemma}
\begin{cproof}{lemma:critical_upper_bound}
Throughout this proof let $k \in \mathbb{N}$, $s \colon \{1, 2, \ldots, k\} \to \{1, 2, \ldots, k\}$ satisfy
\begin{equation}
\textstyle k = \#((\cup_{j \in \mathfrak{H}}\{\theta_j\}) \cap (\scra, \scrb)) \qquad \text{and} \qquad \scra < \theta_{s(1)} < \theta_{s(2)} < \ldots < \theta_{s(k)} < \scrb
\end{equation}
and let $\sigma \in \mathbb{R}$ satisfy $\sigma = \sup_{x \in [\scra, \scrb]} \mathfrak{p}(x)$. \Nobs that \cref{lemma:critical_points} and the assumption that $\mathcal{N}^{\theta}(\scrb) \allowbreak \ge f(\scrb)$ ensure that $\mathcal{N}^{\theta}(\theta_{s(k)}) \allowbreak < f(\theta_{s(k)})$ and
\begin{equation}
\textstyle \Abs{\int_{\theta_{s(k)}}^{\scrb} (\mathcal{N}^{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x} + \sum_{j = 1}^{k - 1} \Abs{\int_{\theta_{s(j)}}^{\theta_{s(j + 1)}} (\mathcal{N}^{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x} = 0.
\end{equation}
\cref{lemma:general_L2_risk_bound} and the fact that for all $j \in \{1, 2, \ldots, k - 1\}$ it holds that $[\theta_{s(j)}, \theta_{s(j + 1)}] \ni x \mapsto \mathcal{N}^{\theta}(x) \allowbreak \in \allowbreak \mathbb{R}$ and $[\theta_{s(k)}, \scrb] \ni x \mapsto \mathcal{N}^{\theta}(x) \in \mathbb{R}$ are affine linear therefore show that
\begin{equation}\label{eqn:lemma:critical_upper_bound:partial_bound_j}
\textstyle \forall\, j \in \{1, 2, \ldots, k - 1\} \colon \int_{\theta_{s(j)}}^{\theta_{s(j + 1)}} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le \lambda \sigma (L + \lambda) (\theta_{s(j + 1)} - \theta_{s(j)}),
\end{equation}
and
\begin{equation}\label{eqn:lemma:critical_upper_bound:partial_bound_k}
\textstyle \int_{\theta_{s(k)}}^{\scrb} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le \lambda^2 \sigma (\scrb - \theta_{s(k)}).
\end{equation}
In the next step we prove that
\begin{equation}\label{eqn:lemma:critical_upper_bound:partial_bound_1}
\textstyle \int_{\scra}^{\theta_{s(1)}} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le \max\{\lambda, \abs{c}\} (L + \max\{\lambda, \abs{c}\}) (\theta_{s(1)} - \scra) \sigma.
\end{equation}
In the following we distinguish between the case $\#\{j \in \mathfrak{H} \colon \theta_j \le \scra\} > 0$ and the case $\#\{j \in \mathfrak{H} \colon \theta_j \le \scra\} = 0$. First we prove \cref{eqn:lemma:critical_upper_bound:partial_bound_1} in the case $\#\{j \in \mathfrak{H} \colon \theta_j \le \scra\} > 0$. \Nobs that the assumption that $\#\{j \in \mathfrak{H} \colon \theta_j \le \scra\} > 0$ and \cref{lemma:critical_points} ensure that $\int_{\scra}^{\theta_{s(1)}} (\mathcal{N}^{\theta}(x) - f(x)) \mathfrak{p}(x) \, \d x = 0$. Combining this with \cref{lemma:general_L2_risk_bound} shows that
\begin{equation}
\textstyle \int_{\scra}^{\theta_{s(1)}} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le
\begin{cases}
\lambda (L + \lambda) (\theta_{s(1)} - \scra) \sigma & \colon \mathcal{N}^{\theta}(\scra) \ge 0 \\
\max\{\lambda, \abs{c}\} (L + \max\{\lambda, \abs{c}\}) (\theta_{s(1)} - \scra) \sigma & \colon \mathcal{N}^{\theta}(\scra) < 0.
\end{cases}
\end{equation}
This proves \cref{eqn:lemma:critical_upper_bound:partial_bound_1} in the case $\#\{j \in \mathfrak{H} \colon \theta_j \le \scra\} > 0$. Next we prove \cref{eqn:lemma:critical_upper_bound:partial_bound_1} in the case $\#\{j \in \mathfrak{H} \colon \theta_j \le \scra\} = 0$. \Nobs that the assumption that $\#\{j \in \mathfrak{H} \colon \theta_j \le \scra\} = 0$ shows that $\forall\, x \in [\scra, \theta_{s(1)}] \colon \mathcal{N}^{\theta}(x) = c$. Combining this with the assumption that $f$ is strictly convex ensures that
\begin{equation}
\textstyle \sup_{x \in [\scra, \theta_{s(1)}]} \abs{\mathcal{N}^{\theta}(x) - f(x)} \le
\begin{cases}
\lambda & \colon [\mathcal{N}^{\theta}(\theta_{s(1)}) < f(\theta_{s(1)})] \wedge [c < 0] \\
\max\{\lambda, \abs{c}\} & \colon [\mathcal{N}^{\theta}(\theta_{s(1)}) < f(\theta_{s(1)})] \wedge [c \ge 0] \\
\abs{c} & \colon \mathcal{N}^{\theta}(\theta_{s(1)}) \ge f(\theta_{s(1)}).
\end{cases}
\end{equation}
Hence, we obtain that
\begin{equation}
\textstyle \int_{\scra}^{\theta_{s(1)}} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le [\max\{\lambda, \abs{c}\}]^2 (\theta_{s(1)} - \scra) \sigma.
\end{equation}
This proves \cref{eqn:lemma:critical_upper_bound:partial_bound_1} in the case $\#\{j \in \mathfrak{H} \colon \theta_j \le \scra\} = 0$. \Nobs that \cref{eqn:lemma:critical_upper_bound:partial_bound_j,eqn:lemma:critical_upper_bound:partial_bound_1,eqn:lemma:critical_upper_bound:partial_bound_k} demonstrate that
\begin{equation}
\begin{split}
& \textstyle \int_{\scra}^{\scrb} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x \\
& \textstyle = \int_{\scra}^{\theta_{s(1)}} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x + \bigl(\sum_{j = 1}^{k - 1} \bigl[\int_{\theta_{s(j)}}^{\theta_{s(j + 1)}} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x\bigr]\bigr) \\
& \textstyle \quad + \int_{\theta_{s(k)}}^{\scrb} (\mathcal{N}^{\theta}(x) - f(x))^2 \mathfrak{p}(x) \, \d x \le \max\{\lambda, \abs{c}\} \sigma (L + \max\{\lambda, \abs{c}\}) (\theta_{s(1)} - \scra) \\
& \textstyle \quad + \sum_{j = 1}^{k - 1} [\lambda \sigma (L + \lambda) (\theta_{s(j + 1)} - \theta_{s(j)})] + \lambda^2 \sigma (\scrb - \theta_{s(k)}) \\
& \textstyle = \max\{\lambda, \abs{c}\} \sigma (L + \max\{\lambda, \abs{c}\}) (\theta_{s(1)} - \scra) + \lambda \sigma (L + \lambda) (\theta_{s(k)} - \theta_{s(1)}) + \lambda^2 \sigma (\scrb - \theta_{s(k)}) \\
& \textstyle \le \max\{\lambda, \abs{c}\} \sigma (L + \max\{\lambda, \abs{c}\}) (\scrb - \scra).
\end{split}
\end{equation}
This establishes \cref{eqn:lemma:critical_upper_bound}.
\end{cproof}


\cfclear
\begin{corollary}[Upper bound for the risk of suitable critical points]\label{cor:suitable_critical_upper_bound}
Assume \cref{setting1}, assume for all $x \in \mathbb{R}$ that $\mathfrak{p}(x) \ge 0$, assume that $f$ is strictly convex, let $L \in (0, \infty)$ satisfy $L = \allowbreak \sup_{x \in [\scra, \scrb]} f'(x)$, let $\mathfrak{H} \subseteq \mathbb{N}$, $\theta = (\theta_1, \ldots, \theta_{\mathfrak{h}}) \in \mathcal{G}^{-1}(\{0\})$ satisfy $\mathfrak{H} = \{1, 2, \ldots, \mathfrak{h}\}$, assume $\min_{j \in \mathfrak{H}} v_j > 0$, assume $\mathcal{N}^{\theta}(\scrb) \ge f(\scrb)$, and assume that $\theta$ is not a \cfadd{def:strict_saddle}\text{descending } critical point of $\mathcal{L}$ \cfload. Then
\begin{enumerate}[label=(\roman*)]
\item
\label{item1:cor:suitable_critical_upper_bound} it holds for all $j \in \mathfrak{H}$ with $\scra < \theta_j < \scrb$ and $\mathcal{N}^{\theta}(\theta_j) < f(\theta_j)$ that
\begin{equation}
\textstyle 0 < f(\theta_j) - \mathcal{N}^{\theta}(\theta_j) \le v_j,
\end{equation}


\item
\label{item2:cor:suitable_critical_upper_bound} it holds that
\begin{equation}
\textstyle \max_{j \in \{i \in \mathfrak{H} \colon [\scra < \theta_i < \scrb \text{ and } \mathcal{N}^{\theta}(\theta_i) < f(\theta_i)]\}} \abs{\mathcal{N}^{\theta}(\theta_j) - f(\theta_j)} \le \max_{j \in \mathfrak{H}} v_j,
\end{equation}

and
\item
\label{item3:cor:suitable_critical_upper_bound} it holds that
\begin{equation}\label{eqn:cor:suitable_critical_upper_bound}
\textstyle \mathcal{L}(\theta) \le [\max\{\max_{j \in \mathfrak{H}} v_j, \abs{c}\}] (L + [\max\{\max_{j \in \mathfrak{H}} v_j, \abs{c}\}]) [\sup_{x \in [\scra, \scrb]} \mathfrak{p}(x)] (\scrb - \scra).
\end{equation}
\end{enumerate}
\end{corollary}
\begin{cproof}{cor:suitable_critical_upper_bound}
Throughout this proof let $k \in \mathbb{N}$, $s \colon \{1, 2, \ldots, k\} \to \{1, 2, \ldots, k\}$, $\theta_{s(k + 1)} \allowbreak \in \allowbreak \mathbb{R}$ satisfy
\begin{equation}
\textstyle k = \# ((\cup_{j \in \mathfrak{H}} \{\theta_j\}) \cap (\scra, \scrb)) \qquad \text{and} \qquad \scra < \theta_{s(1)} < \theta_{s(2)} < \ldots < \theta_{s(k)} < \theta_{s(k + 1)} = \scrb
\end{equation}
and let $A_1, A_2, \ldots, A_{k + 1}, B_1, B_2, \ldots, B_{k + 1} \in \mathbb{R}$ satisfy
\begin{equation}
\begin{split}
& \textstyle (\forall\, x \in [\scra, \theta_{s(1)}] \colon \mathcal{N}^{\theta}(x) = A_1 x + B_1), \quad (\forall\, x \in [\theta_{s(k)}, \scrb] \colon \mathcal{N}^{\theta}(x) = A_{k + 1} x + B_{k + 1}),  \\
& \textstyle \text{and} \quad (\forall\, j \in \{1, 2, \ldots, k - 1\}, x \in [\theta_{s(j)}, \theta_{s(j + 1)}] \colon \mathcal{N}^{\theta}(x) = A_{j + 1} x + B_{j + 1}).
\end{split}
\end{equation}
\Nobs that for all $j \in \{1, 2, \ldots, k\}$ it holds that
\begin{equation}\label{eqn:cor:suitable_critical_upper_bound:slope_change}
\textstyle A_{j + 1} = A_j + \sum_{i \in \{l \in \mathfrak{H} \colon \theta_l = \theta_{s(j)}\}} v_i \qquad \text{and} \qquad B_{j + 1} = B_j - \sum_{i \in \{l \in \mathfrak{H} \colon \theta_l = \theta_{s(j)}\}} v_i \theta_i. 
\end{equation}
Next \nobs that \cref{lemma:distinctness_active_kinks} and the assumption that $\theta$ is not a \cfadd{def:strict_saddle}\text{descending } critical point of $\mathcal{L}$ ensure that for all $j \in \{1, 2, \ldots, k\}$ with $\mathcal{N}^{\theta}(\theta_{s(j)}) < f(\theta_{s(j)})$ it holds that
\begin{equation}
\textstyle \{i \in \mathfrak{H} \backslash \{j\} \colon \theta_i = \theta_{s(j)}\} = \emptyset.
\end{equation}
Combining this with \cref{eqn:cor:suitable_critical_upper_bound:slope_change} shows that for all $j \in \{1, 2, \ldots, k\}$ with $\mathcal{N}^{\theta}(\theta_{s(j)}) < f(\theta_{s(j)})$ it holds that
\begin{equation}\label{eqn:cor:suitable_critical_upper_bound:slope_change_special}
\textstyle A_{j + 1} = A_j + v_{s(j)} \qquad \text{and} \qquad B_{j + 1} = B_j - v_{s(j)} \theta_{s(j)}.
\end{equation}
Let $\mathfrak{j} \in \{1, 2, \ldots, k\}$, $q \in (\theta_{s(\mathfrak{j})}, \theta_{s(\mathfrak{j} + 1)})$ satisfy $\mathcal{N}^{\theta}(q) = f(q)$ and $\forall\, x \in [\theta_{s(\mathfrak{j})}, q) \colon \mathcal{N}^{\theta}(x) < f(x)$. \Nobs that the assumption that $f$ is strictly convex, the Lagrange MVT, and \cref{eqn:cor:suitable_critical_upper_bound:slope_change_special} assure that there exists $c \in (\theta_{s(\mathfrak{j})}, q)$ such that $[\forall\, x \in (\theta_{s(\mathfrak{j})}, q) \colon A_{\mathfrak{j}} < f'(x)]$ and
\begin{equation}
\begin{split}
0 & \textstyle < f(\theta_{s(\mathfrak{j})}) - \mathcal{N}^{\theta}(\theta_{s(\mathfrak{j})}) = f(\theta_{s(\mathfrak{j})}) - f(q) + \mathcal{N}^{\theta}(q) - \mathcal{N}^{\theta}(\theta_{s(\mathfrak{j})}) \\
& \textstyle = f'(c) (\theta_{s(\mathfrak{j})} - q) + A_{\mathfrak{j} + 1} (q - \theta_{s(\mathfrak{j})}) = (A_{\mathfrak{j} + 1} - f'(c)) (q - \theta_{s(\mathfrak{j})}) \\
& \textstyle = (A_{\mathfrak{j}} + v_{s(\mathfrak{j})} - f'(c)) (q - \theta_{s(\mathfrak{j})}) \le v_{s(\mathfrak{j})} (q - \theta_{s(\mathfrak{j})}) \le v_{s(\mathfrak{j})}.
\end{split}
\end{equation}
Therefore, we obtain that for all $j \in \mathfrak{H}$ with $\scra < \theta_j < \scrb$ and $\mathcal{N}^{\theta}(\theta_j) < f(\theta_j)$ it holds that
\begin{equation}
\textstyle 0 < f(\theta_j) - \mathcal{N}^{\theta}(\theta_j) \le v_j.
\end{equation}
This establishes \cref{item1:cor:suitable_critical_upper_bound} and \cref{item2:cor:suitable_critical_upper_bound}. Furthermore, \nobs that \cref{item2:cor:suitable_critical_upper_bound} and \cref{lemma:critical_upper_bound} establish \cref{item3:cor:suitable_critical_upper_bound}.
\end{cproof}


\subsubsection{Convergence of GF}

We next show convergence of the GF trajectory with a suitable convergence rate. Similarly to the previous section, we use the KL inequality for the risk, after having established boundedness of the trajectory.

\cfclear
\begin{lemma}[Boundedness of GF trajectory]\label{lemma:GF_boundedness}
Assume \cref{setting1}, assume $\min_{j \in \{1, 2, \ldots, \mathfrak{h}\}} v_j > 0$, assume for all $x \in \mathbb{R}$ that $\mathfrak{p}(x) \ge 0 < \int_{\mathbb{R}} \mathfrak{p}(y) \, \d y$, and let $\Theta \in C([0, \infty), \mathbb{R}^{\mathfrak{h}})$ satisfy for all $t \in [0, \allowbreak \infty)$ that
\begin{equation}\label{eqn:lemma:GF_boundedness:GF}
\textstyle \Theta_0 \in \mathbb{R}^{\mathfrak{h}} \qquad \text{and} \qquad \Theta_t = \Theta_0 + \int_0^t \mathcal{G}(\Theta_s) \d s.
\end{equation}
Then
\begin{enumerate}[label=(\roman*)]
\item
\label{item1:lemma:GF_boundedness} it holds for all $j \in \{1, 2, \ldots, \mathfrak{h}\}$ that $(\exists\, T \in [0, \infty) \colon \Theta_{j, T} > \scrb)$ implies that $(\forall\, t \in [0, \allowbreak \infty) \allowbreak \colon \allowbreak \Theta_{j, t} \allowbreak = \Theta_{j, 0} > \scrb)$,

\item
\label{item2:lemma:GF_boundedness} it holds for all $j \in \{1, 2, \ldots, \mathfrak{h}\}$ that $(\exists\, T \in [0, \infty) \colon \Theta_{j, T} \le \scrb)$ implies that for all $t \in [0, \infty)$ it holds that
\begin{equation}
\textstyle \scrb \ge \Theta_{j, t} \ge \scra - [v_j]^{-1} \bigl([\mathcal{L}(\Theta_0)]^{1/2} + \bigl[\int_{\mathbb{R}} (f(x) - c)^2 \mathfrak{p}(x) \, \d x\bigr]^{1/2}\bigr) \bigl[\int_{\mathbb{R}} \mathfrak{p}(x) \, \d x\bigr]^{-1/2},
\end{equation}

\item
\label{item3:lemma:GF_boundedness} it holds for all $j \in \{1, 2, \ldots, \mathfrak{h}\}$, $t \in [0, \infty)$ that
\begin{equation}
\textstyle \abs{\Theta_{j, t}} \le \max\Bigl\{\scrb, \abs{\Theta_{j, 0}}, \abs{\scra} + \frac{[v_j]^{-1}}{[\int_{\mathbb{R}} \mathfrak{p}(x) \, \d x]^{1/2}} \bigl([\mathcal{L}(\Theta_0)]^{1/2} + \bigl[\int_{\mathbb{R}} (f(x) - c)^2 \mathfrak{p}(x) \, \d x\bigr]^{1/2}\bigr)\Bigr\},
\end{equation}

and
\item
\label{item4:lemma:GF_boundedness} it holds for all $t \in [0, \infty)$ that
\begin{equation}
\textstyle \norm{\Theta_t} \le \mathfrak{h} (\scrb + \abs{\scra}) + \mathfrak{h}^{1/2} \norm{\Theta_0} + \frac{\sum_{j = 1}^{\mathfrak{h}} [v_{j}]^{-1}}{[\int_{\mathbb{R}} \mathfrak{p}(x) \, \d x]^{1/2}} \bigl([\mathcal{L}(\Theta_0)]^{1/2} + \bigl[\int_{\mathbb{R}} (f(x) - c)^2 \mathfrak{p}(x) \, \d x\bigr]^{1/2}\bigr).
\end{equation}
\end{enumerate}
\end{lemma}
\begin{cproof}{lemma:GF_boundedness}
Throughout this proof let $j \in \{1, 2, \ldots, \mathfrak{h}\}$. \Nobs that \cref{eqn:lemma:GF_boundedness:GF} assures that for all $t, \mathfrak{t} \in [0, \infty)$ with $\Theta_{j, t} \ge \scrb$ it holds that $[0, \infty) \ni \tau \mapsto \Theta_{j, \tau} \in \mathbb{R}$ is differentiable at $\mathfrak{t}$ and
\begin{equation}\label{eqn:lemma:GF_boundedness:Theta_j>1}
\textstyle \frac{\partial}{\partial t} \Theta_{j, t} = \mathcal{G}_j(\Theta_t) = - \frac{\partial}{\partial \Theta_{j, t}} \mathcal{L}(\Theta_t) = 2 v_j \int_{\Theta_{j, t}}^{\scrb} (\mathcal{N}^{\Theta_t}(x) - f(x)) \mathfrak{p}(x) \, \d x = 0.
\end{equation}
Assume that there exist $T \in [0, \infty)$, $\mathscr{c} \in \mathbb{R}$ which satisfy $\mathscr{c} = \Theta_{j, T} > \scrb$ and
\begin{multline}\label{eqn:lemma:GF_boundedness:assumption_inf_sup}
\textstyle \bigl[\inf\{t \in [0, T] \colon (\forall\, x \in [t, T] \colon \Theta_{j, x} > \scrb)\} > 0\bigr] \\
\textstyle \vee \bigl[\sup\{t \in [T, \infty) \colon (\forall\, x \in [T, t] \colon \Theta_{j, x} > \scrb)\} < \infty\bigr] = 1.
\end{multline}
Let $\mathbf{t}, \mathbf{T} \in [0, \infty)$ satisfy $\mathbf{t} = \inf\{t \in [0, T] \colon (\forall\, x \in [t, T] \colon \theta_{j, x} > \scrb)\}$ and $\mathbf{T} = \sup\{t \in [T, \infty) \colon (\forall\, x \in [T, t] \colon \Theta_{j, x} > \scrb)\}$. \Nobs that the fact that $[0, \infty) \ni t \mapsto \Theta_{j, t} \in \mathbb{R}$ is continuous shows that for all $t \in (\mathbf{t}, \mathbf{T})$ it holds that $0 < \mathbf{t} < T < \mathbf{T} < \infty$ and $\Theta_{j, t} > \scrb$. Combining this with \cref{eqn:lemma:GF_boundedness:Theta_j>1} and Lagrange MVT demonstrates that for all $t \in (\mathbf{t}, \mathbf{T}) \backslash \{T\}$ there exists $\mathscr{t} \in (\min\{t, T\}, \max\{t, T\})$ such that
\begin{equation}
\textstyle \Theta_{j, t} = \Theta_{j, t} - \Theta_{j, T} + \mathscr{c} = (t - T) \mathcal{G}_j(\Theta_{\mathscr{t}}) + \mathscr{c} = \mathscr{c}.
\end{equation}
This implies that for all $t \in (\mathbf{t}, \mathbf{T})$ it holds that $\Theta_{j, t} = \mathscr{c}$. The fact that $[0, \infty) \ni t \mapsto \Theta_{j, t} \in \mathbb{R}$ is continuous hence proves that for all $t \in [\mathbf{t}, \mathbf{T}]$ it holds that $\Theta_{j, t} = \mathscr{c}$. Combining this with the fact that $[0, \infty) \ni t \mapsto \Theta_{j, t} \in \mathbb{R}$ is continuous shows that there exists $\varepsilon \in (0, \infty)$ which satisfies for all $t \in [\mathbf{t} - \varepsilon, \mathbf{T} + \varepsilon]$ that $0 \le \mathbf{t} - \varepsilon$ and $\Theta_{j, t} > \scrb$. This, \cref{eqn:lemma:GF_boundedness:Theta_j>1}, and Lagrange MVT demonstrate that for all $t \in [\mathbf{t} - \varepsilon, \mathbf{T} + \varepsilon] \backslash \{T\}$ there exists $\mathscr{t} \in (\min\{t, T\}, \max\{t, T\})$ such that
\begin{equation}
\textstyle \Theta_{j, t} = \Theta_{j, t} - \Theta_{j, T} + \mathscr{c} = (t - T) \mathcal{G}_j(\Theta_{\mathscr{t}}) + \mathscr{c} = \mathscr{c}.
\end{equation}
Hence, we obtain that for all $t \in [\mathbf{t} - \varepsilon, \mathbf{T} + \varepsilon]$ it holds that $\Theta_{j, t} = \Theta_{j, T} = \mathscr{c}$. This contradicts to the assumption in \cref{eqn:lemma:GF_boundedness:assumption_inf_sup}. Therefore, we obtain that for all $t \in (0, \infty)$ it holds that $\Theta_{j, t} = \Theta_{j, 0} = \mathscr{c} > \scrb$. Combining this and the fact that $[0, \infty) \ni t \mapsto \Theta_{j, t} \in \mathbb{R}$ is continuous ensures that for all $t \in [0, \infty)$ it holds that $\Theta_{j, t} = \Theta_{j, 0} = \mathscr{c} > \scrb$. This establishes \cref{item1:lemma:GF_boundedness}. Next \nobs that \cite[Lemma~3.1]{Riekert2021ConvergenceConstTF} and the fact that for all $\mathscr{R} \in [0, \infty)$ it holds that 
\begin{equation}
\textstyle \sup_{\theta \in \mathbb{R}^{\mathfrak{h}}, \norm{\theta} \le \mathscr{R}} \norm{(\nabla \mathcal{L})(\theta)} < \infty
\end{equation}
ensure that for all $t \in [0, \infty)$ it holds that
\begin{equation}\label{eqn:lemma:GF_boundedness:Risk_bound}
\textstyle 0 \le \mathcal{L}(\Theta_t) = \mathcal{L}(\Theta_0) - \int_0^t \norm{\mathcal{G}(\Theta_s)}^2 \, \d s \le \mathcal{L}(\Theta_0).
\end{equation}
H\"{o}lder inequality and the assumption that $\min_{i \in \{1, 2, \ldots, \mathfrak{h}\}} v_i > 0$ therefore demonstrate that for all $t \in [0, \infty)$ with $\Theta_{j, t} \le \min\{\scra, 0\}$ it holds that
\begin{equation}\label{eqn:lemma:GF_boundedness:Theta_j<1}
\begin{split}
& \textstyle \bigl([\mathcal{L}(\Theta_0)]^{1/2} + \bigl[\int_{\mathbb{R}} (f(x) - c)^2 \mathfrak{p}(x) \, \d x\bigr]^{1/2}\bigr)^2 \\
& \textstyle \ge \bigl([\mathcal{L}(\Theta_t)]^{1/2} + \bigl[\int_{\mathbb{R}} (f(x) - c)^2 \mathfrak{p}(x) \, \d x\bigr]^{1/2}\bigr)^2 \ge \int_{\mathbb{R}} (\mathcal{N}^{\Theta_t}(x) - c)^2 \mathfrak{p}(x) \, \d x \\
& \textstyle \ge \int_{\max\{\Theta_{j, t}, \scra\}}^{\scrb} (v_j [x - \Theta_{j, t}])^2 \mathfrak{p}(x) \, \d x = \int_{\scra}^{\scrb} (v_j [x - \Theta_{j, t}])^2 \mathfrak{p}(x) \, \d x \\
& \textstyle \ge (v_j [\scra - \Theta_{j, t}])^2 \int_{\scra}^{\scrb} \mathfrak{p}(x) \, \d x.
\end{split}
\end{equation}
This shows that for all $t \in [0, \infty)$ with $\Theta_{j, t} \le \scrb$ it holds that
\begin{equation}
\textstyle \scrb \ge \Theta_{j, t} \ge \scra - [v_j]^{-1} \bigl([\mathcal{L}(\Theta_0)]^{1/2} + \bigl[\int_{\mathbb{R}} (f(x) - c)^2 \mathfrak{p}(x) \, \d x\bigr]^{1/2}\bigr) \bigl[\int_{\mathbb{R}} \mathfrak{p}(x) \, \d x\bigr]^{-1/2}.
\end{equation}
Combining this with \cref{item1:lemma:GF_boundedness} establishes \cref{item2:lemma:GF_boundedness}. Furthermore, \nobs that \cref{item1:lemma:GF_boundedness} and \cref{item2:lemma:GF_boundedness} assure that for all $t \in [0, \infty)$ it holds that
\begin{equation}
\textstyle \abs{\Theta_{j, t}} \le \max\bigl\{\scrb, \abs{\Theta_{j, 0}}, \abs{\scra} + [v_j]^{-1} \bigl([\mathcal{L}(\Theta_0)]^{1/2} + \bigl[\int_{\mathbb{R}} (f(x) - c)^2 \mathfrak{p}(x) \, \d x\bigr]^{1/2}\bigr) \bigl[\int_{\mathbb{R}} \mathfrak{p}(x) \, \d x\bigr]^{-1/2}\bigr\}.
\end{equation}
This establishes \cref{item3:lemma:GF_boundedness}. Next \nobs that \cref{item3:lemma:GF_boundedness} shows that for all $t \in [0, \infty)$ it holds that
\begin{equation}
\begin{split}
\textstyle \norm{\Theta_t} & \textstyle \le \mathfrak{h} \scrb + \bigl[\sum_{\mathfrak{j} = 1}^{\mathfrak{h}} \abs{\Theta_{\mathfrak{j}, 0}}\bigr] + \mathfrak{h} \abs{\scra} + \frac{\sum_{\mathfrak{j} = 1}^{\mathfrak{h}} [v_{\mathfrak{j}}]^{-1}}{[\int_{\mathbb{R}} \mathfrak{p}(x) \, \d x]^{1/2}} \bigl([\mathcal{L}(\Theta_0)]^{1/2} + \bigl[\int_{\mathbb{R}} (f(x) - c)^2 \mathfrak{p}(x) \, \d x\bigr]^{1/2}\bigr) \\
& \textstyle \le \mathfrak{h} (\scrb + \abs{\scra}) + \mathfrak{h}^{1/2} \norm{\Theta_0} + \frac{\sum_{\mathfrak{j} = 1}^{\mathfrak{h}} [v_{\mathfrak{j}}]^{-1}}{[\int_{\mathbb{R}} \mathfrak{p}(x) \, \d x]^{1/2}} \bigl([\mathcal{L}(\Theta_0)]^{1/2} + \bigl[\int_{\mathbb{R}} (f(x) - c)^2 \mathfrak{p}(x) \, \d x\bigr]^{1/2}\bigr).
\end{split}
\end{equation}
This establishes \cref{item4:lemma:GF_boundedness}.
\end{cproof}




\cfclear
\begin{lemma}[GF convergence]\label{lemma:GF_convergence}
Assume \cref{setting1}, assume that $f$ and $\mathfrak{p}$ are piecewise polynomial, assume $\min_{j \in \{1, 2, \ldots, \mathfrak{h}\}} v_j > 0$, assume for all $x \in \mathbb{R}$ that $\mathfrak{p}(x) \ge 0$, and let $\Theta \in C([0, \infty), \allowbreak \mathbb{R}^{\mathfrak{h}})$ satisfy for all $t \in [0, \infty)$ that
\begin{equation}\label{eqn:lemma:GF_convergence:GF}
\textstyle \Theta_0 \in \mathbb{R}^{\mathfrak{h}} \qquad \text{and} \qquad \Theta_t = \Theta_0 + \int_0^t \mathcal{G}(\Theta_s) \, \d s
\end{equation}
(cf. \cref{def:piecewise_polynomial}). Then there exist $\vartheta \in \mathbb{R}^{\mathfrak{h}}$, $\mathscr{C}, \alpha \in (0, \infty)$ such that for all $t \in [0, \infty)$ it holds that
\begin{equation}\label{eqn:lemma:GF_convergence:limits}
\textstyle \mathcal{G}(\vartheta) = 0, \quad \norm{\Theta_t - \vartheta} \le \mathscr{C} (1 + t)^{- \alpha}, \quad \text{and} \quad 0 \le \mathcal{L}(\Theta_t) - \mathcal{L}(\vartheta) \le \mathscr{C} (1 + t)^{-1}.
\end{equation}
\end{lemma}
\begin{cproof}{lemma:GF_convergence}
\Nobs that the assumption that $f$ and $\mathfrak{p}$ are piecewise polynomial, \cite[Corollary~6.8]{JentzenRiekert2021ExistenceGlobMin}, and the fact that every projection of a semi-algebraic set is again semi-algebraic ensure that $\mathcal{L}$ is semi-algebraic function. Combining this with \cite[Proposition~7.9]{JentzenRiekert2021ExistenceGlobMin} prove that $\mathcal{L}$ is subanalytic function. This, \cite[Corollary~7.11]{JentzenRiekert2021ExistenceGlobMin}, and \cite[Theorem~3.1 and $(4)$]{BolteDaniilidis2006} demonstrate that for all $\theta \in \mathbb{R}^{\mathfrak{h}}$ there exist $\varepsilon, \mathfrak{C} \in (0, \infty)$, $\alpha \in (0, 1)$ such that for all $\vartheta \in \mathbb{R}^{\mathfrak{h}}$ with $\norm{\vartheta - \theta} < \varepsilon$ it holds that $\abs{\mathcal{L}(\vartheta) - \mathcal{L}(\theta)}^{\alpha} \allowbreak \le \mathfrak{C} \norm{\mathcal{G}(\vartheta)}$. Combining this and \cref{lemma:GF_boundedness} with \cite[Corollary~8.4]{JentzenRiekert2021ExistenceGlobMin} establishes \cref{eqn:lemma:GF_convergence:limits}.
\end{cproof}



\subsubsection{Convergence of the GF to suitable critical points}

Next we establish in \cref{prop:convergence_gf_suitable_cp} an upper bound for the risk of the limit point $\vartheta$ of the GF trajectory, under the assumption that $\mathcal{N}^{\vartheta}(\scrb) \ge f(\scrb)$.
Afterwards, we show in \cref{lemma:N>f_at_1} that this assumption is always satisfied under suitable assumptions on the initial value $\Theta_0$ of the GF and the weight parameters $v_j$.
 
\cfclear
\begin{proposition}\label{prop:convergence_gf_suitable_cp}
Assume \cref{setting1}, assume for all $x \in \mathbb{R}$ that $\mathfrak{p}(x) \ge 0$, assume that $f$ is strictly convex, assume that $f$ and $\mathfrak{p}$ are piecewise polynomial, let $\mathfrak{H} \subseteq \mathbb{N}$, $L \in \mathbb{R}$ satisfy $\mathfrak{H} = \{1, 2, \ldots, \mathfrak{h}\}$ and $L = \sup_{x \in [\scra, \scrb]} f'(x)$, assume $\min_{j \in \mathfrak{H}} v_j > 0$, let $\Theta \in C([0, \infty), \mathbb{R}^{\mathfrak{h}})$ satisfy for all $t \in [0, \infty)$ that
\begin{equation}
\textstyle \Theta_0 \in \mathbb{R}^{\mathfrak{h}} \quad \text{and} \quad \Theta_t = \Theta_0 + \int_0^t \mathcal{G}(\Theta_s) \, \d s,
\end{equation}
let $\vartheta \in \mathbb{R}^{\mathfrak{h}}$ satisfy
\begin{equation}\label{eqn:prop:convergence_gf_suitable_cp:vartheta}
\textstyle \limsup_{t \to \infty} (\norm{\Theta_t - \vartheta} + \abs{\mathcal{L}(\Theta_t) - \mathcal{L}(\vartheta)} + \norm{\mathcal{G}(\Theta_t)} + \norm{\mathcal{G}(\vartheta)}) = 0
\end{equation}
(cf.\ \cref{lemma:GF_convergence}), assume $\mathcal{N}^{\vartheta}(\scrb) \ge f(\scrb)$, and assume that $\vartheta$ is not a \cfadd{def:strict_saddle}\text{descending } critical point of $\mathcal{L}$ (cf. \cref{def:strict_saddle,def:piecewise_polynomial}). Then
\begin{equation}\label{eqn:prop:convergence_gf_suitable_cp:risk_bound}
\textstyle \mathcal{L}(\vartheta) \le [\max\{\max_{j \in \mathfrak{H}} v_j, \abs{c}\}] (L + [\max\{\max_{j \in \mathfrak{H}} v_j, \abs{c}\}]) [\sup_{x \in [\scra, \scrb]} \mathfrak{p}(x)] (\scrb - \scra).
\end{equation}
\end{proposition}
\begin{cproof}{prop:convergence_gf_suitable_cp}
\Nobs that \cref{eqn:prop:convergence_gf_suitable_cp:vartheta} and \cref{item3:cor:suitable_critical_upper_bound} in \cref{cor:suitable_critical_upper_bound} establish \cref{eqn:prop:convergence_gf_suitable_cp:risk_bound}.
\end{cproof}


\begin{lemma}\label{lemma:N>f_at_1}
Assume \cref{setting1}, assume for all $x \in \mathbb{R}$ that $\mathfrak{p}(x) \ge 0$, assume that $f$ is strictly convex, let $L \in (0, \infty)$ satisfy $L = \sup_{x \in [\scra, \scrb]} f'(x)$, assume $\min_{j \in \{1, 2, \ldots, \mathfrak{h}\}} v_j > 0$, let $\Theta \in \allowbreak C([0, \infty), \allowbreak \mathbb{R}^{\mathfrak{h}})$ satisfy for all $t \in [0, \infty)$ that
\begin{equation}
\textstyle \Theta_0 \in \mathbb{R}^{\mathfrak{h}} \quad \text{and} \quad \Theta_t = \Theta_0 + \int_0^t \mathcal{G}(\Theta_s) \, \d s,
\end{equation}
assume $\sum_{j \in \{i \in \{1, 2, \ldots, \mathfrak{h}\} \colon \Theta_{i, 0} < \scrb\}} v_j \ge L$, and let $\vartheta = (\vartheta_1, \ldots, \vartheta_{\mathfrak{h}}) \in \mathbb{R}^{\mathfrak{h}}$ satisfy
\begin{equation}\label{eqn:lemma:N>f_at_1:vartheta}
\textstyle \limsup_{t \to \infty} (\norm{\Theta_t - \vartheta} + \abs{\mathcal{L}(\Theta_t) - \mathcal{L}(\vartheta)} + \norm{\mathcal{G}(\Theta_t)} + \norm{\mathcal{G}(\vartheta)}) = 0.
\end{equation}
Then
\begin{equation}
\textstyle \mathcal{N}^{\vartheta}(\scrb) \ge f(\scrb).
\end{equation}
\end{lemma}
\begin{cproof}{lemma:N>f_at_1}
Throughout this proof let $\mathscr{v} \in (\scra, \scrb)$ satisfy 
\begin{equation}
\textstyle \mathscr{v} = \max_{j \in \{i \in \{1, 2, \ldots, \mathfrak{h}\} \colon \scra < \vartheta_i < \scrb\}} \vartheta_j.
\end{equation}
For the sake of contradiction we assume that $\mathcal{N}^{\vartheta}(\scrb) < f(\scrb)$. \Nobs that \cref{eqn:lemma:N>f_at_1:vartheta}, \cref{lemma:critical_points}, and the assumption that $\mathcal{N}^{\vartheta}(\scrb) < f(\scrb)$ ensure that $\mathcal{N}^{\vartheta}(\mathscr{v}) \ge f(\mathscr{v})$. Combining this with \cref{lemma:GF_boundedness}, the assumption that $\sum_{j \in \{i \in \{1, 2, \ldots, \mathfrak{h}\} \colon \Theta_{i, 0} < \scrb\}} v_j \ge L$, and the assumption that $f$ is strictly convex assures that there exists $\mathfrak{j} \in \{i \in \{1, 2, \ldots, \mathfrak{h}\} \colon \Theta_{i, 0} < \scrb\}$ which satisfies $\vartheta_{\mathfrak{j}} = \scrb$. This and \cref{eqn:lemma:N>f_at_1:vartheta} demonstrate that
\begin{equation}\label{eqn:lemma:N>f_at_1:i}
\textstyle \limsup_{t \to \infty} \abs{\Theta_{\mathfrak{j}, t} - \vartheta_{\mathfrak{j}}} = 0.
\end{equation}
The assumption that $\mathcal{N}^{\vartheta}(\scrb) < f(\scrb)$ hence assures that there exists $T \in [0, \infty)$ which satisfies for all $t \in [T, \infty)$ that $\Theta_{\mathfrak{j}, T} < \scrb$, $\scra < \Theta_{\mathfrak{j}, t} \le \scrb$, and $\mathcal{N}^{\Theta_t}(\Theta_{\mathfrak{j}, t}) < f(\Theta_{\mathfrak{j}, t})$. This and \cref{lemma:derivatives_of_the_risk} show that for all $t \in (T, \infty)$ it holds that
\begin{equation}
\textstyle \frac{\partial}{\partial t} \Theta_{\mathfrak{j}, t} = \mathcal{G}_{\mathfrak{j}}(\Theta_t) = - \bigl(\frac{\partial}{\partial \Theta_{\mathfrak{j}, t}} \mathcal{L}\bigr)(\Theta_t) = 2 v_{\mathfrak{j}} \int_{\Theta_{\mathfrak{j}, t}}^{\scrb} (\mathcal{N}^{\Theta_t}(x) - f(x)) \mathfrak{p}(x) \, \d x < 0.
\end{equation}
This implies that $(T, \infty) \ni t \mapsto \Theta_{\mathfrak{j}, t} \in \mathbb{R}$ is decreasing. Therefore, we obtain that for all $t \in (T, \infty)$ it holds that $\Theta_{\mathfrak{j}, t} < \Theta_{\mathfrak{j}, T} < \scrb = \vartheta_{\mathfrak{j}}$. This and \cref{eqn:lemma:N>f_at_1:i} contradict to the assumption that $\mathcal{N}^{\vartheta}(\scrb) < f(\scrb)$. Hence, we obtain that $\mathcal{N}^{\vartheta}(\scrb) \ge f(\scrb)$.
\end{cproof}



\subsection{Convergence of the GF with random initializations}

In this section we
establish in \cref{cor:main}, one of the main results of this article, convergence of GF processes with suitable random initializations. For this we first derive several properties of these random initializations.
Again, the inner and outer weights are assumed to be half-normally distributed with a scaling factor depending on the number of neurons $\mathfrak{h}$, and the inner biases are normally distributed.

\subsubsection{Upper bounds for the Gaussian tails}
\begin{lemma}\label{lemma:Gaussian_tail_bound}
It holds for all $y \in [0, \infty)$ that
\begin{equation}
\textstyle \int_{y}^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(- \frac{x^2}{2}) \, \d x \le \exp(- \frac{y^2}{2}).
\end{equation}
\end{lemma}
\begin{cproof}{lemma:Gaussian_tail_bound}
\Nobs that the integral transformation theorem and the fact that for all $x \in (0, 1)$ it holds that $\Gamma(x) \Gamma(1 - x) = \frac{\pi}{\sin(\pi x)}$ ensure that
\begin{equation}
\textstyle \int_0^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(- \frac{x^2}{2}) \, \d x = \frac{1}{\pi^{1/2}} \int_0^{\infty} t^{-1/2} \exp(-t) \, \d t = \frac{1}{\pi^{1/2}} \Gamma(\frac{1}{2}) = 1.
\end{equation}
Combining this with the integral transformation theorem shows that for all $y \in [0, \infty)$ it holds that
\begin{equation}
\begin{split}
\textstyle \int_{y}^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(- \frac{x^2}{2}) \, \d x & \textstyle = \int_0^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(- \frac{(x + y)^2}{2}) \, \d x = \int_0^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(- \frac{x^2 + 2 x y + y^2}{2}) \, \d x \\
& \textstyle \le \exp(- \frac{y^2}{2}) \int_0^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(- \frac{x^2}{2}) \, \d x = \exp(- \frac{y^2}{2}).
\end{split}
\end{equation}
\end{cproof}




\subsubsection{Size estimates for the slope changes of the realizations of ANNs}

\begin{lemma}[Lower and upper bounds for the slope changes]\label{lemma:bound_slope_changes}
Let $\alpha, \beta \in \mathbb{R}$, let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $\mathfrak{h} \in \mathbb{N}$ let $V_n^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, and $W_n^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, be random variables, and assume for all $\mathfrak{h} \in \mathbb{N}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, $x \in \mathbb{R}$ that $V_1^{\mathfrak{h}}, \ldots, V_{\mathfrak{h}}^{\mathfrak{h}}, W_1^{\mathfrak{h}}, \ldots, W_{\mathfrak{h}}^{\mathfrak{h}}$ are independent and $\P(\mathfrak{h}^{\alpha} W_n^{\mathfrak{h}} \ge x) = \P(\mathfrak{h}^{\beta} V_n^{\mathfrak{h}} \ge x) = \int_{\max\{x, 0\}}^{\infty} [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y$. Then
\begin{enumerate}[label=(\roman*)]
\item
\label{item1:lemma:bound_slope_changes} it holds for all $\mathfrak{h} \in \mathbb{N}$, $r \in \mathbb{R}$ that
\begin{equation}
\textstyle \P(W_1^{\mathfrak{h}} \le \mathfrak{h}^{-r}) \ge 1 - \exp(- \frac{1}{2} \mathfrak{h}^{2 \alpha - 2 r}) \quad \text{and} \quad \P(V_1^{\mathfrak{h}} \le \mathfrak{h}^{-r}) \ge 1 - \exp(- \frac{1}{2} \mathfrak{h}^{2 \beta - 2 r}),
\end{equation}

\item
\label{item2:lemma:bound_slope_changes} it holds for all $\mathfrak{h} \in \mathbb{N}$, $r, s \in \mathbb{R}$ that
\begin{equation}
\textstyle \P\bigl(\max_{j \in \{1, 2, \ldots, \mathfrak{h}\}}(V_j^{\mathfrak{h}} W_j^{\mathfrak{h}}) \le \mathfrak{h}^{- r}\bigr) \ge \bigl[1 - \exp(- \frac{1}{2} \mathfrak{h}^{2 \beta - 2 s})\bigr]^{\mathfrak{h}} \bigl[1 - \exp(- \frac{1}{2} \mathfrak{h}^{2 \alpha + 2 s - 2r})\bigr]^{\mathfrak{h}},
\end{equation}

and
\item
\label{item3:lemma:bound_slope_changes} it holds for all $\mathfrak{h} \in \mathbb{N}$, $\kappa \in \{1, 2, \ldots, \mathfrak{h}\}$, $r \in \mathbb{R}$ that
\begin{equation}
\textstyle \P\bigl(\sum_{j = 1}^{\kappa} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} \le \bigl[\frac{2}{\pi}\bigr] \kappa \mathfrak{h}^{- \alpha - \beta} - \mathfrak{h}^{r} \bigr) \le \kappa \mathfrak{h}^{- 2 \alpha - 2 \beta - 2 r} \bigl(1 - \frac{4}{\pi^2}\bigr).
\end{equation}
\end{enumerate}
\end{lemma}
\begin{cproof}{lemma:bound_slope_changes}
Throughout this proof for every $\mathfrak{h} \in \mathbb{N}$, $\kappa \in \{1, 2, \ldots, \mathfrak{h}\}$ let $S_{\kappa}^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$ satisfy $S_{\kappa}^{\mathfrak{h}} = \sum_{j = 1}^{\kappa} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}}$. \Nobs that \cref{lemma:Gaussian_tail_bound} ensures that for all $\mathfrak{h} \in \mathbb{N}$, $r \in \mathbb{R}$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P(W_1^{\mathfrak{h}} \le \mathfrak{h}^{- r}) = \P(\mathfrak{h}^{\alpha} W_1^{\mathfrak{h}} \le \mathfrak{h}^{\alpha - r}) = 1 - \P(\mathfrak{h}^{\alpha} W_1^{\mathfrak{h}} > \mathfrak{h}^{\alpha - r}) \\
& \textstyle = 1 - \int_{\mathfrak{h}^{\alpha - r}}^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(-\frac{x^2}{2}) \, \d x \ge 1 - \exp(- \frac{1}{2} \mathfrak{h}^{2 \alpha - 2 r}) \ge 0
\end{split}
\end{equation}
and
\begin{equation}
\begin{split}
& \textstyle \P(V_1^{\mathfrak{h}} \le \mathfrak{h}^{- r}) = \P(\mathfrak{h}^{\beta} V_1^{\mathfrak{h}} \le \mathfrak{h}^{\beta - r}) = 1 - \P(\mathfrak{h}^{\beta} V_1^{\mathfrak{h}} > \mathfrak{h}^{\beta - r}) \\
& \textstyle = 1 - \int_{\mathfrak{h}^{\beta - r}}^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(-\frac{x^2}{2}) \, \d x \ge 1 - \exp(- \frac{1}{2} \mathfrak{h}^{2 \beta - 2 r}) \ge 0.
\end{split}
\end{equation}
This establishes \cref{item1:lemma:bound_slope_changes}. Next \nobs that \cref{item1:lemma:bound_slope_changes} and the assumption that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $V_1^{\mathfrak{h}}, \ldots, V_{\mathfrak{h}}^{\mathfrak{h}}, W_1^{\mathfrak{h}}, \ldots, W_{\mathfrak{h}}^{\mathfrak{h}}$ are independent assure that for all $\mathfrak{h} \in \mathbb{N}$, $r, s \in \mathbb{R}$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P(\max\{V_1^{\mathfrak{h}} W_1^{\mathfrak{h}}, V_2^{\mathfrak{h}} W_2^{\mathfrak{h}}, \ldots, V_{\mathfrak{h}}^{\mathfrak{h}} W_{\mathfrak{h}}^{\mathfrak{h}}\} \le \mathfrak{h}^{- r}) \\
& \textstyle = \bigl[\P(V_1^{\mathfrak{h}} W_1^{\mathfrak{h}} \le \mathfrak{h}^{- r})\bigr]^{\mathfrak{h}} \ge \bigl[\P([V_1^{\mathfrak{h}} \le \mathfrak{h}^{- s}] \cap [W_1^{\mathfrak{h}} \le \mathfrak{h}^{s - r}])\bigr]^{\mathfrak{h}} \\ 
& \textstyle = \bigl[\P([V_1^{\mathfrak{h}} \le \mathfrak{h}^{- s}]) \, \P([W_1^{\mathfrak{h}} \le \mathfrak{h}^{s - r}])\bigr]^{\mathfrak{h}} \\
& \textstyle \ge \bigl[1 - \exp(- \frac{1}{2} \mathfrak{h}^{2 \beta - 2 s})\bigr]^{\mathfrak{h}} \bigl[1 - \exp(- \frac{1}{2} \mathfrak{h}^{2 \alpha + 2 s - 2r})\bigr]^{\mathfrak{h}}.
\end{split}
\end{equation}
This establishes \cref{item2:lemma:bound_slope_changes}. In the next step \nobs that the assumption that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $V_1^{\mathfrak{h}}$ and $W_1^{\mathfrak{h}}$ are independent and the integral transformation theorem prove that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}\label{eqn:lemma:bound_slope_changes:exp_d2}
\begin{split}
\textstyle \mathbb{E}(V_1^{\mathfrak{h}} W_1^{\mathfrak{h}}) & \textstyle = \mathbb{E}(V_1^{\mathfrak{h}}) \mathbb{E}(W_1^{\mathfrak{h}}) = \mathfrak{h}^{- \alpha - \beta} \bigl[\int_{- \infty}^{\infty} \frac{1}{(2 \pi)^{1/2}} \abs{x} \exp(- \frac{x^2}{2}) \, \d x\bigr]^2 \\
& \textstyle = \frac{2}{\pi} \mathfrak{h}^{- \alpha - \beta} \bigl[\int_0^{\infty} x \exp(- \frac{x^2}{2}) \, \d x\bigr]^2 = \frac{2}{\pi} \mathfrak{h}^{- \alpha - \beta} ([-\exp(- \frac{x^2}{2})]_{x=0}^{x=\infty})^2 = \frac{2}{\pi} \mathfrak{h}^{- \alpha - \beta}.
\end{split}
\end{equation}
and
\begin{equation}
\begin{split}
\textstyle \mathbb{E}([V_1^{\mathfrak{h}} W_1^{\mathfrak{h}}]^2) & \textstyle = \mathbb{E}([V_1^{\mathfrak{h}}]^2) \mathbb{E}([W_1^{\mathfrak{h}}]^2) = \mathfrak{h}^{- 2 \alpha - 2 \beta} \bigl[\int_{-\infty}^{\infty} \frac{1}{(2 \pi)^{1/2}} \abs{x}^2 \exp(- \frac{x^2}{2}) \, \d x\bigr]^2 \\
& \textstyle = \frac{2}{\pi} \mathfrak{h}^{- 2 \alpha - 2 \beta} \bigl[\int_0^{\infty} x^2 \exp(- \frac{x^2}{2}) \, \d x\bigr]^2 = \frac{4}{\pi} \mathfrak{h}^{- 2 \alpha - 2 \beta} \bigl[\int_0^{\infty} t^{1/2} \exp(-t) \, \d t\bigr]^2 \\
& \textstyle = \frac{4}{\pi} \mathfrak{h}^{- 2 \alpha - 2 \beta} \bigl[\Gamma(\frac{3}{2})\bigr]^2 = \frac{4}{\pi} \mathfrak{h}^{- 2 \alpha - 2 \beta} \bigl[\frac{1}{2}\Gamma(\frac{1}{2})\bigr]^2 = \mathfrak{h}^{- 2 \alpha - 2 \beta}.
\end{split}
\end{equation}
Hence, we obtain that for all $\mathfrak{h} \in \mathbb{N}$, $\kappa \in \{1, 2, \ldots, \mathfrak{h}\}$ it holds that
\begin{equation}
\begin{split}
& \textstyle \mathbb{E}(\abs{S_{\kappa}^{\mathfrak{h}} - \mathbb{E}(S_{\kappa}^{\mathfrak{h}})}^2) = \mathbb{E}([S_{\kappa}^{\mathfrak{h}}]^2) - [\mathbb{E}(S_{\kappa}^{\mathfrak{h}})]^2 \\
& \textstyle = \mathbb{E}\bigl(\sum_{j = 1}^{\kappa}[V_{j}^{\mathfrak{h}} W_{j}^{\mathfrak{h}}]^2 + 2 \sum_{1 \le i < j \le \kappa} V_i^{\mathfrak{h}} W_i^{\mathfrak{h}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}}\bigr) - \kappa^2 [\mathbb{E}(V_1^{\mathfrak{h}} W_1^{\mathfrak{h}})]^2 \\
& \textstyle = \kappa \mathbb{E}([V_1^{\mathfrak{h}} W_1^{\mathfrak{h}}]^2) + (\kappa^2 - \kappa) [\mathbb{E}(V_1^{\mathfrak{h}} W_1^{\mathfrak{h}})]^2 - \kappa^2 [\mathbb{E}(V_1^{\mathfrak{h}} W_1^{\mathfrak{h}})]^2 \\
& \textstyle = \kappa (\mathbb{E}([V_1^{\mathfrak{h}} W_1^{\mathfrak{h}}]^2) - [\mathbb{E}(V_1^{\mathfrak{h}} W_1^{\mathfrak{h}})]^2) = \kappa \mathfrak{h}^{- 2 \alpha - 2 \beta} \bigl(1 - \frac{4}{\pi^2}\bigr).
\end{split}
\end{equation}
Combining this with Markov's inequality and \cref{eqn:lemma:bound_slope_changes:exp_d2} demonstrates that for all $\mathfrak{h} \in \mathbb{N}$, $\kappa \in \{1, 2, \ldots, \mathfrak{h}\}$, $r \in \mathbb{R}$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P(S_{\kappa}^{\mathfrak{h}} \le \bigl[\frac{2}{\pi}\bigr] \kappa \mathfrak{h}^{- \alpha - \beta} - \mathfrak{h}^{r}) \\
& \textstyle = \P(S_{\kappa}^{\mathfrak{h}} - \mathbb{E}(S_{\kappa}^{\mathfrak{h}}) \le - \mathfrak{h}^{r}) \le \P(\abs{S_{\kappa}^{\mathfrak{h}} - \mathbb{E}(S_{\kappa}^{\mathfrak{h}})} \ge \mathfrak{h}^{r}) = \P(\abs{S_{\kappa}^{\mathfrak{h}} - \mathbb{E}(S_{\kappa}^{\mathfrak{h}})}^2 \ge \mathfrak{h}^{2 r}) \\
& \textstyle \le \frac{\mathbb{E}(\abs{S_{\kappa}^{\mathfrak{h}} - \mathbb{E}(S_{\kappa}^{\mathfrak{h}})}^2)}{\mathfrak{h}^{2 r}} = \kappa \mathfrak{h}^{- 2 \alpha - 2 \beta -2 r} \bigl(1 - \frac{4}{\pi^2}\bigr).
\end{split}
\end{equation}
This establishes \cref{item3:lemma:bound_slope_changes}.
\end{cproof}


\begin{corollary}[Lower and upper bounds for the slope changes]\label{cor:bound_outer_weights}
Let $\alpha, \beta \in \mathbb{R}$ satisfy $0 < \alpha + \beta < 1$, let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $\mathfrak{h} \in \mathbb{N}$ let $V_n^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, and $W_n^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, be random variables, and assume for all $\mathfrak{h} \in \mathbb{N}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, $x \in \mathbb{R}$ that $V_1^{\mathfrak{h}}, \ldots, V_{\mathfrak{h}}^{\mathfrak{h}}, W_1^{\mathfrak{h}}, \ldots, W_{\mathfrak{h}}^{\mathfrak{h}}$ are independent and $\P(\mathfrak{h}^{\alpha} W_n^{\mathfrak{h}} \ge x) = \P(\mathfrak{h}^{\beta} V_n^{\mathfrak{h}} \ge x) = \int_{\max\{x, 0\}}^{\infty} [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y$. Then
\begin{enumerate}[label=(\roman*)]
\item
\label{item1:cor:bound_outer_weights} it holds for all $\mathfrak{h} \in \mathbb{N}$, $r \in (0, \alpha + \beta)$ that
\begin{equation}
\textstyle \P(\max\{V_1^{\mathfrak{h}} W_1^{\mathfrak{h}}, V_2^{\mathfrak{h}} W_2^{\mathfrak{h}}, \ldots, V_{\mathfrak{h}}^{\mathfrak{h}} W_{\mathfrak{h}}^{\mathfrak{h}}\} \le \mathfrak{h}^{r - \alpha - \beta}) \ge 1 - 2 \mathfrak{h} \exp(-\frac{1}{2} \mathfrak{h}^{r})
\end{equation}

and
\item
\label{item2:cor:bound_outer_weights} it holds for all $\sigma \in (0, 1]$, $\varepsilon \in (\alpha + \beta, 1]$, $r \in (\max\{(\varepsilon / 2) - \alpha - \beta, 0\}, \varepsilon - \alpha - \beta)$ that there exists $\mathfrak{C}_{\sigma, \varepsilon, r} \in (0, \infty)$ such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} \ge h^{r} \Bigr) \ge 1 - \mathfrak{C}_{\sigma, \varepsilon, r} \mathfrak{h}^{\varepsilon - 2 \alpha - 2 \beta - 2r}.
\end{equation}
\end{enumerate}
\end{corollary}
\begin{cproof}{cor:bound_outer_weights}
Throughout this proof let $\sigma \in (0, 1]$, $\varepsilon \in (\alpha + \beta, 1]$, $r \in (\max\{(\varepsilon / 2) - \alpha - \beta, 0\}, \varepsilon - \alpha - \beta)$ and let $\mathfrak{C}_{\sigma, \varepsilon, r} \in \mathbb{R}$ satisfy 
\begin{equation}\label{eqn:cor:bound_outer_weights:fC_gamma}
\textstyle \mathfrak{C}_{\sigma, \varepsilon, r} = \max\Bigl\{2 \sigma, \bigl[\frac{\pi}{\sigma}\bigr]^{\frac{2 r + 2 \alpha + 2 \beta - \varepsilon}{\varepsilon - \alpha - \beta - r}} \max_{\mathfrak{h} \in \mathbb{N} \cap (0, [\pi / \sigma]^{1/(\varepsilon - \alpha - \beta - r)})} \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} < \mathfrak{h}^{r} \Bigr)\Bigr\}.
\end{equation}
\Nobs that \cref{item2:lemma:bound_slope_changes} in \cref{lemma:bound_slope_changes} ensures that for all $r \in (0, \alpha + \beta)$, $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P(\max\{V_1^{\mathfrak{h}} W_1^{\mathfrak{h}}, V_2^{\mathfrak{h}} W_2^{\mathfrak{h}}, \ldots, V_{\mathfrak{h}}^{\mathfrak{h}} W_{\mathfrak{h}}^{\mathfrak{h}}\} \le \mathfrak{h}^{r - \alpha - \beta}) \ge [1 - \exp(-\frac{1}{2} \mathfrak{h}^{r})]^{2 \mathfrak{h}}.
\end{equation}
Combining this and the fact that for all $\mathfrak{h} \in \mathbb{N}$, $r \in (0, \infty)$ it holds that $\exp(-\frac{1}{2} \mathfrak{h}^{r}) \allowbreak < \allowbreak 1$ with Bernoulli's inequality demonstrates that for all $\mathfrak{h} \in \mathbb{N}$, $r \in (0, \alpha + \beta)$ it holds that
\begin{equation}
\textstyle \P(\max\{V_1^{\mathfrak{h}} W_1^{\mathfrak{h}}, V_2^{\mathfrak{h}} W_2^{\mathfrak{h}}, \ldots, V_{\mathfrak{h}}^{\mathfrak{h}} W_{\mathfrak{h}}^{\mathfrak{h}}\} \le \mathfrak{h}^{r - \alpha - \beta}) \ge [1 - \exp(-\frac{1}{2} \mathfrak{h}^{r})]^{2 \mathfrak{h}} \ge 1 - 2 \mathfrak{h} \exp(-\frac{1}{2} \mathfrak{h}^{r}).
\end{equation}
This establishes \cref{item1:cor:bound_outer_weights}. Next \nobs that \cref{eqn:cor:bound_outer_weights:fC_gamma} ensures for all $\mathfrak{h} \in \mathbb{N} \cap (0, [\pi / \sigma]^{1/(\varepsilon - \alpha - \beta - r)})$ that
\begin{equation}\label{eqn:cor:bound_outer_weights:prob_bound_small_fh}
\textstyle \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} < \mathfrak{h}^{r} \Bigr) \le \bigl[\frac{\pi}{\sigma}\bigr]^{\frac{\varepsilon - 2 \alpha - 2 \beta -2 r}{\varepsilon - \alpha - \beta - r}} \mathfrak{C}_{\sigma, \varepsilon, r} \le \mathfrak{C}_{\sigma, \varepsilon, r} \mathfrak{h}^{\varepsilon - 2 \alpha - 2 \beta -2 r}.
\end{equation}
In the next step \nobs that \cref{item3:lemma:bound_slope_changes} in \cref{lemma:bound_slope_changes} (applied with $\kappa \with \ceil{\sigma \mathfrak{h}^{\varepsilon}}$ in the notation of \cref{item3:lemma:bound_slope_changes} in \cref{lemma:bound_slope_changes}) shows that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} < \frac{2}{\pi} \sigma h^{\varepsilon - \alpha - \beta} - \mathfrak{h}^{r}\Bigr) \le \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} \le \frac{2}{\pi} \ceil{\sigma \mathfrak{h}^{\varepsilon}} h^{- \alpha - \beta} - \mathfrak{h}^{r}\Bigr) \\
& \textstyle \le \ceil{\sigma \mathfrak{h}^{\varepsilon}} \mathfrak{h}^{-2 \beta - 2 \alpha - 2 r} (1 - \frac{4}{\pi^2}) \le 2 \sigma \mathfrak{h}^{\varepsilon - 2 \alpha - 2 \beta - 2 r} (1 - \frac{4}{\pi^2}) \le 2 \sigma \mathfrak{h}^{\varepsilon - 2 \alpha - 2 \beta - 2 r}.
\end{split}
\end{equation}
Combining this with \cref{eqn:cor:bound_outer_weights:fC_gamma} demonstrates that for all $\mathfrak{h} \in \mathbb{N} \cap [[\pi / \sigma]^{1/(\varepsilon - \alpha - \beta - r)}, \infty)$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} < \mathfrak{h}^{r} \Bigr) \le \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} < \frac{2}{\pi} \sigma h^{\varepsilon - \alpha - \beta} - \mathfrak{h}^{r}\Bigr) \\
& \textstyle \le 2 \sigma \mathfrak{h}^{\varepsilon - 2 \alpha - 2 \beta - 2 r} \le \mathfrak{C}_{\sigma, \varepsilon, r} \mathfrak{h}^{\varepsilon - 2 \alpha - 2 \beta - 2 r}.
\end{split}
\end{equation}
This and \cref{eqn:cor:bound_outer_weights:prob_bound_small_fh} prove that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} < \mathfrak{h}^{r} \Bigr) \le \mathfrak{C}_{\sigma, \varepsilon, r} \mathfrak{h}^{\varepsilon - 2 \alpha - 2 \beta - 2 r}.
\end{equation}
Therefore, we obtain that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} \ge \mathfrak{h}^{r} \Bigr) = 1 - \P\Bigl(\sum_{j = 1}^{\ceil{\sigma \mathfrak{h}^{\varepsilon}}} V_j^{\mathfrak{h}} W_j^{\mathfrak{h}} < \mathfrak{h}^{r} \Bigr) \ge 1 - \mathfrak{C}_{\sigma, \varepsilon, r} \mathfrak{h}^{\varepsilon - 2 \alpha - 2 \beta - 2 r}.
\end{equation}
This establishes \cref{item2:cor:bound_outer_weights}.
\end{cproof}

\subsubsection{Location of the kinks of the realizations of ANNs}

\begin{lemma}[Location of the kinks]\label{lemma:kinks_location}
Let $\alpha, \gamma \in \mathbb{R}$, $\scrb \in (0, \infty)$ satisfy $\alpha < \gamma$, let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $\mathfrak{h} \in \mathbb{N}$ let $B_n^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, and $W_n^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, be random variables, and assume for all $\mathfrak{h} \in \mathbb{N}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, $x \in \mathbb{R}$ that $B_1^{\mathfrak{h}}, \ldots, B_{\mathfrak{h}}^{\mathfrak{h}}, W_1^{\mathfrak{h}}, \ldots, W_{\mathfrak{h}}^{\mathfrak{h}}$ are independent, $\mathfrak{h}^{\gamma} B_n^{\mathfrak{h}}$ is standard normal, and $\P(\mathfrak{h}^{\alpha} W_n^{\mathfrak{h}} \ge x) = \int_{\max\{x, 0\}}^{\infty} [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y$. Then it holds for all $\mathfrak{h} \in \mathbb{N} \cap [(2 / (\pi \scrb))^{1 / (\gamma - \alpha)}, \infty)$, $\kappa \in \{1, 2, \ldots, \mathfrak{h}\}$ that
\begin{equation}
\textstyle \P(\min_{n \in \{1, 2, \ldots, \kappa\}}([B_n^{\mathfrak{h}} / W_n^{\mathfrak{h}}]) > - \scrb) \ge 1 - \frac{2 \kappa}{\pi \scrb} \mathfrak{h}^{\alpha - \gamma}. 
\end{equation}
\end{lemma}
\begin{cproof}{lemma:kinks_location}
Throughout this proof let $\xi_{1, n}^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $\mathfrak{h} \in \mathbb{N}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, and $\xi_{2, n}^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $\mathfrak{h} \in \mathbb{N}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, be standard normal distributions and assume for all $\mathfrak{h} \in \mathbb{N}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$ that $B_n^{\mathfrak{h}} = \mathfrak{h}^{- \gamma} \xi_{1, n}^{\mathfrak{h}}$ and $W_n^{\mathfrak{h}} = \mathfrak{h}^{- \alpha} \abs{\xi_{2, n}^{\mathfrak{h}}}$. \Nobs that for all $\mathfrak{h} \in \mathbb{N}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$ it holds that
\begin{equation}
\textstyle \forall\, x \in \mathbb{R} \colon \P([\xi_{1, n}^{\mathfrak{h}} / \xi_{2, n}^{\mathfrak{h}}] \ge x) = \int_x^{\infty} \frac{1}{\pi (x^2 + 1)} \, \d x.
\end{equation}
The fact that for all $x \in [0, \infty]$ it holds that $\lim_{y \to x, y \in (0, \infty)} [\arctan(y) + \arctan(1/y)] = \pi/2$, the fact that for all $x \in [0, \infty)$ it holds that $\arctan(x) \le x$, and the integral transformation theorem therefore imply that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P([B_1^{\mathfrak{h}}/W_1^{\mathfrak{h}}] > - \scrb) = \P([\xi_{1, 1}^{\mathfrak{h}} / \abs{\xi_{2, 1}^{\mathfrak{h}}}] > - \scrb \mathfrak{h}^{\gamma - \alpha}) \ge \P(\scrb \mathfrak{h}^{\gamma - \alpha} > [\xi_{1, 1}^{\mathfrak{h}} / \abs{\xi_{2, 1}^{\mathfrak{h}}}] > - \scrb \mathfrak{h}^{\gamma - \alpha}) \\
& \textstyle \ge \P(\abs{\xi_{1, 1}^{\mathfrak{h}} / \xi_{2, 1}^{\mathfrak{h}}} < \scrb \mathfrak{h}^{\gamma - \alpha}) = \int_{- \scrb \mathfrak{h}^{\gamma - \alpha}}^{\scrb \mathfrak{h}^{\gamma - \alpha}} \frac{1}{\pi (x^2 + 1)} \, \d x = 1 - \int_{- \infty}^{- \scrb \mathfrak{h}^{\gamma - \alpha}} \frac{1}{\pi (x^2 + 1)} \, \d x - \int_{\scrb \mathfrak{h}^{\gamma - \alpha}}^{\infty} \frac{1}{\pi (x^2 + 1)} \, \d x \\
& \textstyle = 1 - 2 \int_{\scrb \mathfrak{h}^{\gamma - \alpha}}^{\infty} \frac{1}{\pi (x^2 + 1)} \, \d x = 1 - \frac{2}{\pi}([\lim_{x \to \infty} \arctan(x)] - \arctan(\scrb \mathfrak{h}^{\gamma - \alpha})) \\
& \textstyle = 1 - \frac{2}{\pi}(\frac{\pi}{2} - \arctan(\scrb \mathfrak{h}^{\gamma - \alpha})) = 1 - \frac{2}{\pi} \arctan(\scrb^{-1} \mathfrak{h}^{\alpha - \gamma}) \ge 1 - \frac{2}{\pi \scrb} \mathfrak{h}^{\alpha - \gamma}.
\end{split}
\end{equation}
Combining this, the assumption that $\alpha < \gamma$, the assumption that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $B_1^{\mathfrak{h}}, \ldots, B_{\mathfrak{h}}^{\mathfrak{h}}, W_1^{\mathfrak{h}}, \ldots, W_{\mathfrak{h}}^{\mathfrak{h}}$ are independent, and the fact that for all $\mathfrak{h} \in \mathbb{N} \cap [(2 / (\pi \scrb))^{1 / (\gamma - \alpha)}, \allowbreak \infty)$ it holds that $\frac{2}{\pi \scrb} \mathfrak{h}^{\alpha - \gamma} \le 1$ with Bernoulli's inequality ensures that for all $\mathfrak{h} \in \mathbb{N} \cap [(2 / (\pi \scrb))^{1 / (\gamma - \alpha)}, \allowbreak \infty)$, $\kappa \in \{1, 2, \ldots, \mathfrak{h}\}$ it holds that
\begin{equation}
\textstyle \P(\min_{n \in \{1, 2, \ldots, \kappa\}}([B_n^{\mathfrak{h}} / W_n^{\mathfrak{h}}]) > - \scrb) = [\P([B_1^{\mathfrak{h}}/W_1^{\mathfrak{h}}] > - \scrb)]^{\kappa} \ge [1 - \frac{2}{\pi \scrb} \mathfrak{h}^{\alpha - \gamma}]^{\kappa} \ge 1 - \frac{2 \kappa}{\pi \scrb} \mathfrak{h}^{\alpha - \gamma}.
\end{equation}
\end{cproof}


\begin{corollary}[Location of the kinks]\label{cor:kinks_location}
Let $\alpha, \gamma \in \mathbb{R}$, $\scrb \in (0, \infty)$ satisfy $\alpha < \gamma$, let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $\mathfrak{h} \in \mathbb{N}$ let $B_n^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, and $W_n^{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, be random variables, and assume for all $\mathfrak{h} \in \mathbb{N}$, $n \in \{1, 2, \ldots, \mathfrak{h}\}$, $x \in \mathbb{R}$ that $B_1^{\mathfrak{h}}, \ldots, B_{\mathfrak{h}}^{\mathfrak{h}}, W_1^{\mathfrak{h}}, \ldots, W_{\mathfrak{h}}^{\mathfrak{h}}$ are independent, $\mathfrak{h}^{\gamma} B_n^{\mathfrak{h}}$ is standard normal, and $\P(\mathfrak{h}^{\alpha} W_n^{\mathfrak{h}} \ge x) = \int_{\max\{x, 0\}}^{\infty} [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y$. Then it holds for all $\mathfrak{h} \in \mathbb{N} \cap [(2 / (\pi \scrb))^{1 / (\gamma - \alpha)}, \allowbreak \infty)$, $\sigma, \varepsilon \in (0, 1]$ that
\begin{equation}
\textstyle \P\bigl(\min_{n \in \{1, 2, \ldots, \ceil{\sigma \mathfrak{h}^{\varepsilon}}\}}([B_n^{\mathfrak{h}} / W_n^{\mathfrak{h}}]) > - \scrb\bigr) \ge 1 - \frac{4 \sigma}{\pi \scrb} \mathfrak{h}^{\varepsilon + \alpha - \gamma}. 
\end{equation}
\end{corollary}
\begin{cproof}{cor:kinks_location}
\Nobs that \cref{lemma:kinks_location} ensures that for all $\mathfrak{h} \in \mathbb{N} \cap [(2 / (\pi \scrb))^{1 / (\gamma - \alpha)}, \allowbreak \infty)$, $\sigma, \varepsilon \in (0, 1]$ it holds that
\begin{equation}
\textstyle \P\bigl(\min_{n \in \{1, 2, \ldots, \ceil{\sigma \mathfrak{h}^{\varepsilon}}\}}([B_n^{\mathfrak{h}} / W_n^{\mathfrak{h}}]) > - \scrb\bigr) \ge 1 - \frac{2 \ceil{\sigma \mathfrak{h}^{\varepsilon}}}{\pi \scrb} \mathfrak{h}^{\alpha - \gamma} \ge 1 - \frac{4 \sigma \mathfrak{h}^{\varepsilon}}{\pi \scrb} \mathfrak{h}^{\alpha - \gamma} = 1 - \frac{4 \sigma}{\pi \scrb} \mathfrak{h}^{\varepsilon + \alpha - \gamma}.
\end{equation}
\end{cproof}


\subsubsection{Size estimates for the outer biases of ANNs}

 
\begin{lemma}[Size estimates for the outer biases]\label{lemma:outer_bias_size}
Let $\delta \in \mathbb{R}$, $\mathfrak{h} \in \mathbb{N}$, let $(\Omega, \mathcal{F}, \P)$ be a probability space, let $C \colon \Omega \to \mathbb{R}$ be random variable, and assume that $\mathfrak{h}^{\delta} C$ is standard normal. Then
\begin{equation}
\textstyle \forall\, \kappa \in \mathbb{R} \colon \P(- \mathfrak{h}^{- \kappa} \le C \le \mathfrak{h}^{- \kappa}) \ge 1 - \exp(-\frac{1}{2} \mathfrak{h}^{2 (\delta - \kappa)}). 
\end{equation}
\end{lemma}
\begin{cproof}{lemma:outer_bias_size}
\Nobs that \cref{lemma:Gaussian_tail_bound}, the assumption that $\mathfrak{h}^{\delta} C$ is standard normal, and the integral transformation theorem ensure that for all $\kappa \in \mathbb{R}$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P(- \mathfrak{h}^{- \kappa} \le C \le \mathfrak{h}^{- \kappa}) = \P(- \mathfrak{h}^{\delta - \kappa} \le \mathfrak{h}^{\delta} C \le \mathfrak{h}^{\delta - \kappa}) = \int_{- \mathfrak{h}^{\delta - \kappa}}^{\mathfrak{h}^{\delta - \kappa}} \frac{1}{(2 \pi)^{1/2}} \exp(- \frac{x^2}{2}) \, \d x \\
& \textstyle = \int_{0}^{\mathfrak{h}^{\delta - \kappa}} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(- \frac{x^2}{2}) \, \d x = 1 - \int_{\mathfrak{h}^{\delta - \kappa}}^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(- \frac{x^2}{2}) \, \d x \ge 1 - \exp(-\frac{1}{2} \mathfrak{h}^{2 (\delta - \kappa)}).
\end{split}
\end{equation}
\end{cproof}

\subsubsection{GF convergence with random initializations}
\label{sec:ANNs_with_ReLU_final_result}

We next combine in \cref{prop:main} the properties of the random initialization established above. Afterwards, we employ \cref{prop:convergence_gf_suitable_cp} to show in \cref{cor:main} convergence of the GF to a good critical point with high probability.

\cfclear
\begin{proposition}\label{prop:main}
Let $\scra \in \mathbb{R}$, $\scrb \in (\max\{\scra, 0\}, \infty)$, let $f \in C^1(\mathbb{R}, \mathbb{R})$ and $\mathfrak{p} \in C(\mathbb{R}, \mathbb{R})$ satisfy for all $x \in (\scra, \scrb)$ that $f'(x) > f(\scra) = 0 \le \mathfrak{p}(x)$, $\mathfrak{p}^{-1}(\mathbb{R} \backslash \{0\}) = (\scra, \scrb)$, and $f$ is strictly convex, let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $\mathfrak{h} \in \mathbb{N}$ let $\mathfrak{d}_{\mathfrak{h}} = 3 \mathfrak{h} + 1$, let $\mathcal{L}_{\mathfrak{h}} \allowbreak \colon \allowbreak \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}} \allowbreak \to \allowbreak \mathbb{R}$ satisfy for all $\theta = (\theta_1, \ldots, \theta_{\mathfrak{d}_{\mathfrak{h}}}) \in \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ that
\begin{equation}
\textstyle \mathcal{L}_{\mathfrak{h}}(\theta) = \int_{\mathbb{R}} (f(x) - \theta_{\mathfrak{d}_{\mathfrak{h}}} - \sum_{j = 1}^{\mathfrak{h}} \theta_{2 \mathfrak{h} + j} \max\{\theta_{\mathfrak{h} + j} x + \theta_{j}\})^2 \mathfrak{p}(x) \, \d x,
\end{equation}
let $\mathcal{G}_{\mathfrak{h}} = (\mathcal{G}_{\mathfrak{h}}^1, \ldots, \mathcal{G}_{\mathfrak{h}}^{\mathfrak{d}_{\mathfrak{h}}}) \colon \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}} \to \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ satisfy for all $\theta = (\theta_1, \ldots, \theta_{\mathfrak{d}_{\mathfrak{h}}}) \in \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$, $k \in \{1, 2, \ldots, \mathfrak{d}_{\mathfrak{h}}\}$ that
\begin{equation}\label{eqn:prop:main:gradient}
\textstyle \mathcal{G}_{\mathfrak{h}}^{k}(\theta) = (\frac{\partial^{-}}{\partial \theta_k} \mathcal{L}_{\mathfrak{h}})(\theta) \mathbbm{1}_{\mathbb{N} \cap (0, \mathfrak{h}]}(k),
\end{equation}
and let $\Theta^{\mathfrak{h}} = (\Theta^{\mathfrak{h}, 1}, \ldots, \Theta^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}) \colon [0, \infty) \times \Omega \to \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ be a stoch. proc. with c.s.p., assume for all $\mathfrak{h} \in \mathbb{N}$, $t \in [0, \infty)$ that $\P(\Theta_t^{\mathfrak{h}} = \Theta_0^{\mathfrak{h}} - \int_0^{t} \mathcal{G}_{\mathfrak{h}}(\Theta_s^{\mathfrak{h}}) \, \d s) = 1$, assume for all $\mathfrak{h} \in \mathbb{N}$ that $\Theta_0^{\mathfrak{h}, 1}, \ldots, \Theta_0^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}$ are independent, let $\alpha, \beta, \gamma \in \mathbb{R}$, $\delta \in (0, \infty)$ satisfy $0 < \alpha + \beta < 1$ and $2 \alpha + \beta < \gamma$, assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \mathbb{N} \cap (0, \mathfrak{h}]$ that $\mathfrak{h}^{\gamma} \Theta_0^{\mathfrak{h}, k}$ and $\mathfrak{h}^{\delta} \Theta_0^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}$ are standard normal, assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \mathbb{N} \cap (\mathfrak{h}, 2 \mathfrak{h}]$, $l \in \mathbb{N} \cap (2 \mathfrak{h}, 3 \mathfrak{h}]$, $x \in \mathbb{R}$ that 
\begin{equation}\label{eqn:prop:main:vjwj}
\textstyle \P\bigl(\mathfrak{h}^{\alpha} \Theta_0^{\mathfrak{h}, k} \ge x\bigr) = \P\bigl(\mathfrak{h}^{\beta} \Theta_0^{\mathfrak{h}, l} \ge x\bigr) = \int_{\max\{x, 0\}}^{\infty} [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y,
\end{equation}
and let $\varepsilon_1, \varepsilon_2 \in (0, \infty)$ satisfy $\varepsilon_1 < \min\{\alpha + \beta, \delta\}$ and $\varepsilon_2 \allowbreak < \min\{1, \gamma - 2 \alpha - \beta, (\gamma - \alpha)/2\}$. Then there exist $\Psi^{\mathfrak{h}} = (\Psi^{\mathfrak{h}, 1}, \ldots, \Psi^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}) \colon \Omega \to \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$, $\mathfrak{h} \in \mathbb{N}$, and $\mathfrak{c} \in (0, \infty)$ such that
\begin{enumerate}[label=(\roman*)]
\item
\label{item1:prop:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P(\limsup_{t \to \infty} \norm{\Theta_t^{\mathfrak{h}} - \Psi^{\mathfrak{h}}} = 0) = 1$,

\item
\label{item2:prop:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}} \Psi^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) = \P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) = 1$,

\item
\label{item3:prop:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\max_{j \in \{1, 2, \ldots, \mathfrak{h}\}}(\Psi^{\mathfrak{h}, \mathfrak{h} + j} \Psi^{\mathfrak{h}, 2 \mathfrak{h} + j} ) \le \mathfrak{h}^{- \varepsilon_1}\bigr) \ge 1 - \mathfrak{c} \mathfrak{h} ^{- \varepsilon_2}$,

\item
\label{item4:prop:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\abs{\Psi^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}} \le \mathfrak{h}^{- \varepsilon_1}\bigr) \ge 1 - \mathfrak{c} \mathfrak{h}^{- \varepsilon_2}$,

\item
\label{item5:prop:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\Psi^{\mathfrak{h}} \text{ is not a \cfadd{def:strict_saddle}\text{descending } critical point of } \mathcal{L}_{\mathfrak{h}}\bigr) = 1$, and

\item
\label{item6:prop:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\mathcal{N}^{\Psi^{\mathfrak{h}}}(\scrb) \ge f(\scrb)\bigr) \ge 1 - \mathfrak{c} \mathfrak{h}^{- \varepsilon_2}$\ifnocf.
\end{enumerate}
(cf.\ \cref{def:strict_saddle}).
\end{proposition}
\begin{cproof}{prop:main}
Throughout this proof let $L, r_1, r_2, r_3 \in (0, \infty)$ satisfy
\begin{equation}\label{eqn:prop:main:consts}
\begin{split}
& \textstyle L = \sup_{x \in [\scra, \scrb]} f'(x), \qquad \max\{\alpha + \beta, \varepsilon_2\} < r_1 < \min\{\gamma - \alpha - \varepsilon_2, 1\}, \\
& \textstyle \frac{r_1 + \varepsilon_2}{2} - \alpha - \beta < r_2 < r_1 - \alpha - \beta, \qquad \text{and} \qquad r_3 < \alpha + \beta - \varepsilon_1,
\end{split}
\end{equation}
let $c_1, c_2, c_3 \in (0, \infty)$ satisfy\footnote{\Nobs that \cref{item2:cor:bound_outer_weights} in \cref{cor:bound_outer_weights} provides the existence of $c_2$.} for all $\mathfrak{h} \in \mathbb{N}$ that
\begin{equation}\label{eqn:prop:main:c1c2c3}
\begin{split}
& \textstyle c_1 \ge \sup_{\mathfrak{k} \in \mathbb{N}} \bigl(2 \mathfrak{k}^{1 + \varepsilon_2} \exp(-\frac{1}{2} \mathfrak{k}^{\min\{r_3, 2 (\delta - \varepsilon_1)\}})\bigr), \\
& \textstyle \P\bigl(\sum_{j = 1}^{\ceil{\mathfrak{h}^{r_1}}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} \Theta_0^{\mathfrak{h}, \mathfrak{h} + 2 j} \ge \mathfrak{h}^{r_2}\bigr) \ge 1 - c_2 \mathfrak{h}^{r_1 - 2 \beta - 2 \alpha - 2 r_2}, \\
& \textstyle \text{and} \quad c_3 \ge \max_{\mathfrak{h} \in \mathbb{N} \cap (0, \max\{L^{1/r_2}, (2 / (\pi \scrb))^{1 / (\gamma - \alpha)}\})} \mathfrak{h}^{\varepsilon_2}\bigl[1 - \P\bigl(\mathcal{N}^{\Psi^{\mathfrak{h}}}(\scrb) \ge f(\scrb)\bigr)\bigr],
\end{split}
\end{equation}
and let $\mathfrak{c} \in (0, \infty)$ satisfy $\mathfrak{c} \ge \max\{c_1, \frac{4}{\pi \scrb} + c_2, c_3\}$. \Nobs that \cref{eqn:prop:main:vjwj} ensures that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}\label{eqn:prop:main:vjwj>0}
\textstyle \P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) = \bigl[\P\bigl(\Theta_0^{\mathfrak{h}, \mathfrak{h} + 1} > 0\bigr)\bigr]^{2 \mathfrak{h}} = \bigl[\int_0^{\infty} \bigl[\frac{2}{\pi}\bigr]^{1/2} \exp(-\frac{y^2}{2}) \, \d y\bigr]^{2 \mathfrak{h}} = 1.
\end{equation}
Combining this with \cref{lemma:GF_convergence} shows that there exists $\Psi^{\mathfrak{h}} = (\Psi^{\mathfrak{h}, 1}, \ldots, \Psi^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}) \colon \Omega \to \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$, $\mathfrak{h} \in \mathbb{N}$, which satisfies for all $\mathfrak{h} \in \mathbb{N}$ that 
\begin{equation}
\textstyle \P(\limsup_{t \to \infty} \norm{\Theta_t^{\mathfrak{h}} - \Psi^{\mathfrak{h}}} = 0) \ge \P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) = 1.
\end{equation}
This establishes \cref{item1:prop:main}. Next \nobs that \cref{eqn:prop:main:gradient} ensures that for all $\mathfrak{h} \in \mathbb{N}$, $k \in \mathbb{N} \cap (\mathfrak{h}, 3 \mathfrak{h} + 1]$, $\theta \in \mathbb{R}^{3 \mathfrak{h}}$ it holds that $\mathcal{G}_{\mathfrak{h}}^{k}(\theta) = 0$. This shows that
\begin{equation}\label{eqn:prop:main:fixed_components_of_gf}
\textstyle \forall\, \mathfrak{h} \in \mathbb{N}, k \in \mathbb{N} \cap (\mathfrak{h}, 3 \mathfrak{h} + 1] \colon \P(\Psi^{\mathfrak{h}, k} = \Theta_0^{\mathfrak{h}, k}) = 1.
\end{equation}
Combining this with \cref{eqn:prop:main:vjwj>0} implies that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}\textstyle \P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}}\Psi^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) = \P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}}\Theta_0^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) = 1.
\end{equation}
This establishes \cref{item2:prop:main}. Next \nobs that \cref{item1:cor:bound_outer_weights} in \cref{cor:bound_outer_weights}, \cref{eqn:prop:main:vjwj}, \cref{eqn:prop:main:consts}, \cref{eqn:prop:main:c1c2c3}, and \cref{eqn:prop:main:fixed_components_of_gf} assure that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P\bigl(\max_{j \in \{1, 2, \ldots, \mathfrak{h}\}}(\Psi^{\mathfrak{h}, \mathfrak{h} + j} \Psi^{\mathfrak{h}, 2 \mathfrak{h} + j}) \le \mathfrak{h}^{- \varepsilon_1}\bigr) \ge \P\bigl(\max_{j \in \{1, 2, \ldots, \mathfrak{h}\}}(\Psi^{\mathfrak{h}, \mathfrak{h} + j} \Psi^{\mathfrak{h}, 2 \mathfrak{h} + j}) \le \mathfrak{h}^{r_3 - \alpha - \beta}\bigr) \\
& \textstyle = \P\bigl(\max_{j \in \{1, 2, \ldots, \mathfrak{h}\}}(\Theta_0^{\mathfrak{h}, \mathfrak{h} + j} \Theta_0^{\mathfrak{h}, 2 \mathfrak{h} + j}) \le \mathfrak{h}^{r_3 - \alpha - \beta}\bigr) \ge 1 - 2 \mathfrak{h} \exp(-\frac{1}{2} \mathfrak{h}^{r_3}) \\
& \textstyle \ge 1 - c_1 \mathfrak{h}^{- \varepsilon_2} \ge 1 - \mathfrak{c} \mathfrak{h}^{- \varepsilon_2}.
\end{split}
\end{equation}
This establishes \cref{item3:prop:main}. Furthermore, \nobs that \cref{lemma:outer_bias_size}, \cref{eqn:prop:main:c1c2c3}, and \cref{eqn:prop:main:fixed_components_of_gf} demonstrate that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P\bigl(\abs{\Psi^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}} \le \mathfrak{h}^{- \varepsilon_1}\bigr) = \P\bigl(\abs{\Theta_0^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}} \le \mathfrak{h}^{- \varepsilon_1}\bigr) \ge 1 - \exp(-\frac{1}{2} \mathfrak{h}^{2 (\delta - \varepsilon_1)}) \ge 1 - c_1 \mathfrak{h}^{- \varepsilon_2} \ge 1 - \mathfrak{c} \mathfrak{h}^{- \varepsilon_2}.
\end{equation}
This establishes \cref{item4:prop:main}. In the next step \nobs that \cref{cor:strict:saddle:conv:random} proves that the set of initial values of the GF such that GF converges to a \cfadd{def:strict_saddle}\text{descending } critical point of the corresponding risk function has Lebesgue measure zero. This establishes \cref{item5:prop:main}. In addition, \nobs that \cref{item2:cor:bound_outer_weights} in \cref{cor:bound_outer_weights}, \cref{eqn:prop:main:vjwj}, \cref{eqn:prop:main:consts}, \cref{eqn:prop:main:c1c2c3}, and \cref{eqn:prop:main:fixed_components_of_gf} demonstrate that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}\label{eqn:prop:main:item4}
\begin{split}
& \textstyle \P\bigl(\sum_{j = 1}^{\ceil{\mathfrak{h}^{r_1}}} \Psi^{\mathfrak{h}, \mathfrak{h} + j} \Psi^{\mathfrak{h}, \mathfrak{h} + 2 j} \ge \mathfrak{h}^{r_2}\bigr) \\
& \textstyle = \P\bigl(\sum_{j = 1}^{\ceil{\mathfrak{h}^{r_1}}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} \Theta_0^{\mathfrak{h}, \mathfrak{h} + 2 j} \ge \mathfrak{h}^{r_2}\bigr) \ge 1 - c_2 \mathfrak{h}^{r_1 - 2 \alpha - 2 \beta - 2 r_2} \ge 1 - c_2 \mathfrak{h}^{- \varepsilon_2}.
\end{split}
\end{equation}
Next \nobs that \cref{eqn:prop:main:c1c2c3} proves that for all $\mathfrak{h} \in \mathbb{N} \cap (0, \max\{L^{1/r_2}, (2 / (\pi \scrb))^{1 / (\gamma - \alpha)}\})$ it holds that
\begin{equation}\label{eqn:prop:main:N>1_at_1}
\textstyle \P\bigl(\mathcal{N}^{\Psi^{\mathfrak{h}}}(\scrb) \ge f(\scrb)\bigr) = 1 - \mathfrak{h}^{- \varepsilon_2} \bigl(\mathfrak{h}^{\varepsilon_2}\bigl[1 - \P\bigl(\mathcal{N}^{\Psi^{\mathfrak{h}}}(\scrb) \ge f(\scrb)\bigr)\bigr]\bigr) \ge 1 - c_3 \mathfrak{h}^{- \varepsilon_2} \ge 1 - \mathfrak{c} \mathfrak{h}^{- \varepsilon_2}.
\end{equation}
In the next step \nobs that \cref{lemma:N>f_at_1}, \cref{cor:kinks_location} (applied with $\sigma \with 1$, $\varepsilon \with r_1$ in the notation of \cref{cor:kinks_location}), \cref{eqn:prop:main:consts}, \cref{eqn:prop:main:c1c2c3}, \cref{eqn:prop:main:fixed_components_of_gf}, and \cref{eqn:prop:main:item4} show that for all $\mathfrak{h} \in \mathbb{N} \cap [\max\{L^{1/r_2}, (2 / (\pi \scrb))^{1 / (\gamma - \alpha)}\}, \infty)$ it holds that
\begin{equation}
\begin{split}
& \textstyle \P\bigl(\mathcal{N}^{\Psi^{\mathfrak{h}}}(\scrb) \ge f(\scrb)\bigr) \\
& \textstyle \ge \P\bigl(\bigl[\sum_{j = 1}^{\ceil{\mathfrak{h}^{r_1}}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} \Theta_0^{\mathfrak{h}, \mathfrak{h} + 2 j} \ge L\bigr] \cap \bigl[\min_{j \in \{1, 2, \ldots, \ceil{\mathfrak{h}^{r_1}}\}}(\Theta_0^{\mathfrak{h}, j} / \Theta_0^{\mathfrak{h}, \mathfrak{h} + j}) > - \scrb\bigr]\bigr) \\
& \textstyle = 1 - \P\bigl(\bigl[\sum_{j = 1}^{\ceil{\mathfrak{h}^{r_1}}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} \Theta_0^{\mathfrak{h}, \mathfrak{h} + 2 j} < L\bigr] \cup \bigl[\min_{j \in \{1, 2, \ldots, \ceil{\mathfrak{h}^{r_1}}\}}(\Theta_0^{\mathfrak{h}, j} / \Theta_0^{\mathfrak{h}, \mathfrak{h} + j}) \le - \scrb\bigr]\bigr) \\
& \textstyle \ge 1 - \P\bigl(\sum_{j = 1}^{\ceil{\mathfrak{h}^{r_1}}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} \Theta_0^{\mathfrak{h}, \mathfrak{h} + 2 j} < L\bigr) - \P\bigl(\min_{j \in \{1, 2, \ldots, \ceil{\mathfrak{h}^{r_1}}\}}(\Theta_0^{\mathfrak{h}, j} / \Theta_0^{\mathfrak{h}, \mathfrak{h} + j}) \le - \scrb\bigr) \\
& \textstyle = \P\bigl(\sum_{j = 1}^{\ceil{\mathfrak{h}^{r_1}}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} \Theta_0^{\mathfrak{h}, \mathfrak{h} + 2 j} \ge L\bigr) + \P\bigl(\min_{j \in \{1, 2, \ldots, \ceil{\mathfrak{h}^{r_1}}\}}(\Theta_0^{\mathfrak{h}, j} / \Theta_0^{\mathfrak{h}, \mathfrak{h} + j}) > - \scrb\bigr) - 1 \\
& \textstyle \ge \P\bigl(\sum_{j = 1}^{\ceil{\mathfrak{h}^{r_1}}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} \Theta_0^{\mathfrak{h}, \mathfrak{h} + 2 j} \ge \mathfrak{h}^{r_2}\bigr) + \P\bigl(\min_{j \in \{1, 2, \ldots, \ceil{\mathfrak{h}^{r_1}}\}}(\Theta_0^{\mathfrak{h}, j} / \Theta_0^{\mathfrak{h}, \mathfrak{h} + j}) > - \scrb\bigr) - 1 \\
& \ge \textstyle 1 - c_2 \mathfrak{h}^{- \varepsilon_2} + 1 - \frac{4}{\pi \scrb} \mathfrak{h}^{r_1 + \alpha - \gamma} - 1 \ge 1 - (c_2 + \frac{4}{\pi \scrb}) \mathfrak{h}^{- \varepsilon_2} \ge 1 - \mathfrak{c} \mathfrak{h}^{- \varepsilon_2}.
\end{split}
\end{equation}
Combining this with \cref{eqn:prop:main:N>1_at_1} proves that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}
\textstyle \P\bigl(\mathcal{N}^{\Psi^{\mathfrak{h}}}(\scrb) \ge f(\scrb)\bigr) \ge 1 - \mathfrak{c} \mathfrak{h}^{- \varepsilon_2}.
\end{equation}
This establishes \cref{item6:prop:main}.
\end{cproof}


\begin{theorem}
\label{cor:main}
Let $\scra \in \mathbb{R}$, $\scrb \in (\max\{\scra, 0\}, \infty)$, let $f \in C^1(\mathbb{R}, \mathbb{R})$, $\mathfrak{p} \in C(\mathbb{R}, \mathbb{R})$ be piecewise polynomial, assume for all $x \in (\scra, \scrb)$ that $f'(x) > f(\scra) = 0 \le \mathfrak{p}(x)$, $\mathfrak{p}^{-1}(\mathbb{R} \backslash \{0\}) = (\scra, \scrb)$, and $f$ is strictly convex, let $(\Omega, \mathcal{F}, \P)$ be a probability space, for every $\mathfrak{h} \in \mathbb{N}$ let $\mathfrak{d}_{\mathfrak{h}} = 3 \mathfrak{h} + 1$, let $\mathcal{L}_{\mathfrak{h}} \allowbreak \colon \allowbreak \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}} \allowbreak \to \allowbreak \mathbb{R}$ satisfy for all $\theta = (\theta_1, \ldots, \theta_{\mathfrak{d}_{\mathfrak{h}}}) \in \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ that
\begin{equation}
\textstyle \mathcal{L}_{\mathfrak{h}}(\theta) = \int_{\mathbb{R}} (f(x) - \theta_{\mathfrak{d}_{\mathfrak{h}}} - \sum_{j = 1}^{\mathfrak{h}} \theta_{2 \mathfrak{h} + j} \max\{\theta_{\mathfrak{h} + j} x + \theta_{j}\})^2 \mathfrak{p}(x) \, \d x,
\end{equation}
let $\mathcal{G}_{\mathfrak{h}} = (\mathcal{G}_{\mathfrak{h}}^1, \ldots, \mathcal{G}_{\mathfrak{h}}^{\mathfrak{d}_{\mathfrak{h}}}) \colon \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}} \to \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ satisfy for all $\theta = (\theta_1, \ldots, \theta_{\mathfrak{d}_{\mathfrak{h}}}) \in \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$, $k \in \{1, 2, \ldots, \mathfrak{d}_{\mathfrak{h}}\}$ that
\begin{equation}\label{eqn:cor:main:gradient}
\textstyle \mathcal{G}_{\mathfrak{h}}^{k}(\theta) = (\frac{\partial^{-}}{\partial \theta_k} \mathcal{L}_{\mathfrak{h}})(\theta) \mathbbm{1}_{\mathbb{N} \cap (0, \mathfrak{h}]}(k),
\end{equation}
and let $\Theta^{\mathfrak{h}} = (\Theta^{\mathfrak{h}, 1}, \ldots, \Theta^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}) \colon [0, \infty) \times \Omega \to \mathbb{R}^{\mathfrak{d}_{\mathfrak{h}}}$ be a stoch. proc. with c.s.p., assume for all $\mathfrak{h} \in \mathbb{N}$, $t \in [0, \infty)$ that $\P(\Theta_t^{\mathfrak{h}} = \Theta_0^{\mathfrak{h}} - \int_0^{t} \mathcal{G}_{\mathfrak{h}}(\Theta_s^{\mathfrak{h}}) \, \d s) = 1$, assume for all $\mathfrak{h} \in \mathbb{N}$ that $\Theta_0^{\mathfrak{h}, 1}, \ldots, \Theta_0^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}$ are independent, let $\alpha, \beta, \gamma \in \mathbb{R}$, $\delta \in (0, \infty)$ satisfy $0 < \alpha + \beta < 1$ and $2 \alpha + \beta < \gamma$, assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \mathbb{N} \cap (0, \mathfrak{h}]$ that $\mathfrak{h}^{\gamma} \Theta_0^{\mathfrak{h}, k}$ and $\mathfrak{h}^{\delta} \Theta_0^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}$ are standard normal, and assume for all $\mathfrak{h} \in \mathbb{N}$, $k \in \mathbb{N} \cap (\mathfrak{h}, 2 \mathfrak{h}]$, $l \in \mathbb{N} \cap (2 \mathfrak{h}, 3 \mathfrak{h}]$, $x \in \mathbb{R}$ that 
\begin{equation}\label{eqn:cor:main:vjwj}
\textstyle \P\bigl(\mathfrak{h}^{\alpha} \Theta_0^{\mathfrak{h}, k} \ge x\bigr) = \P\bigl(\mathfrak{h}^{\beta} \Theta_0^{\mathfrak{h}, l} \ge x\bigr) = \int_{\max\{x, 0\}}^{\infty} [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y
\end{equation}
(cf.\ \cref{def:piecewise_polynomial}). Then there exist $\mathfrak{c}, \kappa \in (0, \infty)$ and random variables $\mathfrak{C}_{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $\mathfrak{h} \in \mathbb{N}$, such that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $\limsup_{t \to \infty} \mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}})] \le \mathfrak{c} \mathfrak{h}^{- \kappa}$ and
\begin{equation}
\textstyle \P(\forall\, t \in (0, \infty) \colon \mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}}) \le \mathfrak{c} \mathfrak{h}^{- \kappa} + \mathfrak{C}_{\mathfrak{h}} t^{- 1}) \ge 1 - \mathfrak{c} \mathfrak{h}^{- \kappa}.
\end{equation}
\end{theorem}
\begin{cproof}{cor:main}
Throughout this proof let $\varepsilon_1, \varepsilon_2 \in (0, \infty)$ satisfy $\varepsilon_1 < \min\{\alpha + \beta, \delta\}$ and $\varepsilon_2 < \min\{1, \gamma - 2 \alpha - \beta, (\gamma - \alpha) / 2\}$, let $\Psi^{\mathfrak{h}} = (\Psi^{\mathfrak{h}, 1}, \ldots, \Psi^{\mathfrak{h}, 2 \mathfrak{h}}) \colon \Omega \to \mathbb{R}^{2 \mathfrak{h}}$, $\mathfrak{h} \in \mathbb{N}$, and $c \in (0, \infty)$ satisfy that
\begin{enumerate}[label=(\Roman*)]
\item
\label{item1:cor:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P(\limsup_{t \to \infty} \norm{\Theta_t^{\mathfrak{h}} - \Psi^{\mathfrak{h}}} = 0) = 1$,

\item
\label{item2:cor:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}} \Psi^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) = \P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) = 1$,

\item
\label{item3:cor:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\max_{j \in \{1, 2, \ldots, \mathfrak{h}\}}(\Psi^{\mathfrak{h}, \mathfrak{h} + j} \Psi^{\mathfrak{h}, 2 \mathfrak{h} + j} ) \le \mathfrak{h}^{- \varepsilon_1}\bigr) \ge 1 - c \mathfrak{h} ^{- \varepsilon_2}$,

\item
\label{item4:cor:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\abs{\Psi^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}} \le \mathfrak{h}^{- \varepsilon_1}\bigr) \ge 1 - c \mathfrak{h}^{- \varepsilon_2}$,

\item
\label{item5:cor:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\Psi^{\mathfrak{h}} \text{ is not a \cfadd{def:strict_saddle}\text{descending } critical point of } \mathcal{L}_{\mathfrak{h}}\bigr) = 1$, and

\item
\label{item6:cor:main} it holds for all $\mathfrak{h} \in \mathbb{N}$ that $\P\bigl(\mathcal{N}^{\Psi^{\mathfrak{h}}}(\scrb) \ge f(\scrb)\bigr) \ge 1 - c \mathfrak{h}^{- \varepsilon_2}$
\end{enumerate}
(cf.\ \cref{prop:main}), and let $C, \varepsilon, L \in (0, \infty)$ satisfy
\begin{equation}\label{eqn:cor:main:Landfc}
\textstyle C \ge \max\{3 c, (L + 1) (\scrb - \scra) \sup_{x \in [\scra, \scrb]} \mathfrak{p}(x)\}, \,\, \varepsilon \le \min\{\varepsilon_1, \varepsilon_2\}, \,\, \text{and} \,\, L = \sup_{x \in [\scra, \scrb]} f'(x)
\end{equation}
(cf.\ \cref{def:strict_saddle}). \Nobs that \cref{eqn:cor:main:gradient}, \cref{eqn:cor:main:vjwj}, \cref{item2:cor:main}, and \cref{lemma:GF_convergence} ensure that there exist random variables $\mathfrak{C}_{\mathfrak{h}} \colon \Omega \to \mathbb{R}$, $\mathfrak{h} \in \mathbb{N}$, which satisfy for all $\mathfrak{h} \in \mathbb{N}$, $T \in (0, \infty)$ that
\begin{equation}\label{eqn:cor:main:Lojasiewicz}
\begin{split}
& \textstyle \min\bigl\{\P\bigl(\forall\, t \in (0, \infty) \colon 0 \le \mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}}) - \mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le \mathfrak{C}_{\mathfrak{h}} t^{-1}\bigr), \P\bigl(0 \le \mathcal{L}_{\mathfrak{h}}(\Theta_T^{\mathfrak{h}}) - \mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le \mathfrak{C}_{\mathfrak{h}} T^{-1}\bigr)\bigr\} \\
& \textstyle \ge \P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}} \Theta_0^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) = 1.
\end{split}
\end{equation}
Combining this with \cref{prop:convergence_gf_suitable_cp}, \cref{item1:cor:main}, \cref{item2:cor:main}, \cref{item3:cor:main}, \cref{item4:cor:main}, \cref{item5:cor:main}, \cref{item6:cor:main}, and \cref{eqn:cor:main:Landfc} demonstrates that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}\label{eqn:cor:main:ineqq}
\begin{split}
& \textstyle \P\bigl(\forall\, t \in (0, \infty) \colon \mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}}) \le C \mathfrak{h}^{- \varepsilon} + \mathfrak{C}_{\mathfrak{h}} t ^{-1}\bigr) \\
& \textstyle \ge \P\bigl([\forall\, t \in (0, \infty) \colon 0 \le \mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}}) - \mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le \mathfrak{C}_{\mathfrak{h}} t^{-1}] \cap [\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le C \mathfrak{h}^{- \varepsilon}]\bigr) \\
& \textstyle \ge \P\bigl(\forall\, t \in (0, \infty) \colon 0 \le \mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}}) - \mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le \mathfrak{C}_{\mathfrak{h}} t^{-1}\bigr) + \P\bigl(\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le C \mathfrak{h}^{- \varepsilon}\bigr) - 1 \\
& \textstyle = \P\bigl(\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le C \mathfrak{h}^{- \varepsilon}\bigr) \ge \P\bigl(\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le C \mathfrak{h}^{- \varepsilon_1}\bigr) \\
& \textstyle \ge \P\bigl(\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le \mathfrak{h}^{- \varepsilon_1} (L + \mathfrak{h}^{- \varepsilon_1}) (\scrb - \scra) \sup_{x \in [\scra, \scrb]} \mathfrak{p}(x)\bigr) \\
& \textstyle \ge \P\bigl([\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}} \Psi^{\mathfrak{h}, \mathfrak{h} + j} > 0] \cap [\max_{j \in \{1, 2, \ldots, \mathfrak{h}\}}(\Psi^{\mathfrak{h}, 2 \mathfrak{h} + j} \Psi^{\mathfrak{h}, \mathfrak{h} + j}) \le \mathfrak{h}^{- \varepsilon_1}] \\
& \textstyle \quad \cap [\Psi^{\mathfrak{h}} \text{ is not a \cfadd{def:strict_saddle}\text{descending } critical point of } \mathcal{L}_{\mathfrak{h}}] \cap [\mathcal{N}^{\Psi^{\mathfrak{h}}}(\scrb) \ge f(\scrb)] \cap [\abs{\Psi^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}} \le \mathfrak{h}^{- \varepsilon_1}]\bigr) \\
& \textstyle \ge \P\bigl(\min_{j \in \{1, 2, \ldots, 2\mathfrak{h}\}} \Psi^{\mathfrak{h}, \mathfrak{h} + j} > 0\bigr) + \P\bigl(\max_{j \in \{1, 2, \ldots, \mathfrak{h}\}}(\Psi^{\mathfrak{h}, 2 \mathfrak{h} + j} \Psi^{\mathfrak{h}, \mathfrak{h} + j}) \le \mathfrak{h}^{- \varepsilon_1}\bigr) \\
& \textstyle \quad + \P\bigl(\Psi^{\mathfrak{h}} \text{ is not a \cfadd{def:strict_saddle}\text{descending } critical point of } \mathcal{L}_{\mathfrak{h}}\bigr) + \P\bigl(\mathcal{N}^{\Psi^{\mathfrak{h}}}(\scrb) \ge f(\scrb)\bigr) \\
& \textstyle \quad + \P\bigl(\abs{\Psi^{\mathfrak{h}, \mathfrak{d}_{\mathfrak{h}}}} \le \mathfrak{h}^{- \varepsilon_1}\bigr) - 4 \ge 1 + (1 - c \mathfrak{h}^{- \varepsilon_2}) + 1 + (1 - c \mathfrak{h}^{- \varepsilon_2}) + (1 - c \mathfrak{h}^{- \varepsilon_2}) - 4 \\
& \textstyle = 1 - 3 c \mathfrak{h}^{- \varepsilon_2} \ge 1 - C \mathfrak{h}^{- \varepsilon_2} \ge 1 - C \mathfrak{h}^{- \varepsilon}.
\end{split}
\end{equation}
Next \nobs that \cref{eqn:cor:main:ineqq} implies that
\begin{equation}
\textstyle \P\bigl(\limsup_{\mathfrak{h} \to \infty} \mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) = 0\bigr) = 1.
\end{equation}
This shows that there exists $\mathscr{c} \in (0, \infty)$ which satisfies
\begin{equation}
\textstyle \forall\, \mathfrak{h} \in \mathbb{N} \colon \mathbb{E}[(\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}))^2] < \mathscr{c}.
\end{equation}
In addition, \nobs that \cref{eqn:cor:main:ineqq} and the H\"{o}lder inequality show that for all $\mathfrak{h} \in \mathbb{N}$ it holds that 
\begin{equation}
\begin{split}
& \textstyle \mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}})] = \mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \mathbbm{1}_{\{\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \le C \mathfrak{h}^{- \varepsilon}\}}] + \mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) \mathbbm{1}_{\{\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) > C \mathfrak{h}^{- \varepsilon}\}}] \\
& \textstyle \le C \mathfrak{h}^{- \varepsilon} + \bigl[\mathbb{E}[(\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}))^2]\bigr]^{1/2} \bigl[\P(\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}) > C \mathfrak{h}^{- \varepsilon})\bigr]^{1/2} \\
& \textstyle \le C \mathfrak{h}^{- \varepsilon} + \bigl[\mathbb{E}[(\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}}))^2]\bigr]^{1/2} [C \mathfrak{h}^{- \varepsilon}]^{1/2} \le C \mathfrak{h}^{- \varepsilon} + \mathscr{c}^{1/2} [C \mathfrak{h}^{- \varepsilon}]^{1/2} \\
& \textstyle \le (C + [\mathscr{c} C]^{1 / 2}) \mathfrak{h}^{-\varepsilon / 2}.
\end{split}
\end{equation}
Combining this with \cref{eqn:cor:main:Lojasiewicz} and the Dominated convergence theorem ensures that for all $\mathfrak{h} \in \mathbb{N}$ it holds that
\begin{equation}\label{eqn:cor:main:expectation_bound}
\textstyle \limsup_{t \to \infty} \mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}})] = \mathbb{E}[\limsup_{t \to \infty} \mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}})] = \mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Psi^{\mathfrak{h}})] \le (C + [\mathscr{c} C]^{1 / 2}) \mathfrak{h}^{-\varepsilon / 2}.
\end{equation}
Let $\mathfrak{c}, \kappa \in (0, \infty)$ satisfy $\mathfrak{c} \ge C + [\mathscr{c} C]^{1/2}$ and $\kappa \le \varepsilon / 2$. \Nobs that \cref{eqn:cor:main:ineqq} and \cref{eqn:cor:main:expectation_bound} demonstrate that for all $\mathfrak{h} \in \mathbb{N}$ it holds that $\limsup_{t \to \infty} \mathbb{E}[\mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}})] \le (C + [\mathscr{c} C]^{1 / 2}) \mathfrak{h}^{-\varepsilon / 2} \le \mathfrak{c} \mathfrak{h}^{- \kappa}$ and 
\begin{equation}
\begin{split}
& \textstyle \P\bigl(\forall\, t \in (0, \infty) \colon \mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}}) \le \mathfrak{c} \mathfrak{h}^{- \kappa} + \mathfrak{C}_{\mathfrak{h}} t ^{-1}\bigr) \\
& \textstyle \ge \P\bigl(\forall\, t \in (0, \infty) \colon \mathcal{L}_{\mathfrak{h}}(\Theta_t^{\mathfrak{h}}) \le C \mathfrak{h}^{- \varepsilon} + \mathfrak{C}_{\mathfrak{h}} t ^{-1}\bigr) \ge 1 - C \mathfrak{h}^{- \varepsilon} \ge 1 - \mathfrak{c} \mathfrak{h}^{- \kappa}.
\end{split}
\end{equation}
\end{cproof}





\section{Numerical simulations}
\label{sec:numerical_simulations}

In this section we complement the analytical findings of this work by means of several numerical simulation results for shallow ANNs with just one hidden layer (see \cref{fig:inner_biases_ReLU_1_hidden_layer} in Subsection~\ref{subsec_num_sim_ReLU_inner_bias_1_hidden_layer}, \cref{fig:all_biases_clipping_1_hidden_layer} in Subsection~\ref{subsec_num_sim_clipping_biases_1_hidden_layer}, \cref{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like} in Subsection~\ref{subsec_num_sim_ReLU_all_params_1_hidden_layer_Xavier}, and \cref{fig:all_parameters_ReLU_1_hidden_layer_He} in Subsection~\ref{subsec_num_sim_ReLU_all_params_1_hidden_layer_He}) as well as for deep ANNs with two (see \cref{fig:all_parameters_ReLU_2_hidden_layers_Xavier} in Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier}, \cref{fig:all_parameters_ReLU_2_hidden_layers_He} in Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_He}, \cref{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam} in Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier_Adam}, and \cref{fig:all_parameters_ReLU_2_hidden_layers_He_Adam} in Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_He_Adam}) or three (see \cref{fig:all_parameters_ReLU_3_hidden_layers_Xavier} in Subsection~\ref{subsec_num_sim_ReLU_all_params_3_hidden_layers_Xavier} and \cref{fig:all_parameters_ReLU_3_hidden_layers_He} in Subsection~\ref{subsec_num_sim_ReLU_all_params_3_hidden_layers_He}) hidden layers. These numerical simulation results all suggest that with high probability the considered GD optimization method overcomes all bad non-global local minima, does not converge to a global minimum, but does converge in probability to good non-optimal generalized critical points (cf.\ \cref{def:limit:subdiff,cor:gradient:limit:zero:abstract}) whose risk values are very close to the optimal risk value. The source codes used to create these numerical simulation results are all presented in \cref{sec:python_source_codes}. All numerical simulations have been performed in {\sc Python} using {\sc TensorFlow~2.9.2}.



\subsection{Training ANNs via stochastic gradient descent (SGD) optimization methods}
\label{subsection:dnn:framework}


In \cref{setting:dnn} below we describe the mathematical framework which we employ in all of our numerical simulations in \cref{subsec_num_sim_clipping_biases_1_hidden_layer,subsec_num_sim_ReLU_inner_bias_1_hidden_layer,subsec_num_sim_ReLU_all_params_1_hidden_layer_Xavier,subsec_num_sim_ReLU_all_params_1_hidden_layer_He,subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier,subsec_num_sim_ReLU_all_params_2_hidden_layers_He,subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier_Adam,subsec_num_sim_ReLU_all_params_2_hidden_layers_He_Adam,subsec_num_sim_ReLU_all_params_3_hidden_layers_Xavier,subsec_num_sim_ReLU_all_params_3_hidden_layers_He} below. In \cref{eqn:setting:dnn:wb,eqn:setting:dnn:affine,eqn:setting:dnn:actmollifier,eqn:setting:dnn:multdim,eqn:setting:dnn:realization} in \cref{setting:dnn} below we introduce the fully-connected feedforward ANNs which we employ to describe our numerical simulation results. We also refer to Figure~\ref{figure_dnn_illustration} for a graphical illustration of the ANN architecture used in \cref{eqn:setting:dnn:wb,eqn:setting:dnn:affine,eqn:setting:dnn:actmollifier,eqn:setting:dnn:multdim,eqn:setting:dnn:realization} in \cref{setting:dnn}.

\begin{setting}
\label{setting:dnn}
Let $(\ell_k)_{k \in \mathbb{N}_0} \subseteq \mathbb{N}$, $L, \mathfrak{d} \in \mathbb{N}$ satisfy $\mathfrak{d} = \sum_{k = 1}^L \ell_k (\ell_{k - 1} + 1)$, for every $\theta = (\theta_1, \dots, \theta_{\mathfrak{d}}) \in \mathbb{R}^{\mathfrak{d}}$ let $\mathfrak{w}^{k, \theta} = (\mathfrak{w}^{k, \theta}_{i, j})_{(i,j) \in \{1, \ldots, \ell_k\} \times \{1, \ldots, \ell_{k - 1}\}} \in \mathbb{R}^{\ell_k \times \ell_{k - 1}}$, $k \in \mathbb{N}$, and $\mathfrak{b}^{k, \theta} = (\mathfrak{b}^{k, \theta}_1, \dots, \allowbreak \mathfrak{b}^{k, \theta}_{\ell_k}) \in \mathbb{R}^{\ell_k}$, $k \in \mathbb{N}$, satisfy for all $k \in \{1, \dots, L\}$, $i \in \{1, \ldots, \ell_k\}$, $j \in \{1, \ldots, \ell_{k - 1}\}$ that
\begin{equation}\label{eqn:setting:dnn:wb}
\mathfrak{w}^{k, \theta}_{i, j} = \theta_{(i - 1) \ell_{k - 1} + j + \sum_{h = 1}^{k - 1} \ell_h (\ell_{h - 1} + 1)} \qquad \text{and} \qquad \mathfrak{b}^{k, \theta}_i = \theta_{\ell_k \ell_{k - 1} + i + \sum_{h = 1}^{k - 1} \ell_h (\ell_{h - 1} + 1)},
\end{equation}
for every $k \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$ let $\mathcal{A}_k^{\theta} = (\mathcal{A}_{k, 1}^{\theta}, \ldots, \mathcal{A}_{k, \ell_k}^{\theta}) \colon \mathbb{R}^{\ell_{k - 1}} \to \mathbb{R}^{\ell_k}$ satisfy for all $x \in \mathbb{R}^{\ell_{k - 1}}$ that
\begin{equation}\label{eqn:setting:dnn:affine}
\mathcal{A}_k^{\theta}(x) = \mathfrak{b}^{k, \theta} + \mathfrak{w}^{k, \theta} x,
\end{equation}
let $\mathbb{A}_r \colon \mathbb{R} \to \mathbb{R}$, $r \in \mathbb{N} \cup\cu{\infty}$, satisfy for all $x \in \mathbb{R}$ that $(\cup_{r \in \mathbb{N}} \cu{\mathbb{A}_r}) \subseteq C^1(\mathbb{R}, \mathbb{R})$, $\sup_{r \in \mathbb{N}} \allowbreak \sup_{y \in \allowbreak [-\abs{x}, \abs{x}]} \allowbreak \abs{(\mathbb{A}_{r})'(y)} < \infty$, and
\begin{equation}\label{eqn:setting:dnn:actmollifier}
\textstyle \limsup_{R \to \infty} \bigl[\sum_{r = R}^{\infty} \indicator{(0,\infty)} \big(\abs {\mathbb{A}_r(x) - \mathbb{A}_\infty(x)} + \limsup_{h \nearrow 0} \bigl| (\mathbb{A}_r)' (x) - \frac{\mathbb{A}_\infty (x + h) - \mathbb{A}_\infty(x)}{h} \bigr|\big) \bigr] = 0,
\end{equation}
for every $r \in \mathbb{N} \cup\{\infty\}$, $k \in \mathbb{N}$ let $\mathfrak{M}_{r, k} \colon \mathbb{R}^{\ell_k} \to \mathbb{R}^{\ell_k}$ satisfy for all $x = (x_1, \ldots, x_{\ell_k}) \in \mathbb{R}^{\ell_k}$ that
\begin{equation}\label{eqn:setting:dnn:multdim}
\textstyle
\mathfrak{M}_{r, k}(x) = (\mathbb{A}_r(x_1), \ldots, \mathbb{A}_r(x_{\ell_k})),
\end{equation}
for every $r \in \mathbb{N} \cup \{\infty\}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$ let $\mathcal{N}^{k, \theta}_r = (\mathcal{N}^{k, \theta}_{r, 1}, \ldots, \mathcal{N}^{k, \theta}_{r, \ell_k}) \colon \mathbb{R}^{\ell_0} \to \mathbb{R}^{\ell_k}$, $k \in \mathbb{N}$, satisfy for all $k \in \mathbb{N}$ that
\begin{equation}\label{eqn:setting:dnn:realization}
\mathcal{N}^{1, \theta}_r = \mathcal{A}^{\theta}_1 \qquad \text{and} \qquad \mathcal{N}^{k + 1, \theta}_r = \mathcal{A}_{k + 1}^{\theta} \circ \mathfrak{M}_{r, k} \circ \mathcal{N}^{k, \theta}_r,
\end{equation}
let $f = (f_1, \ldots, f_{\ell_L}) \colon \mathbb{R}^{\ell_0} \to \mathbb{R}^{\ell_L}$ be measurable, let $(\Omega, \mathcal{F}, \mathbbm{P})$ be a probability space, let $X^n_k \colon \Omega \to \mathbb{R}^{\ell_0}$, $k, n \in \mathbb{N}$, be random variables, let $\mathbf{m} \in \mathbb{N}$, for every $n \in \mathbb{N}$, $r \in \mathbb{N} \cup \{\infty\}$ let $\mathfrak{L}^n_{r} \colon \mathbb{R}^{\mathfrak{d}} \times \Omega \to \mathbb{R}$ satisfy for all $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ that
\begin{equation}\label{eqn:setting:dnn:loss}
\textstyle \mathfrak{L}^{n}_{r}(\theta, \omega) = \frac{1}{\mathbf{m}} \sum_{k = 1}^{\mathbf{m}} \norm{\mathcal{N}^{L, \theta}_r(X^{n}_k (\omega)) - f(X_k^{n}(\omega))}^2,
\end{equation}
let $\fJ \subseteq \mathbb{N}$, for every $n\in \mathbb{N}$ let $\mathfrak{G}_n = (\mathfrak{G}_{n}^1, \ldots, \mathfrak{G}_{n}^{\mathfrak{d}}) \colon \mathbb{R}^{\mathfrak{d}} \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ satisfy for all $\theta \allowbreak \in \allowbreak \mathbb{R}^{\mathfrak{d}}$, $\omega \allowbreak \in \allowbreak \{w \in \Omega \colon ((\nabla_\theta \mathfrak{L}^{n}_r)(\theta, w))_{r \in \mathbb{N}} \text{ is } \text{convergent}\}$, $\mathfrak{j} \in \{1, 2, \ldots, \mathfrak{d}\}$ that
\begin{equation}
\label{eq:generalized_stochastic_gradients}
\textstyle \mathfrak{G}_{n}^{\mathfrak{j}}(\theta, \omega) = \bigl[\lim_{r \to \infty}(\frac{\partial}{\partial \theta_{\mathfrak{j}}} \mathfrak{L}^{n}_r)(\theta, \omega)\bigr] \indicator{\fJ}(\mathfrak{j}),
\end{equation}
for every $n \in \mathbb{N}$ let $\psi_n = (\psi_n^1, \ldots, \psi_n^\mathfrak{d}) \colon \mathbb{R}^{n \mathfrak{d}} \to \mathbb{R}^\mathfrak{d}$ be a function, and let $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \allowbreak \colon \allowbreak \mathbb{N}_0 \allowbreak \times \allowbreak \Omega \to \mathbb{R}^{\mathfrak{d}}$ be a stochastic process which satisfies for all $n \in \mathbb{N}$ that
\begin{equation}\label{eqn:setting:dnn:stochastic_pr}
\textstyle \Theta_n = \Theta_{n-1} - \psi_n(\mathfrak{G}_1(\Theta_0), \mathfrak{G}_2(\Theta_1), \ldots, \mathfrak{G}_n(\Theta_{n-1})).
\end{equation}
\end{setting}

In \cref{eqn:setting:dnn:stochastic_pr} above we make use of the setup in, e.g., (81) in Framework~6.1 in Becker et al.~\cite{Wurstemberger2022LRV} to formulate general stochastic gradient descent optimization methods covering, for instance, the standard stochastic gradient descent optimization method (cf.\ Becker et al.~\cite[Lemma~6.2]{Wurstemberger2022LRV} and \cref{lemma:StandardSGD} below) as well as the adaptive moment estimate method (Adam) (cf.\ Kingma \& Ba~\cite{DBLP:journals/corr/KingmaB14}, Becker et al.~\cite[Lemma~6.8]{Wurstemberger2022LRV}, and \cref{lemma:AdamSGD} below).


\def2.5cm{2.5cm}
\begin{figure}[H]
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tikzpicture}[shorten >=1pt,->,draw=black!50, node distance=2.5cm]
\tikzstyle{every pin edge}=[<-,shorten <=1pt]
\tikzstyle{input neuron}=[very thick, circle,draw=red, fill=red!20, minimum size=30pt,inner sep=0pt]
\tikzstyle{output neuron}=[very thick, circle, draw=green,fill=green!20,minimum size=60pt,inner sep=0pt]
\tikzstyle{hidden neuron}=[very thick, circle,draw=blue,fill=blue!20,minimum size=52pt,inner sep=0pt]
\tikzstyle{annot} = [text width=9em, text centered]
\tikzstyle{annot2} = [text width=4em, text centered]


\node[input neuron] (I-1) at (-3,-1.4) {$x_1$};
\node[input neuron] (I-2) at (-3,-2.6) {$x_2$};
\node(I-dots) at (-3,-3.36) {\vdots};
\node[input neuron] (I-3) at (-3,-4.3) {$x_{\ell_0}$};

\path[yshift = 1.5cm]
node[hidden neuron](H0-1) at (0*2.5cm, -1 cm) {$\mathcal{N}_{\infty, 1}^{1,\theta}(x)$};
\path[yshift = 1.5cm]
node[hidden neuron](H0-2) at (0*2.5cm, -3 cm) {$\mathcal{N}_{\infty, 2}^{1,\theta}(x)$};
\path[yshift = 1.5cm]
node[hidden neuron](H0-3) at (0*2.5cm, -5 cm) {$\mathcal{N}_{\infty, 3}^{1,\theta}(x)$};
\path[yshift = 1.5cm]
node(H0-dots) at (0*2.5cm, -6.2 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H0-4) at (0*2.5cm, -7.6 cm) {$\mathcal{N}_{\infty, \ell_1}^{1,\theta}(x)$};

\path[yshift = 1.5cm]
node[hidden neuron](H1-1) at (1.5*2.5cm, -1 cm) {$\mathcal{N}_{\infty, 1}^{2,\theta}(x)$};
\path[yshift = 1.5cm]
node[hidden neuron](H1-2) at (1.5*2.5cm, -3 cm) {$\mathcal{N}_{\infty, 2}^{2,\theta}(x)$};
\path[yshift = 1.5cm]
node[hidden neuron](H1-3) at (1.5*2.5cm, -5 cm) {$\mathcal{N}_{\infty, 3}^{2,\theta}(x)$};
\path[yshift = 1.5cm]
node(H1-dots) at (1.5*2.5cm, -6.2 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H1-4) at (1.5*2.5cm, -7.6 cm) {$\mathcal{N}_{\infty, \ell_2}^{2,\theta}(x)$};

\path[yshift = 0.5cm]
node(Hdot-1) at (2.7*2.5cm, -1.5 cm) {$\cdots$};
\path[yshift = 0.5cm]
node(Hdot-2) at (2.7*2.5cm, -2.9 cm) {$\cdots$};
\path[yshift = 0.5cm]
node(Hdot-dots) at (2.7*2.5cm, -4.1 cm) {$\ddots$};
\path[yshift = 0.5cm]
node(Hdot-3) at (2.7*2.5cm, -5.5 cm) {$\cdots$};

\path[yshift = 1.5cm]
node[hidden neuron](H2-1) at (3.9*2.5cm, -1 cm) {$\mathcal{N}_{\infty, 1}^{L - 1,\theta}(x)$};
\path[yshift = 1.5cm]
node[hidden neuron](H2-2) at (3.9*2.5cm, -3 cm) {$\mathcal{N}_{\infty, 2}^{L - 1,\theta}(x)$};
\path[yshift = 1.5cm]
node[hidden neuron](H2-3) at (3.9*2.5cm, -5 cm) {$\mathcal{N}_{\infty, 3}^{L - 1,\theta}(x)$};
\path[yshift = 1.5cm]
node(H2-dots) at (3.9*2.5cm, -6.2 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H2-4) at (3.9*2.5cm, -7.6 cm) {$\mathcal{N}_{\infty, \ell_{L - 1}}^{L - 1,\theta}(x)$};


\path[yshift = 1.5cm]
node[output neuron](O-1) at (5.4*2.5cm,-1.9 cm) {$\mathcal{N}_{\infty, 1}^{L,\theta}(x)$}; 
\path[yshift = 1.5cm]
node[output neuron](O-2) at (5.4*2.5cm,-4.2 cm) {$\mathcal{N}_{\infty, 2}^{L,\theta}(x)$}; 
\path[yshift = 1.5cm]
node(O-dots) at (5.4*2.5cm, -5.5 cm) {\vdots};
\path[yshift = 1.5cm]
node[output neuron](O-3) at (5.4*2.5cm,-7 cm) {$\mathcal{N}_{\infty, \ell_L}^{L,\theta}(x)$};
\foreach \source in {1,2,3}
\foreach \dest in {1,2,3,4}
\path[-{>[length=2mm, width=4mm]}, line width = 0.8] (I-\source) edge (H0-\dest);


\foreach \source in {1,2,3,4}
\foreach \dest in {1,2,3,4}
\path[-{>[length=2mm, width=4mm]}, line width = 0.8] (H0-\source) edge (H1-\dest);


\foreach \source in {1,2,3,4}
\foreach \dest in {1,2,3}
\draw[-{>[length=2mm, width=4mm]}, path fading=east] (H1-\source) -- (Hdot-\dest);

\foreach \source in {1,2,3}
\foreach \dest in {1,2,3,4}
\draw[-{>[length=2mm, width=4mm]}, path fading=west] (Hdot-\source) -- (H2-\dest);

\foreach \source in {1,2,3,4}
\foreach \dest in {1,2,3}
\path[-{>[length=2mm, width=4mm]}, line width = 0.8] (H2-\source) edge (O-\dest);


\node[annot,above of=H0-1, node distance=1.5cm, align=center] (hl) {$1^{\text{st}}$ hidden layer\\($2^{\text{nd}}$ layer)};
\node[annot,above of=H1-1, node distance=1.5cm, align=center] (hl) {$2^{\text{nd}}$ hidden layer\\($3^{\text{rd}}$ layer)};
\node[annot,above of=H2-1, node distance=1.5cm, align=center] (hl2) {${(L - 1)}^{\text{th}}$ hidden layer\\($L^{\text{th}}$ layer)};
\node[annot,above of=I-1, node distance=1.1cm, align=center] {Input layer\\ ($1^{\text{st}}$ layer)};
\node[annot,above of=O-1, node distance=1.62cm, align=center] {Output layer\\(${(L + 1)}^{\text{th}}$ layer)};

\node[annot2,below of=H0-4, node distance=1.5cm, align=center] (sl) {$\ell_1$ \\ neurons};
\node[annot2,below of=H1-4, node distance=1.5cm, align=center] (sl) {$\ell_2$ \\ neurons};
\node[annot2,below of=H2-4, node distance=1.5cm, align=center] (sl2) {$\ell_{L - 1}$ neurons};
\node[annot2,below of=I-3, node distance=1.1cm, align=center] {$\ell_0$ \\ neurons};
\node[annot2,below of=O-3, node distance=1.62cm, align=center] {$\ell_L$ \\ neurons};
\end{tikzpicture}
\end{adjustbox}
\caption{Graphical illustration of the ANN architecture used in \cref{eqn:setting:dnn:wb,eqn:setting:dnn:affine,eqn:setting:dnn:actmollifier,eqn:setting:dnn:multdim,eqn:setting:dnn:realization} in \cref{setting:dnn}: The realization function $\mathcal{N}_{\infty}^{L, \theta} \colon \mathbb{R}^{\ell_0} \to \mathbb{R}^{\ell_L}$ of the ANN associated to the parameter vector $\theta = (\theta_1, \ldots, \theta_{\mathfrak{d}}) \in \mathbb{R}^{\mathfrak{d}}$ maps the $\ell_0$-dimensional input vector $x = (x_1, \ldots, x_{\ell_0}) \in \mathbb{R}^{\ell_0}$ to the $\ell_{L}$-dimensional output vector $\mathcal{N}_{\infty}^{L, \theta}(x) = (\mathcal{N}_{\infty, 1}^{L, \theta}(x), \ldots, \mathcal{N}_{\infty, \ell_L}^{L, \theta}(x)) \in \mathbb{R}^{\ell_L}$.}
\label{figure_dnn_illustration} 
\end{figure}


\subsection{Examples of popular SGD optimization methods}


\subsubsection{Standard SGD optimization methods}
\label{subsubsec_Standard_SGD}

The next elementary lemma, \cref{lemma:StandardSGD}, illustrates that the framework in \cref{eqn:setting:dnn:stochastic_pr} in \cref{setting:dnn} covers plain vanilla standard SGD optimization processes. \cref{lemma:StandardSGD} is a slightly adapted variant of, e.g., Becker et al.~\cite[Lemma~6.2]{Wurstemberger2022LRV}.

\begin{lemma}\label{lemma:StandardSGD}
Assume \cref{setting:dnn}, let $(\gamma_n)_{n\in \mathbb{N}}\subseteq (0, \infty)$, and assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, \allowbreak g_n \in \mathbb{R}^{\mathfrak{d}}$ that $\psi_n(g_1, g_2, \ldots, g_n) = \gamma_n g_n$. Then it holds for all $n \in \mathbb{N}$ that 
\begin{equation}\label{eqn:standard_SGD}
\textstyle \Theta_{n} = \Theta_{n-1} - \gamma_n \mathfrak{G}_n(\Theta_{n-1}).
\end{equation}
\end{lemma}
\begin{cproof}{lemma:StandardSGD}
\Nobs that \cref{eqn:setting:dnn:stochastic_pr} establishes \cref{eqn:standard_SGD}.
\end{cproof}



\subsubsection{Adaptive moment estimation (Adam) SGD optimization methods}
\label{subsubsec_Adam_SGD}

The next elementary lemma, \cref{lemma:AdamSGD}, illustrates that the framework in \cref{eqn:setting:dnn:stochastic_pr} in \cref{setting:dnn} covers also the popular Adam optimization processes in Kingma \& Ba~\cite{DBLP:journals/corr/KingmaB14}. \cref{lemma:AdamSGD} is a slightly adapted variant of, e.g., Becker et al.~\cite[Lemma~6.8]{Wurstemberger2022LRV}.

\begin{lemma}\label{lemma:AdamSGD}
Assume \cref{setting:dnn}, let $\alpha, \beta \in (0, 1)$, $\varepsilon \in (0,\infty)$, let $(\gamma_n)_{n\in \mathbb{N}}\subseteq (0, \infty)$, assume $\fJ = \{1, 2, \ldots, \mathfrak{d}\}$, and assume for all $n\in \mathbb{N}$, $i \in \fJ$, $g_1 = (g_1^1, \ldots, g_1^{\mathfrak{d}})$, $g_2 = (g_2^1, \ldots, g_2^{\mathfrak{d}})$, $\dots$, $g_n = (g_n^1, \ldots, g_n^{\mathfrak{d}}) \in \mathbb{R}^{\mathfrak{d}}$ that
\begin{equation}\label{eqn:lemma:AdamSGD:psi}
\textstyle \psi_n^{i} (g_1, g_2, \ldots, g_n) \textstyle = \gamma_n \left[\frac{\sum_{k = 1}^n \alpha^{n-k} (1-\alpha) g_k^{i}}{1 - \alpha^n} \right] \left[\varepsilon + \Bigl[\frac{\sum_{k = 1}^n \beta^{n-k}(1-\beta) \abs{g_k^{i}}^2}{1 - \beta^n}\Bigr]^{\nicefrac{1}{2}} \right]^{-1}.
\end{equation}
Then there exist $\mathbf{m}  = (\mathbf{m}^{1}, \ldots, \mathbf{m}^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ and $\mathbb{M} = (\mathbb{M}^{1}, \ldots, \mathbb{M}^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ such that for all $n \in \mathbb{N}$, $i \in \fJ$ it holds that
\begin{equation}\label{eqn:lemma:AdamSGD:m}
\textstyle \mathbf{m}_0 = 0, \qquad \mathbf{m}_n = \alpha \mathbf{m}_{n-1} + (1 - \alpha) \mathfrak{G}_n(\Theta_{n - 1}),
\end{equation}
\begin{equation}\label{eqn:lemma:AdamSGD:M}
\textstyle \mathbb{M}_0 = 0, \qquad \mathbb{M}_n^{i} = \beta\,\mathbb{M}_{n-1}^{i} + (1 - \beta) \abs{\mathfrak{G}_{n}^i(\Theta_{n - 1})}^2,
\end{equation}
\begin{equation}\label{eqn:lemma:AdamSGD:Theta}
\textstyle \text{and} \qquad \Theta_n^i = \Theta_{n - 1}^i - \gamma_n \left[\frac{\mathbf{m}_n^{i}}{1 - \alpha^n}\right] \left[\varepsilon + \Bigl[\frac{\mathbb{M}_n^{i}}{1 - \beta^n}\Bigr]^{\nicefrac{1}{2}}\right]^{-1}.
\end{equation}
\end{lemma}
\begin{cproof}{lemma:AdamSGD}
Throughout this proof let $\mathbf{m}  = (\mathbf{m}^{1}, \ldots, \mathbf{m}^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ and $\mathbb{M} = (\mathbb{M}^{1}, \ldots, \mathbb{M}^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ satisfy for all $n \in \mathbb{N}_0$, $i \in \fJ$ that
\begin{equation}\label{eqn:lemma:AdamSGD:m_i}
\textstyle \mathbf{m}_n^{i} = \sum_{k = 1}^n \alpha^{n - k} (1 - \alpha) \mathfrak{G}_k^i(\Theta_{k - 1})
\end{equation}
and
\begin{equation}\label{eqn:lemma:AdamSGD:M_i}
\textstyle \mathbb{M}_n^{i} = \sum_{k = 1}^n \beta^{n - k} (1 - \beta) \abs{\mathfrak{G}_k^i(\Theta_{k - 1})}^2.
\end{equation}
\Nobs that \cref{eqn:lemma:AdamSGD:m_i,eqn:lemma:AdamSGD:M_i} ensure that for all $n \in \mathbb{N}$, $i \in \fJ$ it holds that
\begin{equation}\label{eqn:lemma:AdamSGD:m_i:induct}
\begin{split}
& \textstyle \mathbf{m}_n^{i} - \alpha \mathbf{m}_{n - 1}^{i} \\
& \textstyle = \sum_{k = 1}^n \alpha^{n - k} (1 - \alpha) \mathfrak{G}_k^i(\Theta_{k - 1}) - \alpha \sum_{k = 1}^{n - 1} \alpha^{n - 1 - k} (1 - \alpha) \mathfrak{G}_k^i(\Theta_{k - 1}) \\
& \textstyle = (1 - \alpha) \bigl[\sum_{k = 1}^n \alpha^{n - k} \mathfrak{G}_k^i(\Theta_{k - 1}) - \sum_{k = 1}^{n - 1} \alpha^{n - k} \mathfrak{G}_k^i(\Theta_{k - 1})\bigr] = (1 - \alpha) \mathfrak{G}_n^i(\Theta_{n - 1}),
\end{split}
\end{equation}
\begin{equation}\label{eqn:lemma:AdamSGD:M_i:induct}
\begin{split}
& \textstyle \mathbb{M}_n^{i} - \beta \mathbb{M}_{n - 1}^{i} \\
& \textstyle = \sum_{k = 1}^n \beta^{n - k} (1 - \beta) \abs{\mathfrak{G}_k^i(\Theta_{k - 1})}^2 - \beta \sum_{k = 1}^{n - 1} \beta^{n - 1 - k} (1 - \beta) \abs{\mathfrak{G}_k^i(\Theta_{k - 1})}^2 \\
& \textstyle = (1 - \beta) \bigl[\sum_{k = 1}^n \beta^{n - k} \abs{\mathfrak{G}_k^i(\Theta_{k - 1})}^2 - \sum_{k = 1}^{n - 1} \beta^{n - k} \abs{\mathfrak{G}_k^i(\Theta_{k - 1})}^2\bigr] = (1 - \beta) \abs{\mathfrak{G}_n^i(\Theta_{n - 1})}^2,
\end{split}
\end{equation}
and $\mathbf{m}_0 = \mathbb{M}_0 = 0$. Next \nobs that \cref{eqn:setting:dnn:stochastic_pr}, \cref{eqn:lemma:AdamSGD:psi}, \cref{eqn:lemma:AdamSGD:m_i}, and \cref{eqn:lemma:AdamSGD:M_i} demonstrate that for all $n \in \mathbb{N}$, $i \in \fJ$ it holds that
\begin{equation}
\begin{split}
& \textstyle \Theta_n^i - \Theta_{n - 1}^i = - \psi_n^i (\mathfrak{G}_1(\Theta_0), \mathfrak{G}_2(\Theta_1), \ldots, \mathfrak{G}_n(\Theta_{n-1})) \\
& \textstyle = - \gamma_n \left[\frac{\sum_{k = 1}^n \alpha^{n - k} (1 - \alpha) \mathfrak{G}_k^i(\Theta_{k - 1})}{1 - \alpha^n}\right] \left[\varepsilon + \Bigl[\frac{\sum_{k = 1}^n \beta^{n - k} (1 - \beta) \abs{\mathfrak{G}_k^i(\Theta_{k - 1})}^2}{1 - \beta^n}\Bigr]^{\nicefrac{1}{2}}\right]^{-1} \\
& \textstyle = - \gamma_n \left[\frac{\mathbf{m}_n^{i}}{1 - \alpha^n}\right] \left[\varepsilon + \Bigl[\frac{\mathbb{M}_n^{i}}{1 - \beta^n}\Bigr]^{\nicefrac{1}{2}}\right]^{-1}\!.
\end{split}
\end{equation}
Combining this with \cref{eqn:lemma:AdamSGD:m_i:induct,eqn:lemma:AdamSGD:M_i:induct} establishes \cref{eqn:lemma:AdamSGD:m}, \cref{eqn:lemma:AdamSGD:M}, and \cref{eqn:lemma:AdamSGD:Theta}.
\end{cproof}




\subsection{Properties of subdifferentials of convergent stochastic processes}


In the subsequent subsections of this section (\cref{subsec_num_sim_clipping_biases_1_hidden_layer,subsec_num_sim_ReLU_inner_bias_1_hidden_layer,subsec_num_sim_ReLU_all_params_1_hidden_layer_Xavier,subsec_num_sim_ReLU_all_params_1_hidden_layer_He,subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier,subsec_num_sim_ReLU_all_params_2_hidden_layers_He,subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier_Adam,subsec_num_sim_ReLU_all_params_2_hidden_layers_He_Adam,subsec_num_sim_ReLU_all_params_3_hidden_layers_Xavier,subsec_num_sim_ReLU_all_params_3_hidden_layers_He}) we complement the analytical findings of this article through several numerical simulation results for shallow and deep ANNs. These numerical simulation results all suggest that the sum of the $L^2(\P; \mathbb{R}^{\mathfrak{d}})$-norm of the generalized gradient function (cf., \cref{eq:generalized_stochastic_gradients} and \cite[Items~$($v$)$ and $($vi$)$ in Theorem~2.9]{DNNReLUarXiv}) of the risk function composed with the considered SGD type optimization process and of the $L^2(\P; \mathbb{R}^{\mathfrak{d}})$-distance between the SGD type optimization process and a random limit point converges to zero. In this subsection we show in the elementary results in \cref{lem:gradient:limit:zero:abstract,cor:gradient:limit:zero:abstract} below that one can conclude from this convergence property (see \cref{eqn:Frechet_convergence} below) that the random limit point must be a generalized critical point in the sense of limiting Fr\'{e}chet subdifferentiability (see \cref{def:limit:subdiff}). In our formulation of the conclusion of \cref{cor:gradient:limit:zero:abstract} we thus employ the notion of the set of limiting Fr\'{e}chet subdifferentials which we briefly recall in \cref{def:limit:subdiff} below (cf., e.g., \cite[Chapter~10]{RockafellarWets1998}, \cite[Definition~2.9]{EberleJentzenRiekertWeiss2021}, and \cite[Definition~3.1]{JentzenRiekert2021ExistenceGlobMin}).


\begin{lemma}
\label{lem:gradient:limit:zero:abstract}
Let $\mathfrak{d} \in \mathbb{N}$, $p \in ( 0 , \infty )$, let $(\Omega , \mathcal{F} , \P)$ be a probability space, let $\Theta = (\Theta_n )_{n \in \mathbb{N}_0 } \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^\mathfrak{d}$ be a stochastic process, let $\vartheta \colon \Omega \to \mathbb{R}^\mathfrak{d}$ be a random variable, let $\mathfrak{g} \colon \mathbb{R}^\mathfrak{d} \to [0 , \infty ]$ be lower semicontinuous, assume for all $n \in \mathbb{N}_0$ that $\mathbb{E} [ \norm{\Theta_n } ^p + \norm{\vartheta } ^p + \abs{ \mathfrak{g} ( \Theta_n ) }  ^p ] < \infty$, and assume
\begin{equation}\label{eqn:lem:gradient:limit:zero:abstract}
\textstyle \limsup_{n \to \infty} \mathbb{E} [ \norm{ \Theta_n - \vartheta } ^p + \abs{\mathfrak{g} ( \Theta_n ) } ^p ] = 0.
\end{equation}
Then $\P ( \mathfrak{g} ( \vartheta ) = 0 ) = 1$.
\end{lemma}


\begin{cproof}{lem:gradient:limit:zero:abstract}
\Nobs that \cref{eqn:lem:gradient:limit:zero:abstract} implies that there exist $E \in \mathcal{F}$ and a strictly increasing $k \colon \mathbb{N}_0 \to \mathbb{N}_0$ which satisfy $\P ( E ) = 1$ and
\begin{equation}\label{eqn:subseq_k_n}
\textstyle \forall\, \omega \in E \colon \limsup_{n \to \infty} ( \norm{ \Theta_{ k ( n ) } ( \omega ) - \vartheta ( \omega ) } ^p + \abs{ \mathfrak{g}  ( \Theta _ { k ( n ) } ( \omega ) ) } ^p ) = 0.
\end{equation}
\Nobs that \cref{eqn:subseq_k_n} and the fact that $\mathfrak{g}$ is lower semicontinuous show for all $\omega \in E$ that
\begin{equation}
\textstyle
\mathfrak{g} ( \vartheta ( \omega ) ) = \mathfrak{g} \rbr*{ \lim_{n \to \infty } \Theta_{ k ( n ) } ( \omega  ) }
\le \liminf_{n \to \infty } \mathfrak{g}  ( \Theta _ { k ( n ) } ( \omega ) ) = 0 .
\end{equation}
Hence, we obtain for all $\omega \in E$ that $\mathfrak{g} ( \vartheta ( \omega ) ) = 0$.
\end{cproof}


\begin{definition}[Fr\'{e}chet subdifferentials and limiting Fr\'{e}chet subdifferentials]
\label{def:limit:subdiff}
Let $n \in \mathbb{N}$, $\mathcal{L} \in C(\mathbb{R}^n, \mathbb{R})$, $x \in \mathbb{R}^n$. Then we denote by 
$(\mathcal{D} \mathcal{L})(x) \subseteq \mathbb{R}^n$ the set given by
\begin{equation}
\textstyle (\mathcal{D} \mathcal{L})(x) = \cu*{y \in \mathbb{R}^n \colon 
\br*{ \liminf_{\mathbb{R}^n \backslash \cu{  0 } \ni h \to 0 } 
\rbr*{ \frac{\mathcal{L}(x + h) - \mathcal{L}(x) - \spro{y, h}}{\norm{h}}} \geq 0}}
\end{equation}
and we denote by $(\mathbb{D} \mathcal{L})(x) \subseteq \mathbb{R}^n$ the set given by
\begin{equation}
\textstyle (\mathbb{D} \mathcal{L})(x) = \bigcap_{\varepsilon \in (0, \infty) } 
\overline{\br*{\cup_{y \in \cu{ z \in \mathbb{R}^n \colon \norm{ x - z } < \varepsilon}}(\mathcal{D} \mathcal{L})(y)}}.
\end{equation}
\end{definition}


\cfclear
\begin{cor}
\label{cor:gradient:limit:zero:abstract}
Let $\mathfrak{d} \in \mathbb{N}$, $\mathcal{L} \in C ( \mathbb{R}^\mathfrak{d} , \mathbb{R} )$, $p \in ( 0 , \infty )$, let $\mathcal{G} \colon \mathbb{R}^\mathfrak{d} \to \mathbb{R}^\mathfrak{d}$ be measurable, assume for all $\theta \in \mathbb{R}^\mathfrak{d}$ that $\mathcal{G} ( \theta ) \in ( \mathbb{D} \mathcal{L} ) (\theta)$, let $(\Omega , \mathcal{F} , \P)$ be a probability space, let $\Theta = (\Theta_n )_{n \in \mathbb{N}_0 } \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^\mathfrak{d}$ be a stochastic process, let $\vartheta \colon \Omega \to \mathbb{R}^\mathfrak{d}$ be a random variable, assume for all $n \in \mathbb{N}_0$ that $\mathbb{E}[\norm{\Theta_n}^p + \norm{\vartheta}^p + \norm{\mathcal{G}(\Theta_n)}^p] < \infty$, and assume
\begin{equation}\label{eqn:Frechet_convergence}
\textstyle \limsup_{n \to \infty} \mathbb{E} [ \norm{ \Theta_n - \vartheta } ^p + \norm{\mathcal{G} ( \Theta_n ) } ^p ] = 0
\end{equation}
\cfadd{def:limit:subdiff}\cfload. Then $\P(0 \in (\mathbb{D} \mathcal{L}) (\vartheta)) = 1$.
\end{cor}
\begin{cproof}{cor:gradient:limit:zero:abstract}
Throughout this proof let $\mathfrak{g} \colon \mathbb{R}^\mathfrak{d} \to [0 , \infty ]$ satisfy for all $\theta \in \mathbb{R}^\mathfrak{d}$ that
\begin{equation}
\textstyle \mathfrak{g} ( \theta ) = \inf \rbr{ \cu{ \norm{v } \colon v \in ( \mathbb{D} \mathcal{L} ) ( \theta ) } \cup \cu{\infty } }.
\end{equation}
(cf., e.g., (4) in Bolte et al.~\cite{BolteDaniilidis2006}). \Nobs that, e.g., \cite[Lemma 6.1]{JentzenRiekert2021ExistenceGlobMin} ensures that $\mathfrak{g}$ is lower semicontinuous. In addition, \nobs that the assumption that $\forall \, \theta \in \mathbb{R}^\mathfrak{d} \colon \mathcal{G} ( \theta ) \in ( \mathbb{D} \mathcal{L} ) ( \theta ) $ assures for all $\theta \in \mathbb{R}^\mathfrak{d}$ that $\mathfrak{g} ( \theta ) \le \norm{\mathcal{G} ( \theta ) }$. Hence, we obtain that 
\begin{equation}
\textstyle \limsup_{n \to \infty} \mathbb{E} [\norm{\Theta_n - \vartheta}^p + \abs{\mathfrak{g} (\Theta_n)}^p] = 0.
\end{equation}
Combining this with \cref{lem:gradient:limit:zero:abstract} demonstrates that $\P ( \mathfrak{g} ( \vartheta ) = 0 ) =1$. This and the fact that for all $\theta \in \mathbb{R}^\mathfrak{d}$ we have that $(\mathbb{D} \mathcal{L} ) (\theta ) \subseteq \mathbb{R}^\mathfrak{d}$
is closed (cf., e.g., Rockafellar~\cite[Theorem~8.6]{RockafellarWets1998} and Jentzen \& Riekert~\cite[Item~$($v$)$ in Lemma~3.3]{JentzenRiekert2021ExistenceGlobMin}) demonstrate that $\P ( 0 \in ( \mathbb{D} \mathcal{L} ) ( \vartheta ) ) = 1$.
\end{cproof}




\subsection{SGD training of all biases of clipping ANNs (3 layers)}
\label{subsec_num_sim_clipping_biases_1_hidden_layer}


In this subsection we present numerical simulation results (see \cref{fig:all_biases_clipping_1_hidden_layer} below) in a supervised learning framework for the training of shallow clipping ANNs with 3 layers (corresponding to $L = 2$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), an $\ell_1$-dimensional hidden layer, and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where $\ell_1 \in \{10, 100, 1000\}$ and where, as in the statement of \cref{theorem_intro_2} above, not all ANN parameters but only the bias parameters are modified during the training process. We also refer to \cref{figure_deep_3_layers} below for a graphical illustration of the ANN architectures considered in this subsection.


Assume \cref{setting:dnn}, assume $L = 2$, $\ell_0 = \ell_2 = 1$, $\ell_1 \in \{10, 100, 1000\}$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \min\{\max\{x, \allowbreak 0\}, 1\}$ and $f(x) = \abs{x}^{1/4}$, assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$, assume
\begin{equation}\label{eqn:index_all_biases_clipping}
\textstyle \fJ = (\mathbb{N} \cap (\ell_1, 2 \ell_1]) \cup \{3 \ell_1 + 1\},
\end{equation}
assume\footnote{\Nobs that the function $\floor{\cdot} \colon \mathbb{R} \to \mathbb{R}$ satisfies that for all $x \in \mathbb{R}$ it holds that $\floor{x} = \max((- \infty, x] \cap \mathbb{Z})$.} for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ that $\psi_n(g_1, \allowbreak g_2, \allowbreak \ldots, \allowbreak g_n) = 10^{-2} \, 2^{- \floor{\frac{n}{150}}} g_n$, assume for all $j \in \allowbreak \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \ell_1\}$, $x \in \mathbb{R}$ that
\begin{equation}\label{eqn:clipping_distribution}
\textstyle \P\big(\ell_1^{-3} \Theta_{0}^j \le x\big) = \P\big(\ell_1^{7/8} \Theta_{0}^{2 \ell_1 + j} \le x\big) = \int_0^x [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y,
\end{equation}
and assume for all $j \in \{1, 2, \ldots, \ell_1\}$ that $\ell_1^{-3} \Theta_0^{\ell_1 + j}$ and $\Theta_0^{3 \ell_1 + 1}$ are standard normal random variables.


\def2.5cm{4cm}
\begin{figure}[H]
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tikzpicture}[shorten >=1pt,->,draw=black!50, node distance=2.5cm]
\tikzstyle{every pin edge}=[<-,shorten <=1pt]
\tikzstyle{input neuron}=[very thick, circle,draw=red, fill=red!20, minimum size=15pt,inner sep=0pt]
\tikzstyle{output neuron}=[very thick, circle, draw=green,fill=green!20,minimum size=30pt,inner sep=0pt]
\tikzstyle{hidden neuron}=[very thick, circle,draw=blue,fill=blue!20,minimum size=20pt,inner sep=0pt]
\tikzstyle{annot} = [text width=9em, text centered]
\tikzstyle{annot2} = [text width=4em, text centered]

\node[input neuron] (I) at (0,-1.6) {$x$};

\path[yshift = 1.5cm]
node[hidden neuron](H0-1) at (2.5cm, -1 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H0-2) at (2.5cm, -2 cm) {};
\path[yshift = 1.5cm]
node(H0-dots) at (2.5cm, -2.9 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H0-999) at (2.5cm, -4 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H0-1000) at (2.5cm, -5 cm) {};


\path[yshift = 1.5cm]
node[output neuron](O) at (2*2.5cm,-3.1 cm) {$\mathcal{N}^{\theta}(x)$};   


\foreach \dest in {1,2,999,1000}
\path[line width = 0.8 ] (I) edge (H0-\dest);


\foreach \source in {1,2,999,1000}
\path[line width = 0.8 ] (H0-\source) edge (O);

\node[annot,above of=H0-1, node distance=1cm, align=center] (hl) {Hidden layer\\($2^{\text{nd}}$ layer)};
\node[annot,left of=hl, align=center] {Input layer\\ ($1^{\text{st}}$ layer)};
\node[annot,right of=hl, align=center] {Output layer\\($3^{\text{rd}}$ layer)};

\node[annot2,below of=H0-1000, node distance=1cm, align=center] (sl) {$\ell_1$ \\ neurons};
\node[annot2,left of=sl, align=center] {$\ell_0=1$ neurons};
\node[annot2,right of=sl, align=center] {$\ell_2=1$ neurons};
\end{tikzpicture}
\end{adjustbox}
\caption{Graphical illustration of the considered shallow ANN architectures used in Subsections~\ref{subsec_num_sim_clipping_biases_1_hidden_layer}, \ref{subsec_num_sim_ReLU_inner_bias_1_hidden_layer}, \ref{subsec_num_sim_ReLU_all_params_1_hidden_layer_Xavier}, and \ref{subsec_num_sim_ReLU_all_params_1_hidden_layer_He}.}
\label{figure_deep_3_layers}
\end{figure}


Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_all_biases_clipping} represents the index set labeling the bias parameters of the considered shallow ANNs. Within this subsection only those bias parameters of the ANNs but not the weight parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but a special case of the initialization distributions appearing in \cref{theorem_intro_2} above. In addition, \nobs that the above described setup (see \cref{eqn:index_all_biases_clipping,eqn:clipping_distribution} above) ensures that \cref{theorem_intro_2} is applicable with $\alpha = \nicefrac{7}{8}$ and $\beta = 3$. Moreover, \nobs that the assumption that for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ it holds that $\psi_n(g_1, g_2, \ldots, g_n) = 10^{-2} \, 2^{- \floor{\frac{n}{150}}} g_n$ assures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$ that
\begin{equation}
\textstyle \Theta_n = \Theta_{n - 1} - 10^{-2} \, 2^{- \floor{\frac{n}{150}}} \mathfrak{G}_n(\Theta_{n - 1})
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Standard_SGD}). Next \nobs that the assumption that $L = 2$, the assumption that $\ell_0 = \ell_2 = 1$, and the assumption that $\ell_1 \in \{10, \allowbreak 100, \allowbreak 1000\}$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered shallow ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\textstyle \mathfrak{d} = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) = 3 \ell_1 + 1 = 
\begin{cases}
31 & \colon \ell_1 = 10 \\
301 & \colon \ell_1 = 100 \\
3001 & \colon \ell_1 = 1000.
\end{cases}
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 2$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{2, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.


In the left image of Figure~\ref{fig:all_biases_clipping_1_hidden_layer} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 40 distinct subintervals (with 40 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:all_biases_clipping_1_hidden_layer} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the gradient against the number $n \in (\cup_{k = 0}^{498} \{20 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:all_biases_clipping_1_hidden_layer} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9981}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9981}$ against the number $n \in (\cup_{k = 0}^{498} \{20 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:all_biases_clipping_1_hidden_layer} and the lower-right image of Figure~\ref{fig:all_biases_clipping_1_hidden_layer} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:all_biases_clipping_1_hidden_layer} is presented in Subsection~\ref{subsec_code_clipping_biases_1_hidden_layer}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{Opt_SNN_SGD_10_100_1000_clipping_biases_train.pdf}}
\caption{Result of the numerical simulations described in  Subsection~\ref{subsec_num_sim_clipping_biases_1_hidden_layer} (see the {\sc Python} source code in \cref{listing:all_biases_clipping_1_hidden_layer}).}
\label{fig:all_biases_clipping_1_hidden_layer}
\end{figure}




\subsection{SGD training of inner biases of ReLU ANNs (3 layers)}
\label{subsec_num_sim_ReLU_inner_bias_1_hidden_layer}


In this subsection we present numerical simulation results (see \cref{fig:inner_biases_ReLU_1_hidden_layer} below) in a supervised learning framework for the training of shallow ReLU ANNs with 3 layers (corresponding to $L = 2$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), an $\ell_1$-dimensional hidden layer, and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where $\ell_1 \in \{5, 20, 100\}$ and where, as in the statement of \cref{theorem_intro} above, not all ANN parameters but only the inner bias parameters are modified during the training process. We also refer to \cref{figure_deep_3_layers} above for a graphical illustration of the ANN architectures considered in this subsection.

Assume \cref{setting:dnn}, assume $L = 2$, $\ell_0 = \ell_2 = 1$, $\ell_1 \in \{5, 20, 100\}$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \max\{x, \allowbreak 0\}$ and $f(x) = x^4 \sin(x)$, assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$, assume
\begin{equation}\label{eqn:index_inner_biases_ReLU}
\textstyle \fJ = \mathbb{N} \cap (\ell_1, 2 \ell_1],
\end{equation}
assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ that $\psi_n(g_1, \allowbreak g_2, \allowbreak \ldots, \allowbreak g_n) = 10^{-1} \, 2^{- \floor{\frac{n}{500}}} g_n$, assume for all $j \in \allowbreak \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathfrak{d} - 1\} \backslash \fJ$, $x \in \mathbb{R}$ that
\begin{equation}\label{eqn:ReLU_inner_bias_distribution}
\textstyle \P\big(\ell_1^{4/15} \Theta_{0}^j \le x\big) = \int_0^x [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y,
\end{equation}
and assume for all $j \in \fJ \cup \{\mathfrak{d}\}$ that $\ell_1^{9/10} \Theta_{0}^j$ is a standard normal random variable.


Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_inner_biases_ReLU} represents the index set labeling the inner bias parameters of the considered shallow ANNs. Within this subsection only those inner bias parameters of the ANNs but not the outer bias parameter and the weight parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but a special case of the initialization distributions appearing in \cref{theorem_intro} above. In addition, \nobs that the above described setup (see \cref{eqn:index_inner_biases_ReLU,eqn:ReLU_inner_bias_distribution} above) ensures that \cref{theorem_intro} is applicable with $\alpha = \nicefrac{9}{10}$ and $\beta = \nicefrac{4}{15}$. Moreover, \nobs that the assumption that for all $n \in \mathbb{N}$, $g_1, \allowbreak g_2, \allowbreak \ldots, \allowbreak g_n \allowbreak \in \mathbb{R}^{\mathfrak{d}}$ it holds that $\psi_n(g_1, g_2, \ldots, g_n) = 10^{-1} \, 2^{- \floor{\frac{n}{500}}} g_n$ assures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$ that
\begin{equation}
\textstyle \Theta_n = \Theta_{n - 1} - 10^{-1} \, 2^{- \floor{\frac{n}{500}}} \mathfrak{G}_n(\Theta_{n - 1})
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Standard_SGD}). Next \nobs that the assumption that $L = 2$, the assumption that $\ell_0 = \ell_2 = 1$, and the assumption that $\ell_1 \in \{5, 20, 100\}$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered shallow ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\textstyle \mathfrak{d} = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) = 3 \ell_1 + 1 = 
\begin{cases}
16 & \colon \ell_1 = 5 \\
61 & \colon \ell_1 = 20 \\
301 & \colon \ell_1 = 100.
\end{cases}
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 2$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{2, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.


In the left image of Figure~\ref{fig:inner_biases_ReLU_1_hidden_layer} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 200 distinct subintervals (with 200 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:inner_biases_ReLU_1_hidden_layer} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the gradient against the number $n \in (\cup_{k = 0}^{498} \{20 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:inner_biases_ReLU_1_hidden_layer} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9981}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9981}$ against the number $n \in (\cup_{k = 0}^{498} \{20 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:inner_biases_ReLU_1_hidden_layer} and the lower-right image of Figure~\ref{fig:inner_biases_ReLU_1_hidden_layer} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:inner_biases_ReLU_1_hidden_layer} is presented in \cref{subsec_code_ReLU_inner_bias_1_hidden_layer}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{Opt_SNN_SGD_5_20_100_ReLU_inner_bias.pdf}}
\caption{Result of the numerical simulations described in Subsection~\ref{subsec_num_sim_ReLU_inner_bias_1_hidden_layer} (see the {\sc Python} source code in \cref{listing:inner_biases_ReLU_1_hidden_layer}).}
\label{fig:inner_biases_ReLU_1_hidden_layer}
\end{figure}





\subsection{SGD training of all parameters of ReLU ANNs (3 layers)}
\label{subsec_num_sim_ReLU_all_params_1_hidden_layer_Xavier}


In this subsection we present numerical simulation results (see \cref{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like} below) in a supervised learning framework for the training of shallow ReLU ANNs with 3 layers (corresponding to $L = 2$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), an $\ell_1$-dimensional hidden layer, and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where $\ell_1 \in \{5, 20, 150\}$ and where all ANN parameters are modified during the training process. We also refer to \cref{figure_deep_3_layers} above for a graphical illustration of the ANN architectures considered in this subsection.

Assume \cref{setting:dnn}, assume $L = 2$, $\ell_0 = \ell_2 = 1$, $\ell_1 \in \{5, 20, 150\}$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \max\{x, \allowbreak 0\}$ and $f(x) = x^2 + 2x$, assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$, assume
\begin{equation}\label{eqn:index_all_params_ReLU}
\textstyle \fJ = \mathbb{N} \cap [1, \mathfrak{d}],
\end{equation}
assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ that $\psi_n(g_1, \allowbreak g_2, \allowbreak \ldots, \allowbreak g_n) = 10^{-2} g_n$, assume for all $j \in \fJ \backslash ((\ell_1, 2 \ell_1] \cup \{\mathfrak{d}\})$, $x \in \mathbb{R}$ that
\begin{equation}
\textstyle \P\big(\ell_1^{1/7} \Theta_{0}^j \le x\big) = \int_0^x [\frac{2}{\pi}]^{1/2} \exp(-\frac{y^2}{2}) \, \d y,
\end{equation}
and assume for all $j \in \mathbb{N} \cap ((\ell_1, 2 \ell_1] \cup \{\mathfrak{d}\})$ that $\ell_1^{1/2} \Theta_{0}^j$ is a standard normal random variable.

Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_all_params_ReLU} represents the index set labeling all parameters of the considered shallow ANNs. Within this subsection all of the parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but a special case of the initialization distributions appearing in \cref{theorem_intro} above. Moreover, \nobs that the assumption that for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ it holds that $\psi_n(g_1, g_2, \ldots, g_n) = 10^{-2} g_n$ ensures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$ that
\begin{equation}
\textstyle \Theta_n = \Theta_{n - 1} - 10^{-2} \mathfrak{G}_n(\Theta_{n - 1})
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Standard_SGD}). Next \nobs that the assumption that $L = 2$, the assumption that $\ell_0 = \ell_2 = 1$, and the assumption that $\ell_1 \in \{5, \allowbreak 20, \allowbreak 150\}$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered shallow ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\textstyle \mathfrak{d} = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) = 3 \ell_1 + 1 = 
\begin{cases}
16 & \colon \ell_1 = 5 \\
61 & \colon \ell_1 = 20 \\
451 & \colon \ell_1 = 150.
\end{cases}
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 2$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{2, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.

In the left image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 80 distinct subintervals (with 80 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the generalized gradient against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9991}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9991}$ against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like} and the lower-right image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like} is presented in \cref{subsec_code_ReLU_all_params_1_hidden_layer_Xavier}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{Opt_SNN_SGD_5_20_150_ReLU_all_param_train.pdf}}
\caption{Result of the numerical simulations described in  Subsection~\ref{subsec_num_sim_ReLU_all_params_1_hidden_layer_Xavier} (see the {\sc Python} source code in \cref{listing:all_parameters_ReLU_1_hidden_layer_Xavier_like}).} 
\label{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like}
\end{figure}





\subsection{SGD training of all parameters of He initialized ReLU ANNs (3 layers)}
\label{subsec_num_sim_ReLU_all_params_1_hidden_layer_He}



In this subsection we present numerical simulation results (see \cref{fig:all_parameters_ReLU_1_hidden_layer_He} below) in a supervised learning framework for the training of shallow ReLU ANNs with 3 layers (corresponding to $L = 2$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), a 32-dimensional hidden layer (corresponding to $\ell_1 = 32$ in \cref{setting:dnn}), and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where all ANN parameters are modified during the training process. We also refer to \cref{figure_deep_3_layers} above for a graphical illustration of the ANN architecture considered in this subsection.


Assume \cref{setting:dnn}, assume $L = 2$, $\ell_0 = \ell_2 = 1$, $\ell_1 = 32$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \max\{x, \allowbreak 0\}$ and $f(x) = x^2 + 2x$, assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$, assume
\begin{equation}\label{eqn:index_all_params_ReLU_He}
\textstyle \fJ = \mathbb{N} \cap [1, \mathfrak{d}],
\end{equation}
assume for all $k \in \{1, 2, \ldots, L\}$, $i \in \{1, 2, \ldots, \ell_k\}$, $j \in \{1, 2, \ldots, \ell_{k - 1}\}$, $x \in \mathbb{R}$ that
\begin{equation}
\textstyle \P\big(\mathfrak{b}_i^{k, \Theta_0} = 0 \big) = 1 \qquad \text{and} \qquad \P\big(\mathfrak{w}_{i, j}^{k, \Theta_0} \ge x \big) = \int_x^{\infty} \frac{1}{2} \bigl[\frac{\ell_{k - 1}}{\pi}\bigr]^{1/2} \exp\bigl(- \frac{y^2 \ell_{k - 1}}{4}\bigr) \, \d y,
\end{equation}
and assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ that $\psi_n(g_1, \allowbreak g_2, \allowbreak \ldots, \allowbreak g_n) = 10^{-2} g_n$.


Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_all_params_ReLU_He} represents the index set labeling all parameters of the considered shallow ANNs. Within this subsection all of the parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but the He normal initialization (cf., e.g., \cite{tensorflow}) for ReLU ANNs with 3 layers with a 1-dimensional input layer, a 32-dimensional hidden layer, and a 1-dimensional output layer. Moreover, \nobs that the assumption that for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ it holds that $\psi_n(g_1, g_2, \ldots, g_n) = 10^{-2} g_n$ ensures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$ that
\begin{equation}
\textstyle \Theta_n = \Theta_{n - 1} - 10^{-2} \mathfrak{G}_n(\Theta_{n - 1})
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Standard_SGD}). Next \nobs that the assumption that $L = 2$, the assumption that $\ell_0 = \ell_2 = 1$, and the assumption that $\ell_1 = 32$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered shallow ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\textstyle \mathfrak{d} = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) = 3 \ell_1 + 1 = 97.
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 2$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{2, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.


In the left image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_He} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 80 distinct subintervals (with 80 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_He} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the generalized gradient against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_He} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9991}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9991}$ against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_He} and the lower-right image of Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_He} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_He} is presented in \cref{subsec_code_ReLU_all_params_1_hidden_layer_He}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{ReLU_SNN_SGD_1_32_1_He.pdf}}
\caption{Result of the numerical simulations described in  Subsection~\ref{subsec_num_sim_ReLU_all_params_1_hidden_layer_He} (see the {\sc Python} source code in \cref{listing:all_parameters_ReLU_1_hidden_layer_He}).} 
\label{fig:all_parameters_ReLU_1_hidden_layer_He}
\end{figure}





\subsection{SGD training of all parameters of Xavier initialized ReLU ANNs (4 layers)}
\label{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier}


In this subsection we present numerical simulation results (see \cref{fig:all_parameters_ReLU_2_hidden_layers_Xavier} below) in a supervised learning framework for the training of deep ReLU ANNs with 4 layers (corresponding to $L = 3$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), two 50-dimensional hidden layers (corresponding to $\ell_1 = \ell_2 = 50$ in \cref{setting:dnn}), and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where all ANN parameters are modified during the training process. We also refer to \cref{figure_deep_4_layers} below for a graphical illustration of the ANN architecture considered in this subsection.


Assume \cref{setting:dnn}, assume $L = 3$, $\ell_0 = \ell_3 = 1$, $\ell_1 = \ell_2 = 50$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \max\{x, \allowbreak 0\}$ and $f(x) = x^2 + 2x$, assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$, assume
\begin{equation}\label{eqn:index_all_params_ReLU_Xavier_4_Layers}
\textstyle \fJ = \mathbb{N} \cap [1, \mathfrak{d}],
\end{equation}
assume for all $k \in \{1, 2, \ldots, L\}$, $i \in \{1, 2, \ldots, \ell_k\}$, $j \in \{1, 2, \ldots, \ell_{k - 1}\}$, $x \in \mathbb{R}$ that
\begin{equation}
\textstyle \P\big(\mathfrak{b}_i^{k, \Theta_0} = 0 \big) = 1 \qquad \text{and} \qquad \P\big(\mathfrak{w}_{i, j}^{k, \Theta_0} \ge x \big) = \int_x^{\infty} \frac{1}{2} \bigl[\frac{\ell_{k - 1} + \ell_k}{\pi}\bigr]^{1/2} \exp\bigl(- \frac{y^2 (\ell_{k - 1} + \ell_k)}{4}\bigr) \, \d y,
\end{equation}
and assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ that $\psi_n(g_1, \allowbreak g_2, \allowbreak \ldots, \allowbreak g_n) = 10^{-2} g_n$.

\def2.5cm{2.5cm}
\begin{figure}[H]
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tikzpicture}[shorten >=1pt,->,draw=black!50, node distance=2.5cm]
\tikzstyle{every pin edge}=[<-,shorten <=1pt]
\tikzstyle{input neuron}=[very thick, circle,draw=red, fill=red!20, minimum size=15pt,inner sep=0pt]
\tikzstyle{output neuron}=[very thick, circle, draw=green,fill=green!20,minimum size=20pt,inner sep=0pt]
\tikzstyle{hidden neuron}=[very thick, circle,draw=blue,fill=blue!20,minimum size=20pt,inner sep=0pt]
\tikzstyle{annot} = [text width=9em, text centered]
\tikzstyle{annot2} = [text width=4em, text centered]

\node[input neuron] (I) at (0,-1.5) {$x$};

\path[yshift = 1.5cm]
node[hidden neuron](H0-1) at (2.5cm, -1 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H0-2) at (2.5cm, -2 cm) {};
\path[yshift = 1.5cm]
node(H0-dots) at (2.5cm, -2.9 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H0-49) at (2.5cm, -4 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H0-50) at (2.5cm, -5 cm) {};

\path[yshift = 1.5cm]
node[hidden neuron](H1-1) at (2*2.5cm, -1 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H1-2) at (2*2.5cm, -2 cm) {};
\path[yshift = 1.5cm]
node(H1-dots) at (2*2.5cm, -2.9 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H1-49) at (2*2.5cm, -4 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H1-50) at (2*2.5cm, -5 cm) {};


\path[yshift = 1.5cm]
node[output neuron](O) at (3*2.5cm,-3 cm) {$\mathcal{N}^{\theta}(x)$};   


\foreach \dest in {1,2,49,50}
\path[line width = 0.8] (I) edge (H0-\dest);


\foreach \source in {1,2,49,50}
\foreach \dest in {1,2,49,50}
\path[line width = 0.8] (H0-\source) edge (H1-\dest);


\foreach \source in {1,2,49,50}
\path[line width = 0.8] (H1-\source) edge (O);


\node[annot,above of=H0-1, node distance=1cm, align=center] (hl) {$1^{\text{st}}$ hidden layer\\($2^{\text{nd}}$ layer)};
\node[annot,above of=H1-1, node distance=1cm, align=center] (hl2) {$2^{\text{nd}}$ hidden layer\\($3^{\text{rd}}$ layer)};
\node[annot,left of=hl, align=center] {Input layer\\ ($1^{\text{st}}$ layer)};
\node[annot,right of=hl2, align=center] {Output layer\\($4^{\text{th}}$ layer)};

\node[annot2,below of=H0-50, node distance=1cm, align=center] (sl) {$\ell_1=50$ \\ neurons};
\node[annot2,below of=H1-50, node distance=1cm, align=center] (sl2) {$\ell_2=50$ \\ neurons};
\node[annot2,left of=sl, align=center] {$\ell_0=1$ \\ neuron};
\node[annot2,right of=sl2, align=center] {$\ell_3=1$ \\ neuron};
\end{tikzpicture}
\end{adjustbox}
\caption{Graphical illustration of the ANN architectures used in Subsections~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier}, \ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_He}, \ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier_Adam}, and \ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_He_Adam}.}
\label{figure_deep_4_layers}
\end{figure}


Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_all_params_ReLU_Xavier_4_Layers} represents the index set labeling all parameters of the considered deep ANNs. Within this subsection all of the parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but the Xavier normal (Glorot normal) initialization (cf., e.g., \cite{tensorflow}) for ReLU ANNs with 4 layers with a 1-dimensional input layer, two 50-dimensional hidden layers, and a 1-dimensional output layer. Moreover, \nobs that the assumption that for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ it holds that $\psi_n(g_1, g_2, \ldots, g_n) = 10^{-2} g_n$ ensures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$ that
\begin{equation}
\textstyle \Theta_n = \Theta_{n - 1} - 10^{-2} \mathfrak{G}_n(\Theta_{n - 1})
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Standard_SGD}). Next \nobs that the assumption that $L = 3$, the assumption that $\ell_0 = \ell_3 = 1$, and the assumption that $\ell_1 = \ell_2 = 50$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered deep ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\textstyle \mathfrak{d} = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) + \ell_3 (\ell_2 + 1) = 50 (1 + 1) + 50 (50 + 1) + 1 (50 + 1) = 2701.
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 3$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{3, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.


In the left image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 80 distinct subintervals (with 80 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the generalized gradient against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9991}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9991}$ against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier} and the lower-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier} is presented in \cref{subsec_code_ReLU_all_params_2_hidden_layers_Xavier}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{ReLU_DNN_SGD_1_50_50_1_Xavier.pdf}} 
\caption{Result of the numerical simulations described in Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier} (see the {\sc Python} source code in \cref{listing:all_parameters_ReLU_2_hidden_layers_Xavier}).}
\label{fig:all_parameters_ReLU_2_hidden_layers_Xavier}
\end{figure}




\subsection{SGD training of all parameters of He initialized ReLU ANNs (4 layers)}
\label{subsec_num_sim_ReLU_all_params_2_hidden_layers_He}


In this subsection we present numerical simulation results (see \cref{fig:all_parameters_ReLU_2_hidden_layers_He} below) in a supervised learning framework for the training of deep ReLU ANNs with 4 layers (corresponding to $L = 3$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), two 50-dimensional hidden layers (corresponding to $\ell_1 = \ell_2 = 50$ in \cref{setting:dnn}), and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where all ANN parameters are modified during the training process. We also refer to \cref{figure_deep_4_layers} above for a graphical illustration of the ANN architecture considered in this subsection.


Assume \cref{setting:dnn}, assume $L = 3$, $\ell_0 = \ell_3 = 1$, $\ell_1 = \ell_2 = 50$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \max\{x, \allowbreak 0\}$ and $f(x) = x^2 + 2x$, assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$, assume
\begin{equation}\label{eqn:index_all_params_ReLU_He_4_Layers}
\textstyle \fJ = \mathbb{N} \cap [1, \mathfrak{d}],
\end{equation}
assume for all $k \in \{1, 2, \ldots, L\}$, $i \in \{1, 2, \ldots, \ell_k\}$, $j \in \{1, 2, \ldots, \ell_{k - 1}\}$, $x \in \mathbb{R}$ that
\begin{equation}
\textstyle \P\big(\mathfrak{b}_i^{k, \Theta_0} = 0 \big) = 1 \qquad \text{and} \qquad \P\big(\mathfrak{w}_{i, j}^{k, \Theta_0} \ge x \big) = \int_x^{\infty} \frac{1}{2} \bigl[\frac{\ell_{k - 1}}{\pi}\bigr]^{1/2} \exp\bigl(- \frac{y^2 \ell_{k - 1}}{4}\bigr) \, \d y,
\end{equation}
and assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ that $\psi_n(g_1, \allowbreak g_2, \allowbreak \ldots, \allowbreak g_n) = 10^{-2} g_n$.


Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_all_params_ReLU_He_4_Layers} represents the index set labeling all parameters of the considered deep ANNs. Within this subsection all of the parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but the He normal initialization (cf., e.g., \cite{tensorflow}) for ReLU ANNs with 4 layers with a 1-dimensional input layer, two 50-dimensional hidden layers, and a 1-dimensional output layer. Moreover, \nobs that the assumption that for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ it holds that $\psi_n(g_1, g_2, \ldots, g_n) = 10^{-2} g_n$ ensures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$ that
\begin{equation}
\textstyle \Theta_n = \Theta_{n - 1} - 10^{-2} \mathfrak{G}_n(\Theta_{n - 1})
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Standard_SGD}). Next \nobs that the assumption that $L = 3$, the assumption that $\ell_0 = \ell_3 = 1$, and the assumption that $\ell_1 = \ell_2 = 50$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered deep ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\textstyle \mathfrak{d} = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) + \ell_3 (\ell_2 + 1) = 50 (1 + 1) + 50 (50 + 1) + 1 (50 + 1) = 2701.
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 3$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{3, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.


In the left image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 80 distinct subintervals (with 80 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the generalized gradient against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9991}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9991}$ against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He} and the lower-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He} is presented in \cref{subsec_code_ReLU_all_params_2_hidden_layers_He}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{ReLU_DNN_SGD_1_50_50_1_He.pdf}} 
\caption{Result of the numerical simulations described in  Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_He} (see the {\sc Python} source code in \cref{listing:all_parameters_ReLU_2_hidden_layers_He}).}
\label{fig:all_parameters_ReLU_2_hidden_layers_He}
\end{figure}




\subsection{Adam training of all parameters of Xavier initialized ReLU ANNs (4 layers)}
\label{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier_Adam}


In this subsection we present numerical simulation results (see \cref{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam} below) in a supervised learning framework for the training of deep ReLU ANNs with 4 layers (corresponding to $L = 3$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), two 50-dimensional hidden layers (corresponding to $\ell_1 = \ell_2 = 50$ in \cref{setting:dnn}), and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where all ANN parameters are modified during the training process. We also refer to \cref{figure_deep_4_layers} above for a graphical illustration of the ANN architecture considered in this subsection.


Assume \cref{setting:dnn}, assume $L = 3$, $\ell_0 = \ell_3 = 1$, $\ell_1 = \ell_2 = 50$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \max\{x, \allowbreak 0\}$ and $f(x) = x^2 + 2x$, assume
\begin{equation}\label{eqn:index_all_params_ReLU_Xavier_Adam_4_Layers}
\textstyle \fJ = \mathbb{N} \cap [1, \mathfrak{d}],
\end{equation}
assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$, $i \in \{1, 2, \ldots, \mathfrak{d}\}$ that 
\begin{equation}\label{eqn:Adam_l_rate}
\textstyle \psi_n^{i} (g_1, g_2, \ldots, g_n) \textstyle = 10^{-2} \, 2^{- \floor{\frac{n}{500}}} \left[\frac{\sum_{k = 1}^n 0.9^{n-k} 10^{-1} g_k^{i}}{1 - 0.9^n} \right] \left[10^{-8} + \Bigl[\frac{\sum_{k = 1}^n 0.999^{n-k} 10^{-3} \abs{g_k^{i}}^2}{1 - 0.999^n}\Bigr]^{\nicefrac{1}{2}} \right]^{-1},
\end{equation}
assume for all $k \in \{1, 2, \ldots, L\}$, $i \in \{1, 2, \ldots, \ell_k\}$, $j \in \{1, 2, \ldots, \ell_{k - 1}\}$, $x \in \mathbb{R}$ that
\begin{equation}\label{eqn:Adam_Theta_0}
\textstyle \P\big(\mathfrak{b}_i^{k, \Theta_0} = 0 \big) = 1 \qquad \text{and} \qquad \P\big(\mathfrak{w}_{i, j}^{k, \Theta_0} \ge x \big) = \int_x^{\infty} \frac{1}{2} \bigl[\frac{\ell_{k - 1} + \ell_k}{\pi}\bigr]^{1/2} \exp\bigl(- \frac{y^2 (\ell_{k - 1} + \ell_k)}{4}\bigr) \, \d y,
\end{equation}
and assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$.

Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_all_params_ReLU_Xavier_Adam_4_Layers} represents the index set labeling all parameters of the considered deep ANNs. Within this subsection all of the parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but the Xavier normal (Glorot normal) initialization (cf., e.g., \cite{tensorflow}) for ReLU ANNs with 4 layers with a 1-dimensional input layer, two 50-dimensional hidden layers, and a 1-dimensional output layer. Moreover, \nobs that \cref{eqn:Adam_l_rate} ensures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{d}\}$ that
\begin{equation}
\textstyle \Theta_n^i = \Theta_{n - 1}^i - 10^{-2} \, 2^{- \floor{\frac{n}{500}}} \left[\frac{\sum_{k = 1}^n 0.9^{n-k} 10^{-1} \mathfrak{G}_k^i(\Theta_{k - 1})}{1 - 0.9^n} \right] \left[10^{-8} + \Bigl[\frac{\sum_{k = 1}^n 0.999^{n-k} 10^{-3} \abs{\mathfrak{G}_k^i(\Theta_{k - 1})}^2}{1 - 0.999^n}\Bigr]^{\nicefrac{1}{2}} \right]^{-1}
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Adam_SGD}). Next \nobs that the assumption that $L = 3$, the assumption that $\ell_0 = \ell_3 = 1$, and the assumption that $\ell_1 = \ell_2 = 50$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered deep ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\textstyle \mathfrak{d} = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) + \ell_3 (\ell_2 + 1) = 50 (1 + 1) + 50 (50 + 1) + 1 (50 + 1) = 2701.
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 3$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{3, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.


In the left image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 80 distinct subintervals (with 80 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the generalized gradient against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9991}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9991}$ against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam} and the lower-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam} is presented in \cref{subsec_code_ReLU_all_params_2_hidden_layers_Xavier_Adam}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{ReLU_DNN_Adam_1_50_50_1_Xavier.pdf}} 
\caption{Result of the numerical simulations described in Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier_Adam} (see the {\sc Python} source code in \cref{listing:all_parameters_ReLU_2_hidden_layers_Xavier_Adam}).}
\label{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam}
\end{figure}




\subsection{Adam training of all parameters of He initialized ReLU ANNs (4 layers)}
\label{subsec_num_sim_ReLU_all_params_2_hidden_layers_He_Adam}


In this subsection we present numerical simulation results (see \cref{fig:all_parameters_ReLU_2_hidden_layers_He_Adam} below) in a supervised learning framework for the training of deep ReLU ANNs with 4 layers (corresponding to $L = 3$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), two 50-dimensional hidden layers (corresponding to $\ell_1 = \ell_2 = 50$ in \cref{setting:dnn}), and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where all ANN parameters are modified during the training process. We also refer to \cref{figure_deep_4_layers} above for a graphical illustration of the ANN architecture considered in this subsection.


Assume \cref{setting:dnn}, assume $L = 3$, $\ell_0 = \ell_3 = 1$, $\ell_1 = \ell_2 = 50$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \max\{x, \allowbreak 0\}$ and $f(x) = x^2 + 2x$, assume
\begin{equation}\label{eqn:index_all_params_ReLU_He_Adam_4_Layers}
\textstyle \fJ = \mathbb{N} \cap [1, \mathfrak{d}],
\end{equation}
assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$, $i \in \{1, 2, \ldots, \mathfrak{d}\}$ that 
\begin{equation}\label{eqn:Adam_l_rate_He}
\textstyle \psi_n^{i} (g_1, g_2, \ldots, g_n) \textstyle = 10^{-2} \, 2^{- \floor{\frac{n}{500}}} \left[\frac{\sum_{k = 1}^n 0.9^{n-k} 10^{-1} g_k^{i}}{1 - 0.9^n} \right] \left[10^{-8} + \Bigl[\frac{\sum_{k = 1}^n 0.999^{n-k} 10^{-3} \abs{g_k^{i}}^2}{1 - 0.999^n}\Bigr]^{\nicefrac{1}{2}} \right]^{-1},
\end{equation}
assume for all $k \in \{1, 2, \ldots, L\}$, $i \in \{1, 2, \ldots, \ell_k\}$, $j \in \{1, 2, \ldots, \ell_{k - 1}\}$, $x \in \mathbb{R}$ that
\begin{equation}\label{eqn:Adam_Theta_0_He}
\textstyle \P\big(\mathfrak{b}_i^{k, \Theta_0} = 0 \big) = 1 \qquad \text{and} \qquad \P\big(\mathfrak{w}_{i, j}^{k, \Theta_0} \ge x \big) = \int_x^{\infty} \frac{1}{2} \bigl[\frac{\ell_{k - 1}}{\pi}\bigr]^{1/2} \exp\bigl(- \frac{y^2 \ell_{k - 1}}{4}\bigr) \, \d y,
\end{equation}
and assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$.

Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_all_params_ReLU_He_Adam_4_Layers} represents the index set labeling all parameters of the considered deep ANNs. Within this subsection all of the parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but the He normal initialization (cf., e.g., \cite{tensorflow}) for ReLU ANNs with 4 layers with a 1-dimensional input layer, two 50-dimensional hidden layers, and a 1-dimensional output layer. Moreover, \nobs that \cref{eqn:Adam_l_rate_He} ensures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$, $i \in \{1, 2, \ldots, \mathfrak{d}\}$ that
\begin{equation}
\textstyle \Theta_n^i = \Theta_{n - 1}^i - 10^{-2} \, 2^{- \floor{\frac{n}{500}}} \left[\frac{\sum_{k = 1}^n 0.9^{n-k} 10^{-1} \mathfrak{G}_k^i(\Theta_{k - 1})}{1 - 0.9^n} \right] \left[10^{-8} + \Bigl[\frac{\sum_{k = 1}^n 0.999^{n-k} 10^{-3} \abs{\mathfrak{G}_k^i(\Theta_{k - 1})}^2}{1 - 0.999^n}\Bigr]^{\nicefrac{1}{2}} \right]^{-1}
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Adam_SGD}). Next \nobs that the assumption that $L = 3$, the assumption that $\ell_0 = \ell_3 = 1$, and the assumption that $\ell_1 = \ell_2 = 50$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered deep ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\textstyle \mathfrak{d} = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) + \ell_3 (\ell_2 + 1) = 50 (1 + 1) + 50 (50 + 1) + 1 (50 + 1) = 2701.
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 3$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{3, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.


In the left image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He_Adam} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 80 distinct subintervals (with 80 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He_Adam} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the generalized gradient against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He_Adam} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9991}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9991}$ against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He_Adam} and the lower-right image of Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He_Adam} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He_Adam} is presented in \cref{subsec_code_ReLU_all_params_2_hidden_layers_He_Adam}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{ReLU_DNN_Adam_1_50_50_1_He.pdf}} 
\caption{Result of the numerical simulations described in Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_He_Adam} (see the {\sc Python} source code in \cref{listing:all_parameters_ReLU_2_hidden_layers_He_Adam}).}
\label{fig:all_parameters_ReLU_2_hidden_layers_He_Adam}
\end{figure}




\subsection{SGD training of all parameters of Xavier initialized ReLU ANNs (5 layers)}
\label{subsec_num_sim_ReLU_all_params_3_hidden_layers_Xavier}


In this subsection we present numerical simulation results (see \cref{fig:all_parameters_ReLU_3_hidden_layers_Xavier} below) in a supervised learning framework for the training of deep ReLU ANNs with 5 layers (corresponding to $L = 4$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), three 32-dimensional hidden layers (corresponding to $\ell_1 = \ell_2 = \ell_3 = 32$ in \cref{setting:dnn}), and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where all ANN parameters are modified during the training process. We also refer to \cref{figure_deep_5_layers} below for a graphical illustration of the ANN architecture considered in this subsection.


Assume \cref{setting:dnn}, assume $L = 4$, $\ell_0 = \ell_4 = 1$, $\ell_1 = \ell_2 = \ell_3 = 32$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \max\{x, \allowbreak 0\}$ and $f(x) = x^2 + 2x$, assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$, assume
\begin{equation}\label{eqn:index_all_params_ReLU_Xavier_5_Layers}
\textstyle \fJ = \mathbb{N} \cap [1, \mathfrak{d}],
\end{equation}
assume for all $k \in \{1, 2, \ldots, L\}$, $i \in \{1, 2, \ldots, \ell_k\}$, $j \in \{1, 2, \ldots, \ell_{k - 1}\}$, $x \in \mathbb{R}$ that
\begin{equation}
\textstyle \P\big(\mathfrak{b}_i^{k, \Theta_0} = 0 \big) = 1 \qquad \text{and} \qquad \P\big(\mathfrak{w}_{i, j}^{k, \Theta_0} \ge x \big) = \int_x^{\infty} \frac{1}{2} \bigl[\frac{\ell_{k - 1} + \ell_k}{\pi}\bigr]^{1/2} \exp\bigl(- \frac{y^2 (\ell_{k - 1} + \ell_k)}{4}\bigr) \, \d y,
\end{equation}
and assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ that $\psi_n(g_1, \allowbreak g_2, \allowbreak \ldots, \allowbreak g_n) = 10^{-2} g_n$.


\def2.5cm{2.5cm}
\begin{figure}[H]
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tikzpicture}[shorten >=1pt,->,draw=black!50, node distance=2.5cm]
\tikzstyle{every pin edge}=[<-,shorten <=1pt]
\tikzstyle{input neuron}=[very thick, circle,draw=red, fill=red!20, minimum size=15pt,inner sep=0pt]
\tikzstyle{output neuron}=[very thick, circle, draw=green,fill=green!20,minimum size=20pt,inner sep=0pt]
\tikzstyle{hidden neuron}=[very thick, circle,draw=blue,fill=blue!20,minimum size=20pt,inner sep=0pt]
\tikzstyle{annot} = [text width=9em, text centered]
\tikzstyle{annot2} = [text width=4em, text centered]

\node[input neuron] (I) at (0,-1.5) {$x$};

\path[yshift = 1.5cm]
node[hidden neuron](H0-1) at (2.5cm, -1 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H0-2) at (2.5cm, -2 cm) {};
\path[yshift = 1.5cm]
node(H0-dots) at (2.5cm, -2.9 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H0-31) at (2.5cm, -4 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H0-32) at (2.5cm, -5 cm) {};

\path[yshift = 1.5cm]
node[hidden neuron](H1-1) at (2*2.5cm, -1 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H1-2) at (2*2.5cm, -2 cm) {};
\path[yshift = 1.5cm]
node(H1-dots) at (2*2.5cm, -2.9 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H1-31) at (2*2.5cm, -4 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H1-32) at (2*2.5cm, -5 cm) {};

\path[yshift = 1.5cm]
node[hidden neuron](H2-1) at (3*2.5cm, -1 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H2-2) at (3*2.5cm, -2 cm) {};
\path[yshift = 1.5cm]
node(H1-dots) at (3*2.5cm, -2.9 cm) {\vdots};
\path[yshift = 1.5cm]
node[hidden neuron](H2-31) at (3*2.5cm, -4 cm) {};
\path[yshift = 1.5cm]
node[hidden neuron](H2-32) at (3*2.5cm, -5 cm) {};

\path[yshift = 1.5cm]
node[output neuron](O) at (4*2.5cm,-3 cm) {$\mathcal{N}^{\theta}(x)$};   


\foreach \dest in {1,2,31,32}
\path[line width = 0.8] (I) edge (H0-\dest);


\foreach \source in {1,2,31,32}
\foreach \dest in {1,2,31,32}
\path[line width = 0.8] (H0-\source) edge (H1-\dest);


\foreach \source in {1,2,31,32}
\foreach \dest in {1,2,31,32}
\path[line width = 0.8] (H1-\source) edge (H2-\dest);


\foreach \source in {1,2,31,32}
\path[line width = 0.8] (H2-\source) edge (O);


\node[annot,above of=H0-1, node distance=1cm, align=center] (hl) {$1^{\text{st}}$ hidden layer\\($2^{\text{nd}}$ layer)};
\node[annot,above of=H1-1, node distance=1cm, align=center] (hl2) {$2^{\text{nd}}$ hidden layer\\($3^{\text{rd}}$ layer)};
\node[annot,above of=H2-1, node distance=1cm, align=center] (hl3) {$3^{\text{rd}}$ hidden layer\\($4^{\text{th}}$ layer)};
\node[annot,left of=hl, align=center] {Input layer\\ ($1^{\text{st}}$ layer)};
\node[annot,right of=hl3, align=center] {Output layer\\($5^{\text{th}}$ layer)};

\node[annot2,below of=H0-32, node distance=1cm, align=center] (sl) {$\ell_1=32$ \\ neurons};
\node[annot2,below of=H1-32, node distance=1cm, align=center] (sl2) {$\ell_2=32$ \\ neurons};
\node[annot2,below of=H2-32, node distance=1cm, align=center] (sl3) {$\ell_3=32$ \\ neurons};
\node[annot2,left of=sl, align=center] {$\ell_0=1$ \\ neuron};
\node[annot2,right of=sl3, align=center] {$\ell_4=1$ \\ neuron};
\end{tikzpicture}
\end{adjustbox}
\caption{Graphical illustration of the ANN architectures used in Subsections~\ref{subsec_num_sim_ReLU_all_params_3_hidden_layers_Xavier} and \ref{subsec_num_sim_ReLU_all_params_3_hidden_layers_He}.}
\label{figure_deep_5_layers}
\end{figure}


Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_all_params_ReLU_Xavier_5_Layers} represents the index set labeling all parameters of the considered deep ANNs. Within this subsection all of the parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but the Xavier normal (Glorot normal) initialization (cf., e.g., \cite{tensorflow}) for ReLU ANNs with 5 layers with a 1-dimensional input layer, three 32-dimensional hidden layers, and a 1-dimensional output layer. Moreover, \nobs that the assumption that for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ it holds that $\psi_n(g_1, g_2, \ldots, g_n) = 10^{-2} g_n$ ensures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$ that
\begin{equation}
\textstyle \Theta_n = \Theta_{n - 1} - 10^{-2} \mathfrak{G}_n(\Theta_{n - 1})
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Standard_SGD}). Next \nobs that the assumption that $L = 4$, the assumption that $\ell_0 = \allowbreak \ell_4 \allowbreak = 1$, and the assumption that $\ell_1 = \ell_2 = \ell_3 = 32$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered deep ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\begin{split}
\textstyle \mathfrak{d} & \textstyle = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) + \ell_3 (\ell_2 + 1) + \ell_4 (\ell_3 + 1) \\
& \textstyle  = 32 (1 + 1) + 32 (32 + 1) + 32 (32 + 1) + 1 (32 + 1) = 2209.
\end{split}
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 4$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{4, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.


In the left image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_Xavier} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 80 distinct subintervals (with 80 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_Xavier} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the generalized gradient against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_Xavier} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9991}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9991}$ against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_Xavier} and the lower-right image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_Xavier} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_Xavier} is presented in \cref{subsec_code_ReLU_all_params_3_hidden_layers_Xavier}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{ReLU_DNN_SGD_1_32_32_32_1_Xavier.pdf}} 
\caption{Result of the numerical simulations described in Subsection~\ref{subsec_num_sim_ReLU_all_params_3_hidden_layers_Xavier} (see the {\sc Python} source code in \cref{listing:all_parameters_ReLU_3_hidden_layers_Xavier}).}
\label{fig:all_parameters_ReLU_3_hidden_layers_Xavier}
\end{figure}




\subsection{SGD training of all parameters of He initialized ReLU ANNs (5 layers)}
\label{subsec_num_sim_ReLU_all_params_3_hidden_layers_He}


In this subsection we present numerical simulation results (see \cref{fig:all_parameters_ReLU_3_hidden_layers_He} below) in a supervised learning framework for the training of deep ReLU ANNs with 5 layers (corresponding to $L = 4$ in \cref{setting:dnn}) with a 1-dimensional input layer (corresponding to $\ell_0 = 1$ in \cref{setting:dnn}), three 32-dimensional hidden layers (corresponding to $\ell_1 = \ell_2 = \ell_3 = 32$ in \cref{setting:dnn}), and a 1-dimensional output layer (corresponding to $\ell_L = 1$ in \cref{setting:dnn}) where all ANN parameters are modified during the training process. We also refer to \cref{figure_deep_5_layers} above for a graphical illustration of the ANN architecture considered in this subsection.


Assume \cref{setting:dnn}, assume $L = 4$, $\ell_0 = \ell_4 = 1$, $\ell_1 = \ell_2 = \ell_3 = 32$, and $\mathbf{m} = 1024$, assume for all $x \in \mathbb{R}$ that $\mathbb{A}_{\infty} (x) = \allowbreak \max\{x, \allowbreak 0\}$ and $f(x) = x^2 + 2x$, assume for all $k \in \{1, \allowbreak 2, \allowbreak \ldots, \allowbreak \mathbf{m}\}$, $n \in \mathbb{N}$, $x \in [0, 1]$ that $\P(X_k^n < x) = x$, assume
\begin{equation}\label{eqn:index_all_params_ReLU_He_5_Layers}
\textstyle \fJ = \mathbb{N} \cap [1, \mathfrak{d}],
\end{equation}
assume for all $k \in \{1, 2, \ldots, L\}$, $i \in \{1, 2, \ldots, \ell_k\}$, $j \in \{1, 2, \ldots, \ell_{k - 1}\}$, $x \in \mathbb{R}$ that
\begin{equation}
\textstyle \P\big(\mathfrak{b}_i^{k, \Theta_0} = 0 \big) = 1 \qquad \text{and} \qquad \P\big(\mathfrak{w}_{i, j}^{k, \Theta_0} \ge x \big) = \int_x^{\infty} \frac{1}{2} \bigl[\frac{\ell_{k - 1}}{\pi}\bigr]^{1/2} \exp\bigl(- \frac{y^2 \ell_{k - 1}}{4}\bigr) \, \d y,
\end{equation}
and assume for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ that $\psi_n(g_1, \allowbreak g_2, \allowbreak \ldots, \allowbreak g_n) = 10^{-2} g_n$.


Let us add a few comments regarding the above presented setup. \Nobs that the set $\fJ$ in \cref{eqn:index_all_params_ReLU_He_5_Layers} represents the index set labeling all parameters of the considered deep ANNs. Within this subsection all of the parameters of the ANNs are modified during the training procedure. Furthermore, \nobs that the distribution of $\Theta_0$ is nothing else but the He normal initialization (cf., e.g., \cite{tensorflow}) for ReLU ANNs with 5 layers with a 1-dimensional input layer, three 32-dimensional hidden layers, and a 1-dimensional output layer. Moreover, \nobs that the assumption that for all $n \in \mathbb{N}$, $g_1, g_2, \ldots, g_n \in \mathbb{R}^{\mathfrak{d}}$ it holds that $\psi_n(g_1, g_2, \ldots, g_n) = 10^{-2} g_n$ ensures that the stochastic process $\Theta = (\Theta^1, \ldots, \Theta^{\mathfrak{d}}) \colon \mathbb{N}_0 \times \Omega \to \mathbb{R}^{\mathfrak{d}}$ in \cref{eqn:setting:dnn:stochastic_pr} satisfies for all $n \in \mathbb{N}$ that
\begin{equation}
\textstyle \Theta_n = \Theta_{n - 1} - 10^{-2} \mathfrak{G}_n(\Theta_{n - 1})
\end{equation}
(cf.\ Subsection~\ref{subsubsec_Standard_SGD}). Next \nobs that the assumption that $L = 4$, the assumption that $\ell_0 = \allowbreak \ell_4 \allowbreak = 1$, and the assumption that $\ell_1 = \ell_2 = \ell_3 = 32$ demonstrate that the number $\mathfrak{d} \in \mathbb{N}$ of the employed real parameters to describe the considered deep ANNs (see \cref{setting:dnn}) satisfies
\begin{equation}
\begin{split}
\textstyle \mathfrak{d} & \textstyle = \ell_1(\ell_0 + 1) + \ell_2 (\ell_1 + 1) + \ell_3 (\ell_2 + 1) + \ell_4 (\ell_3 + 1) \\
& \textstyle  = 32 (1 + 1) + 32 (32 + 1) + 32 (32 + 1) + 1 (32 + 1) = 2209.
\end{split}
\end{equation}
In addition, \nobs that \cref{eqn:setting:dnn:loss}, the assumption that $L = 4$, and the assumption that $\mathbf{m} = 1024$ assure that for all $n \in \mathbb{N}$, $\theta \in \mathbb{R}^{\mathfrak{d}}$, $\omega \in \Omega$ it holds that $\mathfrak{L}_{\infty}^{n} (\theta, \omega) = \frac{1}{1024} \sum_{k = 1}^{1024} \abs{\mathcal{N}_{\infty}^{4, \theta}(X_k^n (\omega)) - f(X_k^n (\omega))}^2$.


In the left image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_He} we approximately plot 300 samples of the random variable $\mathfrak{L}_{\infty}^{10001}(\Theta_{10000})$ in a histogram with 80 distinct subintervals (with 80 bins) on the $x$-axis. In the upper-right image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_He} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norm $(\mathbb{E}[\norm{\mathfrak{G}_{n + 1}(\Theta_{n})}]^2)^{1/2}$ of the generalized gradient against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In the lower-right image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_He} we approximately plot the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-distance $(\mathbb{E}[\norm{\Theta_{n} - \Theta_{9991}}]^2)^{1/2}$ between $\Theta_n$ and $\Theta_{9991}$ against the number $n \in (\cup_{k = 0}^{998} \{10 k + 1\})$ of SGD training steps. In, both, the upper-right image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_He} and the lower-right image of Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_He} we approximate the expectations in the $L^2 (\P; \mathbb{R}^{\mathfrak{d}})$-norms by means of 300 independent Monte Carlo samples. The source code used to create Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_He} is presented in \cref{subsec_code_ReLU_all_params_3_hidden_layers_He}.


\begin{figure}[H]
\centering
\subfloat{\includegraphics[width=16cm]{ReLU_DNN_SGD_1_32_32_32_1_He.pdf}} 
\caption{Result of the numerical simulations described in  Subsection~\ref{subsec_num_sim_ReLU_all_params_3_hidden_layers_He} (see the {\sc Python} source code in \cref{listing:all_parameters_ReLU_3_hidden_layers_He}).}
\label{fig:all_parameters_ReLU_3_hidden_layers_He}
\end{figure}



\section{{\sc Python} source codes}
\label{sec:python_source_codes}

In this section we present the {\sc Python} source codes used to create the figures in \cref{sec:numerical_simulations}.
\begin{remark}
Please \nobs that each of the {\sc Python} source codes below does not need to be copied from this PDF file but can be downloaded as a separate \underline{\emph{.py}} file on https://arxiv.org by, first, clicking on \underline{\emph{Other formats}} at \underline{\emph{Download}} at the https://arxiv.org webpage associated to this article and by, thereafter, clicking on \underline{\emph{Download source}}.
\end{remark}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_clipping_biases_1_hidden_layer}}
\label{subsec_code_clipping_biases_1_hidden_layer}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:all_biases_clipping_1_hidden_layer}., label=listing:all_biases_clipping_1_hidden_layer]
{P_Clipping_SNN_all_biases.py}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_ReLU_inner_bias_1_hidden_layer}}
\label{subsec_code_ReLU_inner_bias_1_hidden_layer}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:inner_biases_ReLU_1_hidden_layer}., label=listing:inner_biases_ReLU_1_hidden_layer]
{P_ReLU_SNN_inner_bias.py}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_ReLU_all_params_1_hidden_layer_Xavier}}
\label{subsec_code_ReLU_all_params_1_hidden_layer_Xavier}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_Xavier_like}., label=listing:all_parameters_ReLU_1_hidden_layer_Xavier_like]
{P_ReLU_SNN_all_params_Xavier_like.py}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_ReLU_all_params_1_hidden_layer_He}}
\label{subsec_code_ReLU_all_params_1_hidden_layer_He}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:all_parameters_ReLU_1_hidden_layer_He}., label=listing:all_parameters_ReLU_1_hidden_layer_He]
{P_ReLU_SNN_all_params_He.py}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier}}
\label{subsec_code_ReLU_all_params_2_hidden_layers_Xavier}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier}., label=listing:all_parameters_ReLU_2_hidden_layers_Xavier]
{P_ReLU_SGD_DNN_4Layer_Xavier.py}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_He}}
\label{subsec_code_ReLU_all_params_2_hidden_layers_He}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He}., label=listing:all_parameters_ReLU_2_hidden_layers_He]
{P_ReLU_SGD_DNN_4Layer_He.py}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_Xavier_Adam}}
\label{subsec_code_ReLU_all_params_2_hidden_layers_Xavier_Adam}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_Xavier_Adam}., label=listing:all_parameters_ReLU_2_hidden_layers_Xavier_Adam]
{P_ReLU_Adam_DNN_4Layer_Xavier.py}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_ReLU_all_params_2_hidden_layers_He_Adam}}
\label{subsec_code_ReLU_all_params_2_hidden_layers_He_Adam}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:all_parameters_ReLU_2_hidden_layers_He_Adam}., label=listing:all_parameters_ReLU_2_hidden_layers_He_Adam]
{P_ReLU_Adam_DNN_4Layer_He.py}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_ReLU_all_params_3_hidden_layers_Xavier}}
\label{subsec_code_ReLU_all_params_3_hidden_layers_Xavier}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_Xavier}., label=listing:all_parameters_ReLU_3_hidden_layers_Xavier]
{P_ReLU_SGD_DNN_5Layer_Xavier.py}


\subsection{{\sc Python} source code for Subsection~\ref{subsec_num_sim_ReLU_all_params_3_hidden_layers_He}}
\label{subsec_code_ReLU_all_params_3_hidden_layers_He}
\lstinputlisting[language=Python, caption={\sc Python} source code used to create Figure~\ref{fig:all_parameters_ReLU_3_hidden_layers_He}., label=listing:all_parameters_ReLU_3_hidden_layers_He]
{P_ReLU_SGD_DNN_5Layer_He.py}



\subsection*{Acknowledgements}
This project has been funded by the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) under Germany’s Excellence Strategy EXC 2044-390685587, Mathematics Münster: Dynamics-Geometry-Structure.



\newpage
\bibliographystyle{acm}
