
\section{Introduction}


The bandit problem has found wide applications in various areas such as clinical trials \citep{durand2018contextual}, finance \citep{shen2015portfolio}, recommendation systems \citep{zhou2017large}, among others.
Accurate uncertainty quantification is the key to address the exploration-exploitation trade-off. 
Most existing bandit algorithms critically rely on certain analytical property of the imposed model (e.g., linear bandits) to quantify the uncertainty and derive the exploration strategy.
Thompson Sampling \citep[TS,][]{thompson1933likelihood} and Upper Confidence Bound \citep[UCB,][]{auer2002finite} are two prominent examples, which are typically based on explicit-form posterior distributions or confidence sets, respectively. 

However, in many real problems, the reward model is fairly complex: e.g., a general graphical model \citep{chapelle2009dynamic} or a pipeline with multiple prediction modules and manual rules. 
In these cases, it is typically impossible to quantify the uncertainty in an analytical way, and frameworks such as TS or UCB are either methodologically not applicable or computationally infeasible.
Motivated by the real needs, we are concerned with the following question: 




\textit{Can we design a practical bandit algorithm framework that is general, adaptive, and computationally tractable, with certain theoretical guarantee? 
}




A straightforward idea is to apply the bootstrap method \citep{efron1992bootstrap}, a widely applicable data-driven approach for measuring uncertainty. 
However, as discussed in Section \ref{sec:related_work}, most existing bootstrap-based bandit algorithms are either heuristic without a theoretical guarantee, computationally intensive, or only applicable in limited scenarios. 
To address these limitations, we propose a new exploration strategy based on \textit{multiplier bootstrap} \citep{van1996weak}, an easy-to-adapt bootstrap framework that only requires randomly weighted data points. 
We further show that a naive application of multiplier bootstrap may result in linear regret, and we introduce a suitable way to add additional perturbations for sufficient exploration.  










\textbf{Contribution. }
Our contributions are three-fold. 
First,  we propose a general-purpose bandit algorithm framework, \textit{Multiplier Bootstrap-based Exploration} (\name{MBE}). 
The main advantage of \name{MBE} is that it is \textit{general}: it is applicable to any reward model amenable to weighted loss minimization,  without need of analytical-form uncertainty quantification or case-by-case algorithm design. 
As a data-driven exploration strategy, \name{MBE} is also \textit{adaptive} to different environments. 

Second, theoretically, we prove near-optimal regret bounds for \name{MBE} under sub-Gaussian multi-armed bandits (MAB), in both the instance-dependent and the instance-independent sense.
Compared with all existing results for bootstrap-based bandit algorithms, our result is strictly more general (see Table \ref{tab:comparison}), since existing results only apply to some special cases of sub-Gaussian distributions. 
To overcome the technical challenges, 
we proved a novel concentration inequality for some function of sub-exponential variables, and also developed the first \textit{finite-sample} concentration and anti-concentration analysis for multiplier bootstrap, to the best of our knowledge. 
Given the broad applications of multiplier bootstrap in statistics and machine learning, we believe our theoretical analysis has separate interest. 


\input{table_comparison}




Third, with extensive simulation and real data experiments, we demonstrate that \name{MBE} yields comparable performance with existing  algorithms in different MAB settings and three real-world problems (online learning to rank, online combinatorial optimization, and dynamic slate optimization). 
This supports that \name{MBE} is easily generalizable, as it requires minimal modifications and derivations to match the performance of those near-optimal algorithms specifically designed for each problem. 
Moreover, we also show that \name{MBE} adapts to different environments and is relatively robust, due to its data-driven nature. 















\section{Related Work}\label{sec:related_work}

The most popular bandit algorithms, arguably, include $\epsilon$-greedy \citep{watkins1989learning}, TS, and UCB. 
$\epsilon$-greedy is simple and thus widely used. 
However, its exploration strategy is not aware of the uncertainty in data and thus is known to be statistically sub-optimal. 
TS and UCB reply on posteriors and confidence sets, respectively. 
Yet, their closed forms only exist in limited cases, such as MAB or linear bandits. 
For a few other models (such as generalized linear model or neural nets), 
we know how to construct the \textit{approximate} posteriors or confidence sets \citep{filippi2010parametric, li2017provably, phan2019thompson, kveton2020randomized, zhou2020neural}, though the corresponding algorithms are usually costly or conservative. 
In more general cases, it is often not clear how to adapt UCB and TS in a valid and efficient way. 
Moreover, the dependency on the probabilistic model assumptions also pose challenges to being robust. 






To enable wider applications of bandit algorithms, several bootstrap-based (and related perturbation-based) methods have been proposed in the literature. 
Most algorithms are TS-type, by  replacing the posterior with a bootstrap distribution.
We next review the related papers, and summarize those with near-optimal regret bounds in Table \ref{tab:comparison}. 






Arguably, the non-parametric bootstrap is the most well-known bootstrap method, which works by re-sampling data with replacement. 
\citet{vaswani2018new} proposes a version of non-parametric bootstrap 
with forced exploration to achieve 
a $\mathcal{O}(T^{2/3})$ regret bound in Bernoulli MAB. 
\name{GIRO} proposed in \citet{kveton2019garbage} successfully achieves a rate-optimal regret bound in Bernoulli MAB, by adding Bernoulli perturbations to non-parametric bootstrap. 
However, due to the re-sampling nature of non-parametric bootstrap, it is hard to be updated efficiently other than in Bernoulli MAB (see Section \ref{sec:computation_variant}). 
Specifically, the computational cost of re-sampling scales quadratically in $T$. 









Another line of research is the residual bootstrap-based approach (\name{ReBoot}) \citep{hao2019bootstrapping, wang2020residual, tang2021robust, wu2022residual}. 


For each arm, \name{ReBoot} randomly perturbs the residuals of the corresponding observed rewards with respect to the estimated model to quantify the uncertainty for its mean reward. 
We note that, although these methods also use random weights, they are applied to residuals and hence are in a way fundamentally different from ours. 
The limitation is that, by design, this approach is only applicable to problems with a  \textit{fixed}  and  \textit{finite} set of arms, since the residuals are attached closely to each arm (see Appendix \ref{sec:ReBoot} for more details). 











The perturbed history exploration (\name{PHE}) algorithm \citep{kveton2019perturbed, kveton2019perturbedLB, kveton2019perturbedGLB} is also related. 
\name{PHE} works by adding additive noise to the observed rewards.  
\citet{osband2019deep} applies similar ideas to reinforcement learning. 
However, \name{PHE} has two main limitations. 
First, for models where adding additive noise is not feasible (e.g., decision trees), \name{PHE} is not applicable. 
Second, as demonstrated in both \cite{wang2020residual} and our experiments, the fact that \name{PHE} relies on only the extrinsically injected noise for exploration makes it less robust. 
For a complex structured problem, it may not be clear how to add the noise in a sound way \citep{wang2020residual}. 
In contrast, it is typically more natural (and hence easier to be accepted) to leverage the intrinsic randomness in the observed data. 







Finally, we note that multiplier bootstrap has been considered in the bandit literature, mostly as a computationally efficient approximation to non-parametric bootstrap studied in those papers. 
\citet{eckles2014thompson}  studies the direct adaption of multiplier bootstrap (see Section \ref{sec:fail_naive}) in simulation, and its empirical performance in contextual bandits is studied later \citep{tang2015personalized, elmachtoub2017practical, riquelme2018deep, bietti2021contextual}. 
However, no theoretical guarantee is provided in these works. 
In fact, as demonstrated in Section \ref{sec:fail_naive}, such a naive adaptation may have a linear regret.
\citet{osband2015bootstrapped} shows that, in Bernoulli MAB, a variant of multiplier bootstrap is mathematically equivalent to TS. 
No further theoretical or numerical results are established except for this special case. 
Our work is the first systematic study of multiplier bootstrap in bandits.  
Our unique contributions include: 
we identify the potential failure of naively applying multiplier bootstrap, 
highlight the importance of additional perturbations, 
design a general algorithm framework to make this heuristic idea concrete, 
provide the first theoretical guarantee in general MAB settings, 
and conduct extensive numerical experiments to study its generality and adaptivity. 




\section{Multiplier Bootstrap-based Exploration}
\subsection{Failure of the naive adaption of multiplier bootstrap}\label{sec:fail_naive}
To design an exploration strategy based on multiplier bootstrap, 
a natural idea is to replace the posterior distribution in TS with the bootstrap distribution. 
Specifically, at every time point, 
we sample a function $\widehat{f}$ following the multiplier bootstrap procedure as described in Section \ref{sec:preliminary},
and then take the greedy action $\argmax_{a \in \mathcal{A}_t} \widehat{f}(\mathbf{x}_t, a)$. 
However, perhaps surprisingly, such an adaptation may not be valid. 
The main reason is that the intrinsic randomness in a finite dataset is, in some cases, not enough to guarantee sufficient exploration. 
We illustrate with the following toy example. 
\begin{example}\label{eg1}
Consider a two-armed Bernoulli bandit. 
Let the mean rewards of the two arms be $p_1$ and $p_2$, respectively. 
Without loss of generality, assume $1 > p_1 > p_2 > 0$. 
Let $\mathbb{P}(\omega = 0) = 0$. 
Then, with non-zero probability, an agent following the naive adaption of multiplier bootstrap (breaking ties randomly; see Algorithm \ref{alg:MBTS-naive} in Appendix \ref{sec:naive} for details) takes arm $1$ only once. 
Therefore, The agent suffers a linear regret. 

Proof. We first define two events $\mathcal{E}_1 = \{A_t = 1, R_1 = 0\}$ and $\mathcal{E}_2 = \{A_2 = 2, R_2 = 1\}$. 
By design, 
at time $t = 1$, the agent randomly choose an arm and hence will pull arm $1$ with probability $0.5$. 
Then the observed reward $R_1$ is $0$ with probability $1 - p_1$. 
Therefore, $\mathbb{P}(\mathcal{E}_1) = 0.5(1-p_1)$. 
Conditioned on $\mathcal{E}_1$, at $t = 2$, the agent will pull arm $2$ (since multiplying $R_1 = 0$ with any weight always gives $0$), then it will observe reward $R_2 = 1$ with probability $p_2$. 
Conditioned on $\mathcal{E}_1 \cap \mathcal{E}_2$, 
by induction, the agent will pull arm $2$ for any $t > 2$. 
This is because the only reward record for arm $1$ is $R_1 = 0$ and hence its weighted average is always $0$, which is smaller than the weighted average for arm $2$, which is at least positive. 
In conclusion, with probability at least $0.5 \times (1-p_1) \times p_2 > 0$, the algorithm takes the optimal arm $1$ only once. 
\end{example}


\subsection{Main algorithm}\label{sec:main_alg}

The failure of the naive application of multiplier bootstrap implies that some additional randomness is needed to ensure sufficient exploration. 
In this paper, we consider achieving that by adding \textit{pseudo-rewards}, an approach that proves its effectiveness in a few other setups \citep{kveton2019garbage, wang2020residual}. 
The intuition is as follows. 
The under-exploration issue happens when, by randomness, the observed rewards are in the low-value region (compared with the expected reward). 
Therefore, if we can blend in some data points with rewards that have a relatively wide coverage, then the agent would have a higher chance to explore. 




These discussions motivate the design of our main algorithm, Multiplier Bootstrap-based Exploration (\name{MBE}), as in Algorithm \ref{alg:MBTS-general}. 
Specifically, at every round, in addition to the observed reward, we additionally add two pseudo-rewards with value $0$ and $1$. 
The pseudo-rewards are associated with the pulled arm and the context (if exists). 
Then, we solve a weighted loss minimization problem to update the model estimation  (line 8). 
The weights are first sampled from a multiplier distribution  (line 7), and then those of pseudo-rewards are additionally multiplied by a tuning parameter $\lambda$. 
In MAB, the estimates are arm-wise weighted average of all (observed or pseudo-) rewards 
$\overline{Y}_k = \sum_{\ell: A_\ell = k} (\omega_{\ell}  R_{k, \ell} +  \lambda  \omega_{\ell}')   /   \sum_{\ell: A_\ell = k} (\omega_{\ell} + \lambda \omega_{\ell}' + \lambda \omega_{\ell}'') $. 
See Appendix \ref{sec:MBTS_MAB} for details. 









We make three remarks on the algorithm design. 
First, we choose to add pseudo-rewards at the boundaries of the mean reward range (i.e.,  $[0,1]$), since such a design naturally induces a high variance (and hence more exploration). 
Adding pseudo-rewards in other manners is also possible. 
Second, the tuning parameter $\lambda$ controls the amount of extrinsic perturbation and determines the degree of exploration (together with the dispersion of $\rho(\omega)$).
In Section \ref{sec:theory}, we give a theoretically valid range for $\lambda$. 
Finally and critically, besides guaranteeing sufficient exploration, we need to make sure the optimal arm can still be identified (asymptotically) after adding the pseudo-rewards. 
Intuitively, this is guaranteed, since we shift and scale the (asymptotic) mean reward from $f(\mathbf{x}, a)$ to $\big( f(\mathbf{x}, a) + \lambda \big) / (1 + 2\lambda) = 
 f(\mathbf{x}, a) / (1 + 2\lambda) + \lambda / (1 + 2\lambda)$, which preserves the order between arms. 
A detailed analysis for MAB can be found in Appendix \ref{sec:MBTS_MAB}. 














We conclude this section by re-visiting Example \ref{eg1} to provide some insights into how do the pseudo-rewards help. 
\addtocounter{example}{-1}
\begin{example}[Continued]
    Even under the event $\mathcal{E}_1 \, \cap \, \mathcal{E}_2$, 
    Algorithm \ref{alg:MBTS-general} keeps the chance to explore. 
    To see this, consider the example where the multiplier distribution is $2 \times \text{Bernoulli}(0.5)$. 
    Then, we have 
    $\mathbb{P}(A_3 = 1) \ge \mathbb{P}(\overline{Y}_1 > \overline{Y}_0) = 
            \mathbb{P}\left(\frac{\lambda \omega_1'}{\omega_1 + \lambda \omega_1' + \lambda \omega_1''} > \frac{\omega_2 + \lambda \omega_2'}{\omega_2 + \lambda \omega_2' + \lambda \omega_2''}\right)
            \ge 
            \mathbb{P}(\omega'_1 = 2, \omega_1 = \omega_1'' = \omega_2 = \omega_2' = \omega_2'' = 0) = (1/2)^6. $
   
   
   
   
   
   
   
   
   
    Therefore, the agent still has chance to choose the optimal arm. 
\end{example} 


































   
   
   
   
   
   
   
   
   
   
   



\input{Alg/alg_general}


\subsection{Computationally efficient implementation}\label{sec:computation_variant}






Efficient computation is critical for real applications of bandit algorithms. 
One potential limitation of Algorithm  \ref{alg:MBTS-general}  is the computational burden: 
at every decision point, 
we need to re-sample the weights for all historical observations (line 8). 
This leads to a total computational cost of order $\mathcal{O}(T^2)$, similar to \name{GIRO}. 




Fortunately, one prominent advantage of multiplier bootstrap over other bootstrap methods (such as non-parametric bootstrap or residual bootstrap) is that the (approximate) bootstrap distribution can be efficiently updated in an online manner, such that the per-round computation cost does not grow over time. 
Suppose we have a dataset $\mathcal{D}_t$ at time  $t$, and 
denote $\mathcal{B}(\mathcal{D}_t)$ as the corresponding bootstrap distribution for $f$.
With multiplier bootstrap, it is feasible to update $\mathcal{B}(\mathcal{D}_{t+1})$ approximately based on $\mathcal{B}(\mathcal{D}_{t})$. 
We detail the procedure below and elaborate more in Algorithm \ref{alg:MBTS-practical}. 












Specifically, we maintain $B$ different models $\{\widehat{f}_{b,t}\}_{b=1}^B$ 
and the corresponding history (with random weights) as $\{\mathcal{H}_b, \mathcal{H}'_b\}_{b=1}^B$. 
 $\{\widehat{f}_{b,t}\}_{b=1}^B$ can be regarded as sampled from $\mathcal{B}(\mathcal{D}_t)$ and hence the empirical distribution over them is an approximation to the bootstrap distribution. 
At every time point $t$, for each replicate $b$, we only need to sample one weight for the new data point and then update $\widehat{f}_{b,t}$ as  $\widehat{f}_{b,t+1}$.  
Then, $\{\widehat{f}_{b,t+1}\}_{b=1}^B$ are still $B$ valid samples from $\mathcal{B}(\mathcal{D}_{t+1})$ and hence still a valid approximation. 
We note that, since we only have one new data point, 
the updating of $f$ can typically be done efficiently (e.g., with closed-form updating or via online gradient descent). 
The per-round computational cost is hence independent of $t$. 




Such an approximation is a common practice in the online bootstrap literature and can be regarded as an ensemble sampling-type algorithm \citep{lu2017ensemble, qin2022an}. 
The hyper-parameter $B$ is typically not treated as a tuning parameter but depends on the available computational resource \citep{hao2019bootstrapping}. 
In our numerical experiments, this practical variant shows desired performance with $B = 50$. 
Moreover, the algorithm is embarrassingly parallel and also easy to implement: 
given an existing implementation for estimating $f$ (i.e., solving \eqref{eqn:weighted_loss_mini}), 
the major requirement is to replicate it for $B$ times and use random weights for each. 
This feature is attactive in real applications.  










\input{Alg/alg_practical}











\section{Preliminary}\label{sec:preliminary
\textbf{Setup. } 
We consider a general stochastic bandit problem. 
For any positive integer $M$, let $[M] = \{1, \dots, M\}$. 
 At each round $t \in [T]$, the agent observes a context vector $\mathbf{x}_t$ (it is empty in non-contextual problems) and an action set $\mathcal{A}_t$, then chooses an action $A_t \in \mathcal{A}_t$, and finally receives the corresponding reward $R_t = f(\mathbf{x}_t, A_t) + \epsilon_t$, 
Here, $f$ is an unknown function and $\epsilon_t$ is the noise term.
Without loss of generality, we assume $f(\mathbf{x}_t, A_t) \in [0,1]$. 
The goal is to minimize the cumulative regret 
\[
\operatorname{Reg}_T = \sum_{t=1}^T {\mathbb{E}} \big[ \max_{a \in \mathcal{A}_t} f(\mathbf{x}_t, a)  - f(\mathbf{x}_t, A_t) \big].
\]
At time $t$, 
with an existing dataset $\mathcal{D}_t = \{(\mathbf{x}_l, A_l, R_l)\}_{l \in [t]}$, 
to decide the action $A_{t+1}$, 
most algorithms typically first estimate $f$ in some function class $\mathcal{F}$ by solving a weighted loss minimization problem (also called weighted empirical risk minimization or cost-sensitive training) 
\begin{equation}\label{eqn:weighted_loss_mini}
\widehat{f} = 
\argmin_{f \in \mathcal{F}} 
\frac{1}{t}
\sum_{l=1}^{t}
\omega_l \mathcal{L}\big( 
f(\mathbf{x}_l, A_l), R_l
\big) + {J}(f). 
\end{equation}
   
   
   
   
Here, $\mathcal{L}$ is a loss function (e.g., $\ell_2$ loss or negative log-likelihood), $\omega_l$ is the weight of the $l$th data point, and $J$ is an optional penalty function. 
We consider the weighted problem as it is general and related to our proposal below. 
One can just set $\omega_l \equiv 1$ to get the unweighted problem. 
As the simplest example, consider the $K$-armed bandit problem where $\mathbf{x}_l$ is empty and $\mathcal{A}_l = [K]$.  
Let $\mathcal{L}$ be the $\ell_2$ loss, $J \equiv 0$, and $f(\mathbf{x}_l, A_l) \equiv r_{A_l}$ where $r_k$ is the mean reward of the $k$-th arm. 
Then, \eqref{eqn:weighted_loss_mini} reduces to 
$\argmin_{\{r_1, \dots, r_K\}} \sum_{l=1}^t \omega_l (R_l - r_{A_l})^2$, 
which gives the estimator $\widehat{r}_k = 
(\sum_{l:A_l = k} \omega_l)^{-1}
\sum_{l:A_l = k} \omega_l R_l$, i.e., the arm-wise weighted average. 
Similarly, in linear bandits, \eqref{eqn:weighted_loss_mini} reduces to the weighted least-square problem (see Appendix \ref{sec:MBTS_LB} for details). 





\textbf{Challenges. } 
The estimation of $f$, together with the related uncertainty quantification, forms the foundation of most bandit algorithms. 
In the literature, $\mathcal{F}$ is typically a class of models that permit closed-form uncertainty quantification (e.g., linear models, Gaussian processes, etc.).
However, in many real applications, the reward model can yield a fairly complicated structure, e.g., a hierarchical pipeline with both classification and regression modules. 
Manually specified rules are also commonly part of the model.
It is challenging to quantify the uncertainty of these complicated models in analytical forms. 
Even when feasible, 
the dependency on the probabilistic model assumptions also pose challenges to being robust. 






Therefore, in this paper, we focus on the bootstrap-based approach due to its generality and data-driven nature.  
Bootstrapping, as a general approach to quantify the model uncertainty, has many variants. 
The most popular one, arguably, is non-parametric bootstrap (used in \name{GIRO}), which constructs bootstrap samples by re-sampling the dataset with replacement. 
However, due to the re-sampling nature, it is computationally intense (see Section \ref{sec:computation_variant} for more discussions). 
In contrast, multiplier bootstrap \citep{van1996weak}, as an efficient and easy-to-implement alternative, is popular in statistics and machine learning. 



\textbf{Multiplier bootstrap. }
The main idea of multiplier bootstrap is to learn the model using randomly weighted data points. 
Specifically, given a multiplier weight distribution $\rho(\omega)$, 
for every bootstrap sample,
we first randomly sample $\{\omega_t^{MB}\}_{t=1}^{t'} \sim \rho(\omega)$, and then solve \eqref{eqn:weighted_loss_mini} with $\omega_t = \omega_t^{MB}$ to obtain $\widehat{f}^{MB}$. 
Repeat the procedure and the distribution of $\widehat{f}^{MB}$ forms the \textit{bootstrap distribution} that quantifies our uncertainty over $f$. 
The popular choices of  $\rho(\omega)$ include  $\mathcal{N}(1, \sigma_{\omega}^2)$,  $\text{Exp}(1)$,  $\text{Poisson}(1)$, and the double-or-nothing distribution $ 2 \times \text{Bernoulli}(0.5)$.



























\section{Regret Analysis}\label{sec:theory}
In this section, we provide the regret bound for Algorithm \ref{alg:MBTS-general} under MAB with sub-Gaussian rewards.
We regard this as the first step towards the theoretical understanding of \name{MBE}, and leave the analysis of more general settings to future research. 
We call a random variable $X$ as $\sigma$-sub-Gaussian if ${\mathbb{E}} \exp \{ t (X - {\mathbb{E}} X)\} \leq \exp \{ {t^2 \sigma^2} / {2}\}$ for any $t \in \mathbb{R}$.
















   
   
       
           
           
       
   
   



   

   

   











\begin{theorem}\label{thm.MAB.1}
Consider a $K$-armed bandit, where the reward distribution of arm $k$ is $1$-sub-Gaussian with mean $\mu_k$. Suppose arm $1$ is the unique best arm that has the highest mean reward and $\Delta_k=\mu_1-\mu_k$. 
Take the multiplier weight distribution as $\mathcal{N}(1, \sigma_{\omega}^2)$ in Algorithm \ref{alg:MBTS-MAB-2}. 
Let the tuning parameters satisfy $\lambda \geq \left(1 + 4 / \sigma_{\omega} \right) + \sqrt{4  \left(1 + 4 / \sigma_{\omega} \right)/ \sigma_{\omega}  }$.
   
   
  
  
Then, the problem-dependent regret is upper bounded by
\[
     \operatorname{Reg}_T \leq \sum_{k = 2}^K \Big\{ 7 \Delta_k  + \frac{10 \big[ C_1^*(\lambda, \sigma_{\omega}) + C_2^*( \lambda, \sigma_{\omega}) \big]}{\Delta_k} \log T \Big\},
\]
   
   
and the problem-independent regret is  bounded by
\begin{align*}
    \operatorname{Reg}_T  \leq 7 K \mu_1 +  C_1^*(\lambda, \sigma_{\omega}) K \log T + 2 \sqrt{ C_2^* (\lambda, \sigma_{\omega})  K T \log T}. 
\end{align*}
Here, 
$C_1^*(\lambda, \sigma_{\omega}) = 8 \sqrt{2}  C^*_3(\lambda, \sigma_{\omega}) + 38  \sigma_{\omega}^2$ and
$C_2^*( \lambda, \sigma_{\omega}) =  5 \lambda^2 + \left[ 45 (3 + \sigma_{\omega}^2)\lambda^4 C^*_3(\lambda, \sigma_{\omega})  + 38  \sigma_{\omega}^2 \right]$  
are tuning parameter-related components. 
$C^*_3(\lambda, \sigma_{\omega})$ is a logarithmic term as $\log \big[ ( 1 + 15 \sigma_{\omega}^{-2} + 3 \sigma_{\omega} + 10 \sigma_{\omega}^2)  \lambda^2\big] / (3 \log 2) + 1$. \\ 

\end{theorem}


    



The two regret bounds are known as near-optimal (up to a logarithm term) in both the problem-dependent and problem-independent sense \citep{lattimore2020bandit}. 
Notably, recall that the Gaussian distribution and all bounded distributions belong to the sub-Gaussian class. 
Therefore, as reviewed in Table \ref{tab:comparison}, our theory is strictly more general than all existing results for bootstrap-based MAB algorithms.









\textbf{Technical challenges. }
It is particularly  challenging to analyze \name{MBE} due to two reasons. 
First, the probabilistic analysis of multiplier bootstrap itself is technically challenging, since the same random weights appear in both the denominator and the numerator (recall \name{MBE} uses the weighted averages 
$\big\{ \overline{Y}_k = \sum_{\ell: A_\ell = k} (\omega_{\ell}  R_{k, \ell} +  \lambda  \omega_{\ell}')   /   \sum_{\ell: A_\ell = k} (\omega_{\ell} + \lambda \omega_{\ell}' + \lambda \omega_{\ell}'')   \big\}_{k=1}^K$ to select actions in MAB).
It is notoriously complicated to analyze the ratio of random variables, especially when they are correlated. 
Besides, existing bootstrap-based papers rely on the properties of specific \textit{parametric} reward classes (e.g., Bernoulli in \citet{kveton2019garbage} and Gaussian  in  \citet{wang2020residual}), while  we lose these nice structures when considering sub-Gaussian rewards. 

To overcome these challenges, 
we start with  carefully defining two good events on which the weighted average $\overline{Y}_k$, the non-weighted average (with pseudo-rewards)  $\overline{R}_{k}^* = \big( {\sum_{\ell: A_\ell = k} (R_{k, \ell} + 1 \times \lambda + 0 \times \lambda) } \big) / \big( {\sum_{\ell: A_\ell = k} (1 + \lambda + \lambda) } \big)$, and the shifted asymptotic mean $(\mu_k + \lambda) / ({1 + 2 \lambda})$ are close to each other (see Appendix \ref{proof_thm}).
To bound the probability of the bad event and to control the regret on the bad event, 
we face two major technical challenges. 
First, when transforming the ratio into an analyzable form, a summation of correlated sub-Gaussian and sub-exponential variables appears and is hard to analyze. 
We carefully design and analyze a novel event to remove the correlation and the sub-Gaussian terms (see proof of Lemma \ref{lem_bounding_a_k_s_2}). 
Second, the proof needs a new concentration inequality for functions of sub-exponential variables that do not exist in the literature. 
We obtain such a new concentration inequality (Lemma \ref{lem_subE_ineq}) via careful analysis of sub-exponential distributions. 




We believe our new concentration inequality is of separate interest for the analysis of sub-exponential distributions. 
Moreover, to the best of our knowledge, our proof provides the first \textit{finite-sample} concentration and anti-concentration analysis for multiplier bootstrap, which has broad applications in statistics and machine learning. 










   
   
       
       
   
    
   
   
   
       
       
   














\textbf{Tuning parameters. }
In Theorem \ref{thm.MAB.1}, 
\name{MBE} has two tuning parameters $\lambda$ and $\sigma_{\omega}$. 
Intuitively, $\lambda$ controls the amount of external perturbation and $\sigma_{\omega}$ controls the magnitude of exploration from bootstrap. 
In general, higher values of these two parameters facilitate exploration but also lead to a slower convergence. 
The condition $\lambda \geq \left(1 + 4 / \sigma_{\omega} \right) + \sqrt{4  \left(1 + 4 / \sigma_{\omega} \right)/ \sigma_{\omega}  }$ requires that (i) $\lambda$ is not too small and (ii) the joint effect of $\lambda$ and $\sigma$ is not too small. 
Both are intuitive and reasonable. 
In practice, the theoretical condition could be loose: e.g., it requires $\lambda \geq 5 + 2\sqrt{5}$ when $\sigma_{\omega} = 1$. 
As we observe in Section \ref{sec:expriments}, \name{MBE} with a smaller $\lambda$ (e.g., $0.5$) still empirically performs well. 










\iffalse
\red{The argument below does not seem meaningful to me at all. I strongly suggest to remove these arguments if that is all we have, as what we discussed previously.}

For the optimistic property, note that the introducing of the multiplier weights will give extra variance. Indeed, 
\[
    \begin{aligned}
        \operatorname{var}( \overline{Y}_{k, s}) & = \frac{1}{(1+2 \lambda)^2}\left[\left(\frac{\operatorname{var}\left(R_k\right)}{s}+\mu_k^2\right)\left(\frac{\sigma_\omega^2}{s}+1\right)-\mu_k^2\right] + o(s^{-1}) \\
        & > \frac{1}{(1+2 \lambda)^2} \frac{\operatorname{var}\left(R_k\right)}{s} + o(s^{-1}) =\operatorname{var} (\overline{R}_{k, s}^* ) + o(s^{-1}),
    \end{aligned}
\]
which means the target function we use will have more chance to explore. 
\fi
\iffalse
Another informal intuition is if we choose $\lambda$ and $\sigma_{\omega}^2$ carefully, the  the probability of the optimal arm be underestimated is relatively small at \textit{EACH} round. Indeed, we will see 
\[
    \begin{aligned}
        & \mathbb{P} \left(\overline{Y}_{1, s} \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right) \\
        \leq & 1 - \overbrace{{\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - ({\mu_1 + \lambda})/ ({1 + 2 \lambda}) \right)^2 }\right\} \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}}^{\text{the scaling and shifting level should be mild: $\lambda$ satisfying some conditions.}}  \\
        & \qquad \qquad \qquad - \underbrace{\big[ 1 - \Phi(\sqrt{2 \pi})\big] \cdot \mathbb{P} \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}}_{\text{regular concentration: $\sigma_1^2 $ should also be controlled by $\sigma_{\omega}^2$}} \\
        \leq & \Phi(\sqrt{2 \pi}) - \exp \{ - C s\}
    \end{aligned}
\]
under the conditions of Theorem \ref{thm.MAB}.

While formally speaking, the optimistic property is the expected number of rounds that the optimal arm has been being underestimated is not too large.
\fi

\iffalse
\todohw{Unfortunately, applying the anti-concentration inequality in appendix seems not work.

Regarding the claim $\overline{Y}_{k, s}$ is optimistic, denote $\overline{R}_{k, s}^*$ as the mean of the shifted version and consider the event that the scaled-shifted sample mean of the optimal arm is less than the expectation, i.e.
\[
    E : = \left\{  \overline{R}_{1, s}^* = \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \varepsilon \right\}
\]
with some positive $\varepsilon > 0$, then
\[
    \begin{aligned}
        \mathbb{P} (E) & = \mathbb{P} \left( \overline{R}_{1, s}^* = \frac{\mu_k + \lambda}{1 + 2 \lambda} - \varepsilon  \right) \\
        & \leq \mathbb{P} \left( \overline{R}_{1, s}^* \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \varepsilon  \right) \\
        & = \mathbb{P} \big( (\overline{R}_{1, s} - \mu_1) \leq - \varepsilon (1 + 2 \lambda)\big) \leq \exp \left\{ - \frac{\sigma_1^2}{s \varepsilon^2 (1 + 2 \lambda)^2}\right\}.
    \end{aligned}
\]
On the other hand, by the proof in Section C.2 in appendix, we will also have 
(i)
\[
    \begin{aligned}
        \mathbb{P} \left( \overline{Y}_{1, s} \geq  \frac{\mu_1 + \lambda}{1 + 2 \lambda} \mid E \right) & = {\mathbb{E}} \Bigg[ \mathbb{P} \left( \overline{Y}_{1, s} \geq  \frac{\mu_1 + \lambda}{1 + 2 \lambda} \mid \mathcal{H}_{1, s} \right) \mid E \Bigg] \\
        & \geq {\mathbb{E}} \Bigg[ \exp \left\{ - \frac{3 s (1 + 2 \lambda)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} \cdot 1 \left\{ \overline{R}_{1,s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \mid E \Bigg] \\
        & \qquad + {\mathbb{E}} \Bigg[ \big[ 1 - \Phi(\sqrt{2 \pi})\big] \cdot 1 \left\{  \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \mid E \Bigg] \\
        & =  {\mathbb{E}} \Bigg[ \big[ 1 - \Phi(\sqrt{2 \pi})\big]  \cdot  1 \left\{ - \varepsilon (1 + 2 \lambda) < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \mid E \Bigg] \\
        & =  \big[ 1 - \Phi(\sqrt{2 \pi})\big].
    \end{aligned}
\]
But 
\[
    \exp \left\{ - \frac{\sigma_1^2}{s \varepsilon^2 (1 + 2 \lambda)^2}\right\} \longrightarrow 1 >  \big[ 1 - \Phi(\sqrt{2 \pi})\big].
\]
(ii)
\[
    \begin{aligned}
        \mathbb{P} \left( \overline{Y}_{1, s} \geq  \frac{\mu_1 + \lambda}{1 + 2 \lambda} \mid E \right) & = {\mathbb{E}} \Bigg[ \mathbb{P} \left( \overline{Y}_{1, s} \geq  \frac{\mu_1 + \lambda}{1 + 2 \lambda} \mid \mathcal{H}_{1, s} \right) \mid E \Bigg] \\
        & \geq {\mathbb{E}} \Bigg[ \exp \left\{ - \frac{3 s (1 + 2 \lambda)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} \cdot 1 \left\{ \mu_1 - \overline{R}_{1,s} \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \mid E \Bigg] \\
        & \qquad + {\mathbb{E}} \Bigg[ \big[ 1 - \Phi(\sqrt{2 \pi})\big] \cdot 1 \left\{ \mu_1 -  \overline{R}_{1, s} < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \mid E \Bigg] \\
        & \geq {\mathbb{E}} \Bigg[  \exp \left\{ - \frac{3 s (1 + 2 \lambda)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} 1 \left\{ \varepsilon (1 + 2 \lambda) \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \mid E \Bigg] \\
        \left(\text{Take } \lambda \geq \frac{1}{2} \left[ \left( \frac{2 \pi}{\varepsilon}\right)^{1 / 4} - 1\right] \vee 0 \right)& \geq {\mathbb{E}} \Bigg[  \exp \left\{ - \frac{3 s (1 + 2 \lambda)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} \mid E \Bigg] \\
        & = \frac{{\mathbb{E}} \Bigg[  \exp \left\{ - \frac{3 s (1 + 2 \lambda)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} 1(E) \Bigg] }{\mathbb{P} (E)} 
    \end{aligned}
\]
But require the lower bound of sub-Gaussian variable.
}
\todohw{Another asymptotic way to analyze the optimistic:

Regarding the claim $\overline{Y}_{k, s}$ is optimistic, we first calculate the conditional variance of the target function under history. We write
\[
    \overline{Y}_{k, s} = \frac{\left(s^{-1} \sum_{i = 1}^s R_{k, i} \omega_i - \overline{R}_{k, s} \right) + \overline{R}_{k, s} + \lambda}{\left( s^{-1} \sum_{i = 1}^s \omega_i - 1\right) + 1 + 2 \lambda} =: \frac{Z_{k, 1} + \overline{R}_{k, s} + \lambda}{Z_{k, 2} + 1 + 2 \lambda} 
\]
with 
\[
    Z_{k, 1} \mid \mathcal{H}_{k, s} \, \sim \, N \left( 0, \frac{\sigma_{\omega}^2}{s^2}\sum_{i = 1}^s R_{k, i}^2 \right), \qquad Z_{k, 2} \mid \mathcal{H}_{k, s} \, \sim \, N\left( 0,  \frac{\sigma_{\omega}^2}{s}\right)
\]
and 
\[
    \operatorname{cov} ( Z_{k, 1},  Z_{k, 2} \mid \mathcal{H}_{k, s}) = \frac{\sigma_{\omega}^2}{s} \overline{R}_{k, s}.
\]
Then by Geary–Hinkley transformation (see \cite{hayya1975note}), we uncorrelated $\overline{Y}_{k, s} \mid \mathcal{H}_{k, s}$ as
\[
    \overline{Y}_{k, s}  = \frac{Z_{k, 1}’ + \lambda(2 \overline{R}_{k ,s} - 1)}{Z_{k, 2} + 1 + 2 \lambda} + \overline{R}_{k, s}, \qquad Z_{k, 1}' \mid \mathcal{H}_{k, s} \, \sim \, N \left( 0, \frac{\sigma_{\omega}^2}{s^2}\sum_{i = 1}^s R_{k, i}^2 - \frac{\overline{R}_{k, s}^2}{s}\right).
\]
Hence, again by \cite{hayya1975note}, the conditional variance of $\overline{Y}_{k, s}$ will converge to 
\[
    \begin{aligned}
        \operatorname{var}(\overline{Y}_{k, s}  \mid \mathcal{H}_{k, s}) \, & \longrightarrow \, \frac{\lambda^2 (2 \overline{R}_{k, s} - 1)^2}{(1 + 2 \lambda)^2} + \exp \left\{ \frac{\operatorname{var}(Z_{k, 1} \mid \mathcal{H}_{k, s})}{ \lambda^2 (2 \overline{R}_{k ,s} - 1)^2} + \frac{\operatorname{var}(Z_{k, s} \mid \mathcal{H}_{k, s})}{(1 + 2 \lambda)^2}\right\} \\
        & \qquad \qquad \qquad \left[\exp \left\{ \frac{\operatorname{var}(Z_{k, 1} \mid \mathcal{H}_{k, s})}{ \lambda^2 (2 \overline{R}_{k ,s} - 1)^2} + \frac{\operatorname{var}(Z_{k, s} \mid \mathcal{H}_{k, s})}{(1 + 2 \lambda)^2}\right\} - 1 \right] \\
        & \geq \frac{\lambda^2 (2 \overline{R}_{k, s} - 1)^2}{(1 + 2 \lambda)^2}
    \end{aligned}
\]
Denote $\overline{R}_{k, s}^*$ as the mean of the shifted version, consider the event that the scaled-shifted sample mean of the optimal arm is less than the expectation, i.e.
\[
    E : = \left\{  \overline{R}_{1, s}^* = \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \varepsilon \right\}
\]
with some positive $\varepsilon > 0$, then
\[
    \begin{aligned}
        \mathbb{P} (E) & = \mathbb{P} \left( \overline{R}_{1, s}^* = \frac{\mu_k + \lambda}{1 + 2 \lambda} - \varepsilon  \right) \leq \mathbb{P} \left( \overline{R}_{1, s}^* \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \varepsilon  \right).
    \end{aligned}
\]
Note that the variance of $ \operatorname{var} (\overline{R}_{1, s}^*) = \frac{1}{(1 + 2 \lambda)^2 s} \operatorname{var}(R_{1, 1})$ and $2 \mu_1  - 1 > 0$, the for sufficient large $T$, take $\lambda \geq \frac{\sqrt{\operatorname{var} (R_{1, 1})}}{(2 \mu_1 - 1)}$, we have $\operatorname{var} (\overline{Y}_{1, s} \mid \mathcal{H}_{1, s}) \geq \operatorname{var}(\overline{R}_{1, 1})$. Therefore, under this case, 
\[
    \mathbb{P} (E) \leq  \mathbb{P} \left( \overline{Y}_{1, s} \geq \overline{R}_{1, s}^* + \varepsilon \mid E \right) \leq \mathbb{P} \left( \overline{Y}_{1, s} \geq  \frac{\mu_1 + \lambda}{1 + 2 \lambda} \mid E \right)
\]
which means any unfavorable history is less likely than being optimistic under that history. Thus, $\overline{Y}_{k, s}$ is optimistic.}
\fi























   
   
       
           
       
   
   




   









   
   
   
   
   
   
       
       
   
   
       
   
   
   

   

   

   
   
   
   
      
   
   
   
       
   
   





\section{Experiments}\label{sec:expriments}


In this section, we study the empirical performance of \name{MBE} with both simulation (Section \ref{sec:exp_MAB}) and real datasets (Section \ref{sec:exp_CB}). 


\begin{figure*}[!t]
    \includegraphics[width=\linewidth]{Fig/MAB_larger.pdf}
    \vspace{-.8cm}
    \caption{Simulation results under MAB. The error bars indicate the standard errors, which may not be visible when the width is small. } 
   
    \label{sim_MAB}
\end{figure*}




 
\subsection{MAB Simulation}\label{sec:exp_MAB}
We first experiment with simulated MAB instances. 
The goal is to (i) further validate our theoretical findings, (ii) check whether \name{MBE} can yield comparable performance with standard methods, and (iii) study the robustness and adaptivity
 of \name{MBE}. 
We also experimented with linear bandits and the main findings are similar. 
To save space, we defer its results to Appendix \ref{sec:results_LB}. 


We compare \name{MBE} with \name{TS} \citep{thompson1933likelihood},  \name{PHE} \citep{kveton2019perturbed}, \name{ReBoot} \citep{wang2020residual}, and \name{GIRO} \citep{kveton2019garbage}. 
The last three algorithms are the existing bootstrap- or perturbation-type algorithms reviewed in Section \ref{sec:related_work}. 
Specifically, \name{PHE} explores by  perturbing observed rewards with additive noise, without leveraging the intrinsic uncertainly in the data, 
\name{ReBoot} explores by perturbing the residuals of the rewards observed for each arm, 
and \name{GIRO} re-samples observed data points.  
In all experiments below, the weights of \name{MBE} are sampled from $\mathcal{N}(1,\sigma_{\omega}^2)$ \footnote{We also experimented with other weight distributions with similar main conclusions. 
Using Gaussian weights allows us to study impact of different multiplier magnitudes more clearly. 
}. 
We fix $\lambda = 0.5$ and run \name{MBE} with three different values of $\sigma_{\omega}^2$ as $0.5, 1$ and $1.5$. 
We also compare with the naive adaption of multiplier bootstrap (i.e., no pseudo-rewards; denoted as \name{Naive MB}). 
We run Algorithm \ref{alg:MBTS-practical} with $B=50$ replicates. 








 

We first study $10$-armed bandits, where the mean reward of each arm is independently sampled from $\text{Beta}(1, 8)$. 
We consider three reward distributions, including Bernoulli, Gaussian, and exponential. 
For Gaussian MAB, the reward noise is sampled from $\mathcal{N} (0, 1)$. 
The other two distributions are determined by their means. 
For \name{TS}, we always use the correct reward distribution class and its conjugate prior. 
The prior mean and variance are calibrated using the true model. 
Therefore, we use \name{TS} as a strong and standard baseline. 
For \name{GIRO} and \name{ReBoot}, we use the default implementations as they work well. 
For \name{PHE}, the original paper adds Bernoulli perturbation since it only studies bounded reward distributions. 
We extend \name{PHE} by sampling additive noise from the same distribution family as the true rewards, as did in \citet{wu2022residual}. 
\name{GIRO}, \name{ReBoot} and \name{PHE} all have one tuning parameter that control the degree of exploration. 
We tune these hyper-parameters over $\{2^{k-4}\}_{k=0}^6$, and report the best performance of each method. 
Without tuning, these algorithms generally do not perform well using the hyper-parameters suggested in the original papers, due to the differences in settings. 
We tuned \name{Naive MB} as well. 













\textbf{Results. }
Results over $100$ runs are  displayed in Figure \ref{sim_MAB}. 
Our findings can be summarized as follows. 
First, without knowledge of the problem settings (e.g., the reward distribution family and its parameters, and the prior distribution) and without heavy tuning, \name{MBE} performs favorably and close to \name{TS}. 
Second,  pseudo-rewards are indeed important in exploration, otherwise the algorithm suffers a linear regret. 
Third, \name{MBE} has a stable performance with different $\sigma_\omega$ (note that other methods are tuned for their best performance). 
This is thanks to the data-driven nature of  \name{MBE}. 
Finally, the other three general-purpose exploration strategies perform reasonably after tuning, as expected. 
However, \name{GIRO} is computationally intense. 
For example, in Gaussian bandits, 
the time cost for \name{GIRO} is $2$ minutes while all the other algorithms can complete within $10$ seconds. 
The computational burden is due to the limitation of non-parametric bootstrap (see Section \ref{sec:computation_variant}). 
\name{ReBoot} also performs reasonably, yet by design it is not easy to extend it to many more complex problems (e.g., problems in Section \ref{sec:exp_CB}). 


\textbf{Adaptivity. }
\name{PHE} relies on sampling additive noise from an appropriate distribution, and \name{TS} has similar dependency. 
In the results above, we provide auxiliary information about the environment to them and need to modify their implementation in different setups. 
In contrast, \name{MBE} automatically adapts to these problems. 
As argued in Section \ref{sec:related_work}, one main advantage of \name{MBE} over them is its adaptiveness.
To see this, we consider the following procedure: 
we run the Gaussian versions of \name{TS} and \name{PHE} in Bernoulli MAB, and run their Bernoulli versions in Gaussian MAB. 
We also run \name{MBE} with $\sigma_\omega^2 = 0.5$. 
\name{MBE} does not require any modifications across the two problems. 
The results presented in Figure \ref{fig:robust_dist} clearly demonstrate that \name{MBE} adapts to reward distributions. 



Similarly, we also studied the adaptivity of these methods against the reward distribution scale (the standard deviation of the Gaussian noise, $\sigma$) and the task distribution (we sample the mean rewards from $\text{Beta}(\alpha, 8)$ and vary the parameter $\alpha$). 
For all settings, we use the algorithms tuned for Figure \ref{sim_MAB}. 
We observe that, \name{MBE} shows impressive adaptiveness, while \name{PHE} and \name{TS} may not perform well when the environment is not close to the one they are tuned for. 
Recall that, in real applications, heavy tuning is not possible without ground truths. 
This demonstrates the adaptivity of \name{MBE}, as a data-driven exploration strategy.

\textbf{Additional results. }
In Appendix \ref{sec:results_robust_lam}, we also try different values of $\lambda$ and $B$ for \name{MBE}. 
We also repeat the main experiment with $K = 25$. 
Our main observations above still hold and \name{MBE} is relatively robust to these tuning parameters. 



   




\begin{figure}[!t]
    \includegraphics[width=\linewidth]{Fig/MAB_dist_robust.pdf}
    \vspace{-.8cm}
    \caption{Robust results against the reward distribution class. }
    \label{fig:robust_dist}
    \vspace{-.2cm}
\end{figure}







\begin{figure}[t]
    \centering
    \subcaptionbox{Trend with  $\alpha$ }
  {\includegraphics[width=0.22\textwidth]{Fig/Robust_alpha.pdf}  }
  \hspace{.1cm}
    \subcaptionbox{Trend with $\sigma$}
  {\includegraphics[width=0.22\textwidth]{Fig/Robust_sigma.pdf}  }
 
 
    \vspace{-.1cm}
    \caption{Results with different reward variances and  task distributions. 
    For the x-axis in both figures and the y-axis in the second one, we plot at the logarithmic scale for better visualization.}
   
\end{figure}













































\input{real_figure.tex}

\subsection{Real data applications}\label{sec:exp_CB}
The main advantage of \name{MBE} is that it easily generalizes to complex models. 
In this section, we use real datasets to study this property. 
Our goal is to investigate that, without problem-specific algorithm design and without heavy tuning, whether \name{MBE} can achieve comparable performance with strong problem-specific baselines proposed in the literature.  








We study the three problems considered in \citet{wan2022towards}, including 
cascading bandits for online learning to rank  \citep{kveton2015cascading}, 
combinatorial semi-bandits for online combinatorial optimization \citep{chen2013combinatorial}, 
and multinomial logit (MNL) bandits for dynamic slate optimization \citep{agrawal2017thompson, agrawal2019mnl}.  
All these are practical and important problems in real life. 
Yet, these domain models all have unique structures and require a case-by-case algorithm design. 
For example, the rewards in MNL bandits follow multinomial distributions that have complex dependency with the pulled arms. 
To derive the posterior or confidence bound, one has to use a delicately designed epoch-type procedure \citep{agrawal2019mnl}. 



We compared \name{MBE} with state-of-the-art baselines in the literature, including \name{TS-Cascade} \citep{zhong2021thompson} and \name{CascadeKL-UCB} \citep{kveton2015cascading} for cascading bandits, 
\name{CUCB} \citep{chen2016combinatorial} and \name{CTS} \citep{wang2018thompson} for semi-bandits, 
and \name{MNL-TS} \citep{agrawal2017thompson} and \name{MNL-UCB} \citep{agrawal2019mnl} for MNL bandits. 
To save space, we denote the TS-type algorithms by \name{TS} and UCB-type ones by \name{UCB}. 
We also study \name{PHE} and $\epsilon$-greedy (\name{EG}) as two other general-purpose exploration strategies. 











We use the three datasets studied in  \citet{wan2022towards}.  
Specifically, we use the Yelp rating dataset \citep{zong2016cascading} to recommend and rank $K$ restaurants, 
use the Adult dataset \cite{Dua:2019} to send advertisements to $K/2$ men and $K/2$ women (a combinatorial semi-bandit problem with continuous rewards),  
and use the MovieLens dataset \citep{harper2015movielens} to display $K$ movies. 
In our experiments, we fix $K= 4$ and randomly sample $30$ items from the dataset to choose from. 
We provide a summary of these datasets and problems in Appendix \ref{sec:exp_details}, and refer interested readers to \citet{wan2022towards} and references therein for more details. 





For the baseline methods, as in  Section \ref{sec:exp_MAB}, we either use the default hyperparameters in \citet{wan2022towards} or tune them extensively and present their best performance. 
For \name{EG}, we set the exploration rate $\epsilon_t = \min(1, a/2\sqrt{t})$ with tuning parameter  $a$. 
For \name{MBE}, with every bootstrap sample, we estimate the reward model via maximum weighted likelihood estimation, which yields nice closed-form solution that allows online updating in all three problems. 
The other implementation details are similar to Section \ref{sec:exp_MAB}. 







We present the results in Figure \ref{fig:real}. 
The overall findings are consistent with simulation. 
First, without any additional derivations or algorithm design, 
\name{MBE} matches the performance of problem-specific algorithms. 
Second, pseudo-rewards are important to guarantee sufficient exploration, and naively applying multiplier bootstrap may fail. 
Third, \name{MBE} has relatively stable performance with $\sigma_\omega$, as its exploration is mostly data-driven. 
In contrast, we found that the hyper-parameters of \name{PHE} and \name{EG} have to be carefully tuned, due to that they rely on externally added perturbation or forced exploration. 
For example, the best parameters for \name{EG} are $a = 5$, $0.1$ and $0.5$ in three problems. 
Finally, we observe that \name{PHE} does not perform well in MNL and cascading bandits, where the outcomes are binary. 
From a closer look, we find one possible reason: 
the response rates (i.e., the probabilities for the binary outcome to be $1$) in the two datasets are low, so \name{PHE} introduces too much (additive) noise for exploration purpose, which slows down the estimation convergence. 























\section{Conclusion}

In this paper, we propose a new bandit exploration strategy, Multiplier Bootstrap-based Exploration (\name{MBE}). 
The main advantage of \name{MBE} is its generality: for any reward model that can be estimated via weighted loss minimization, 
the idea of  \name{MBE} is applicable and requires minimal efforts on derivation or implementation of the exploration mechanism. 
As a data-driven method, \name{MBE} also shows nice adaptivity. 
We prove near-optimal regret bounds for  \name{MBE} in the sub-Gaussian MAB setup, which is more general compared with other bootstrap-based bandit papers. 
Numerical experiments demonstrate that  \name{MBE} is general, efficient, and adaptive. 



There are a few meaningful future extensions. 
First, the regret analysis for \name{MBE} (and more generally, other bootstrap-based bandit methods) in more complicated setups would be valuable. 
Second, adding pseudo-rewards at every round is needed for the analysis. 
We hypothesize that there exists a more adaptive way of adding them. 
Last, the practical implementation of \name{MBE} relies on an ensemble of models to approximate the bootstrap distribution and the online regression oracle to update the model estimation. 
Our numerical experiments show that such an approach works well empirically, but it would be still meaningful to have more theoretical understanding. 






















\section{Main Proof}\label{proof_thm}

This section gives the proof of the main regret bound (Theorem \ref{thm.MAB.1}). 
Section \ref{sec_lem_components} gives the major lemmas required to bound the regret components used in this section. 
Section \ref{sec_lem_tech} lists all supporting technical lemmas, including the lower bound of the Gaussian tail and some novel results on the concentration property of sub-Gaussian and sub-exponential distributions.

We will present a complete version of the theory of \name{MBE} under MAB, Theorem \ref{thm.MAB}, as well as its proof, in which we allow arbitrary variance proxy $\sigma_k^2$ rather than requiring them to be one. 

\begin{theorem}\label{thm.MAB}
Consider a $K$-armed bandit, where the reward distribution of arm $k$ is $\operatorname{subG}(\sigma_k^2)$ with mean $\mu_k$. Suppose $\mu_1 = \max_{k \in [K]} \ \mu_k$ and $\Delta_k = \mu_1 - \mu_k$. 
Take the multiplier weight distribution as $\mathcal{N}(1, \sigma_{\omega}^2)$ in Algorithm \ref{alg:MBTS-MAB-2}. 
Let the tuning parameters satisfy $\lambda \geq \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1 \right) + \sqrt{\frac{4\sigma_1}{\sigma_{\omega}} \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1\right)}$,
   
   
  
  
Then the problem-dependent regret is upper bounded by
\[
     \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \Bigg[ 7 + \left\{ C_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) + \frac{C_2(\sigma_1,  \sigma_k, \lambda, \sigma_{\omega})}{\Delta_k^{2}} \right\} \log T \Bigg],
\]
   
   
where 
\[
    C_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) = 10 \bigg[ 8 \sqrt{2} \max_{k \in [K]} \sigma_{k}^2 \left( \frac{\log D_2 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega})}{3 \log 2} + 1\right) + 38  \sigma_{\omega}^2 \bigg],
\]
and
\[
    \begin{aligned}
        C_2(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) & = 50 \lambda^2 \sigma_k^2 + 10 \bigg[ D_1 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega}) \left( \frac{\log D_2 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega})}{3 \log 2} + 1\right) + 38  \sigma_{\omega}^2 \bigg],
    \end{aligned}
\]
with 
\[
    \begin{aligned}
        D_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) & = \bigg[ \big(1 +  8 \sqrt{2} \max_{k \in [K] } \sigma_k^2 \big) \bigg( 16 + \frac{\sigma_{\omega}^2}{\sigma_1^2} \bigg) + 16 \sigma_1^4 + 3 \frac{\sigma_{\omega}^2}{\sigma_1^2} + 3 \sigma_{\omega}^2 + 1 \bigg] \sigma_{\omega}^2 \lambda^4,
    \end{aligned}
\]
and
\[
    \begin{aligned}
        D_2(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) & = 3 \bigg[ 1 +  \frac{3 \sqrt{\pi}\sigma_{\omega}^2}{2 \sigma_1} \left( \frac{\sigma_1}{\sigma_{\omega}} + 3 \lambda \right) + 16 \sqrt{\pi} \max_{k \in [K]} \sigma_{k}^2 \left( \frac{\sigma_{\omega}^2}{16 \sigma_1^4} + \frac{1}{\sigma_{\omega}^2} \right) + \frac{\lambda^2 \sigma_{\omega}^2}{4 \sigma_1^4} \bigg].
    \end{aligned}
\]
Furthermore, the problem-independent regret is upper bounded by

\[
    \operatorname{Reg}_T  \leq 7 K \mu_1 + \max_{k \in [K] \setminus \{ 1 \}}  C_1(\sigma_1, \sigma_k, \lambda) K \log T + 2 \sqrt{\max_{k \in [K] \setminus \{ 1 \}}  C_2 (\sigma_1, \sigma_k, \lambda) K T \log T}.
   
\] 
\end{theorem}

Before starting our proof, we first give the definition of sub-exponential variables: we call a mean-zero variable $X$ sub-exponential with parameters $\lambda$ and $\alpha$ if
\[
    {\mathbb{E}} \exp \{ t X \} \leq \exp \left\{ \frac{t^2 \lambda^2}{2}\right\}, \qquad |t| \leq \frac{1}{\alpha}.
\]
And we denote as $X \sim \operatorname{subE}(\lambda, \alpha)$ if sub-exponential $X$ has parameters $\lambda$ and $\alpha$.
We have the following concentration {property}
\[
    \mathbb{P} (X \geq t) \leq \exp \left\{ - \frac{1}{2} \left( \frac{t^2}{\lambda^2} \wedge \frac{t}{\alpha}\right) \right\}, \qquad  {\forall} \, t \geq 0,
\]
where $a \wedge b$ is defined as $\min \{ a, b\}$. See \citet{zhang2020concentration} and \citet{zhang2022sharper} for more details. 
For simplicity, 
we denote $\operatorname{subE}(\lambda) := \operatorname{subE}(\lambda, \lambda)$. 
We call a random variable $X$ sub-Gaussian with variance proxy $\sigma^2$ if ${\mathbb{E}} \exp \{ t (X - {\mathbb{E}} X)\} \leq \exp \{ {t^2 \sigma^2} / {2}\}$ for any $t \in \mathbb{R}$, and denote it as $X\sim \operatorname{subG}(\sigma^2)$.

\textbf{Notations:} \textit{Denote $\mathbb{P}_{\xi}(A) = \int_{A} \mathrm{d} F_{\xi}(x)$ as the probability of event $A$, where $F_{\xi}(x)$ is the distribution function of the random variable $\xi$, and similarly, we denote ${\mathbb{E}}_{\xi} f(\xi) = \int f(x) \, \mathrm{d} F_{\xi} (x)$ as the expectation. Recall that we write two functions $a(s, T) \lesssim b(s, T)$ if $a(s, T) \leq c \times b(s, T)$ for some constant $c$ free of $s$ and $T$. And we write $a(s, T) \asymp b(s, T)$ if both $a(s, T) \lesssim b(s, T)$ and $a(s, T) \gtrsim b(s, T)$. Furthermore, we define $a \vee b = \max\{a, b\}$ and $a \wedge b = \min \{ a, b\}$ for any real numbers $a$ and $b$. Similarly, we can define $a \vee b \vee c = \max\{ a, b, c\}$ and $a \wedge b \wedge c = \min\{ a, b, c\}$ for any $a, b, c \in \mathbb{R}$.}

\begin{proof}
\vspace{1ex}

\textbf{Step 0: Decomposition of the regret bound.} Our proof relies on the following decomposition of the cumulative regret \citep{kveton2019garbage}. 
We denote the first $s$ rewards from pulling arm $k$ as  $\mathcal{H}_{k, s}$, with the $i$-th observations denoted as $R_{k, i}$. 
Let $Q_{k, s} (\tau) = \mathbb{P} \big( \overline{Y}_{k, s} > \tau \mid \mathcal{H}_{k, s} \big)$ be the tail probability that $\overline{Y}_{k, s}$ conditioned on history $\mathcal{H}_{k, s}$ is a least $\tau$, and $N_{k, s} (\tau) = 1 / Q_{k, s} (\tau) - 1$ is the expected number of rounds that the arm $k$ being underestimated given $s$ sample rewards. Here
\[
    \overline{Y}_{k, s} = \frac{\sum_{i = 1}^s \left[ \omega_i R_{k, i} + \omega_i' \cdot (1 \times \lambda) + \omega_i'' \cdot (0 \times \lambda) \right]}{\sum_{i = 1}^s (\omega_i + \lambda \omega_i' + \lambda \omega_i'')}
\]
is the objective function defined in Algorithm \ref{alg:MBTS-MAB-2}.

\begin{lemma}\label{lem.a.b}
Suppose in MAB we select arms according to the rule 
$A_t = \arg \max_{k \in [K]} \overline{Y}_{k, t}$ with $\overline{Y}_{k, t}$ defined in Algorithm \ref{alg:MBTS-MAB-2}.
Then for any $\{ \tau_k \}_{k = 2}^K \subseteq \mathbb{R}$, 
the expected $T$-round regret can be bounded above as
\[
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big),
\]
where $a_k = \sum_{s = 0}^{T - 1} a_{k, s}$ and $ b_k = \sum_{s = 0}^{T - 1} b_{k, s} + 1$, and $a_{k, s} = {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] $ and $b_k = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big)$.
\end{lemma}
Recall the summation index $s$ is the number of times we pull the $k$-th arm. 

The definitions of $a_{k}$ and $b_{k}$ have important meanings: $a_{k}$ represents the expected number of rounds that optimal arm $1$ has been being underestimated, whereas $b_{k}$ is the probability that the suboptimal arm $k$ is being overestimated.
Here we only need to consider the lower bound of the tail of the distribution of the rewards from the optimal arm. 
The intuition is: (i) we only need the rewards from the optimal arm taking a relatively large value with a probability that is not too small; (ii) we do not care about the tiny probability of a large reward from suboptimal arms. 

Therefore, our target is then to bound $a_k$ and $b_k$ for any $k \geq 2$. 
These are completed in Step 1 and Step 2 below, respectively. 


\vspace{2ex}



\textbf{Step 1: Bounding $a_k$.} 

We first provide a roadmap 
for proving $a_k$ is bounded by a term of $O(\log T)$ order:
For a given constant level $\tau_k$, the probability of the optimal arm 1 being underestimated given $s$ rewards is $1-$ $Q_{1, s}\left(\tau_k\right)$. If we pick the level to satisfy $\tau_k < \frac{\mu_1 + \lambda}{1 + 2 \lambda}$, the theory of large deviation gives
\[
    \lim_{s \rightarrow \infty} Q_{1, s} (\tau_k) = 1 .
\]
Hence the expected number of rounds to observe a not-underestimated case $N_{1, s}(\tau_k) =\frac{1}{Q_{1, s}(\tau_k)}-1$ has the property $\lim_{s \rightarrow \infty} N_s (\tau_k) = 1$ as the number of pulls $s$ grows to infinity. Thus, given the round $T$, there exists a constant $s_0(T)$ such that $N_s\left(\tau_k\right) \leq T$ for all $s$ over $s_0(T)$. Consequently, the quantity $a_k$ in regret bound will be bounded by
\[
    a_k \leq \sum_{s=0}^{s_0(T)} {\mathbb{E}} \left[ N_{1, s} (\tau_k)\wedge T\right] .
\]
The fact that the constant $s_0(T)$ is at the order of $\log T$ will be shown in Lemma \ref{lem_bounding_a_k_s_1}, Lemma \ref{lem_bounding_a_k_s_2}, and Lemma \ref{lem_bounding_a_k_s_3}. For small number of pulls $s \leq s_0(T)$, we show in Lemma \ref{lem_bounding_a_k_s_constant} that ${\mathbb{E}} \left[ N_{1, s} (\tau_k)\wedge T\right] \leq 2 + 4 e^{9 / 8} $ for any $s \in \mathbb{N}$. Thus, it is enough to conclude that $a_k$ can be bounded by a term of $O(\log T)$ order.

To formally bound $a_k$ in the non-asymptotic sense following the intuition above, we need to decompose $a_k$.
For the decomposition, a common approach is to use indicators on \textit{good} events. Denote the shifted (sample-) mean reward as 
\[
    \overline{R}_{k, s}^* = \frac{\sum_{i = 1}^s R_{k, i} + \lambda s}{s(1 + 2 \lambda)} = \frac{\lambda}{1 + 2 \lambda} + \frac{1}{1 + 2 \lambda} \overline{R}_{k, s}.
\]
where $\overline{R}_{k, s} = \frac{1}{s} \sum_{i = s} R_{k, i}$ is the mean reward of the $k$-th arm. Then we can define the following \textit{good} events for $l$-th arm as
\[
    A_{l, s} = \left\{ - C_1 \Delta_k < \overline{R}_{l, s}^* - \frac{\mu_l + \lambda}{1 + 2 \lambda}  < C_1 \Delta_k \right\},
\]
and
\[
    G_{l, s} = \left\{ - C_2 \Delta_k \leq \overline{Y}_{l, s} - \overline{R}_{l, s}^* \leq C_2 \Delta_k \right\}.
\]

The definitions of $A_{l, s}$ and $G_{l, s}$ are intuitive: $A_{l, s}$ represents the sample mean does not derivative from the true mean too far, and events on $G_{l, s}$ means $\overline{Y}_{l, s}$ is not too far away from the scaled-shifted sample mean $\overline{R}_{l, s}^*$. Here $C_1, C_2 \in (0, 1)$ are some constants.

Therefore, by using these good events, we decompose $a_{k, s}$ as the following three parts:
\begin{equation}\label{def_a_k_s_1}
    a_{k, s, 1} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}^c)\Big],
\end{equation}
\begin{equation}\label{def_a_k_s_2}
    a_{k, s, 2} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s}^c)\Big],
\end{equation}
and
\begin{equation}\label{def_a_k_s_3}
    a_{k, s, 3} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s})\Big].
\end{equation}
Let $C_1 = \frac{1}{6 \lambda}$ and $C_2 = \frac{1}{12 \lambda}$ with fixed $\lambda > 1$, then $C_1, C_2 \in (0, 1)$. Denote
\[
    \mathfrak{a}_1 = \frac{192 \sigma_{\omega}^2 \lambda^4}{3 (1 + 2\lambda)^2 \Delta_k^2} , \qquad \mathfrak{b}_1 = \frac{2 \sigma_{\omega}^2 \left[ 96 \lambda^4 \sigma_1^2 + (1 + 2 \lambda)^2 C_2^2 \Delta_k^2 \right]}{3(1 + 2 \lambda)^2 \Delta_k^2},
\]
\[
    \mathfrak{a}_2 = \frac{36 \sigma_{\omega}^2 \lambda^4}{3 (\lambda - 1)^2 \Delta_k^2}, \qquad \mathfrak{b}_2 = \frac{\sigma_{\omega}^2\left[ 72 \lambda^4 \sigma_1^2 + 25 (1 + 2\lambda)^2 \Delta_k^2 \right]}{6 (\lambda - 1)^2  \Delta_k^2}, \qquad \Lambda_k = 8 \sqrt{2} \sigma_k^2.
\]
Consider the case $T \geq 2$. Define 
\[
    s_{a, j}(T) = \max \{ s: a_{k, s, j} \geq T^{-1}\}, \qquad j = 1, 2, 3.
\]
Lemma \ref{lem_bounding_a_k_s_1} in Appendix \ref{sec_lem_components} proves that by taking
\[
    s \geq s_{a, 1} (T) = \frac{144 \lambda^2 \sigma^2_1}{(1 + 2\lambda)^2 \Delta_k^2} \log T,
\]
we will have $a_{k, s, 1} \leq T^{-1}$.
Similarly, Lemma \ref{lem_bounding_a_k_s_2} and Lemma \ref{lem_bounding_a_k_s_3} in Appendix \ref{sec_lem_components} say that: if we take
\[
    \begin{aligned}
       s \geq s_{a, 2}(T) = \Bigg[ \big[ \Lambda_1 + \mathfrak{a}_1 + \mathfrak{b}_1 + \Lambda_1 \mathfrak{a}_1 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_1}}{2} + \frac{\sqrt{2 \pi} \Lambda_1 \mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right]  \right\}  + 1\right)\\
        &  \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T
    \end{aligned}
\]
and
\[
    \begin{aligned}
        s \geq s_{a, 3} (T) =  \Bigg[ \big[ \Lambda_1 + \mathfrak{a}_2 + \mathfrak{b}_2 + \Lambda_1 \mathfrak{a}_2 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 +\frac{\sqrt{\pi \mathfrak{b}_1}}{2}  + \frac{\sqrt{2 \pi} \Lambda_1 \mathfrak{a}_2}{2 \mathfrak{b}_2^2} + \frac{2 \mathfrak{a}_2}{\mathfrak{b}_2} \right]  \right\}  + 1\right) \\
        & \vee \frac{18 \sigma_{\omega}^2 (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T,
    \end{aligned}
\]
then $a_{k, s, 2} \leq T^{-1}$ and $a_{k, s, 3} \leq T^{-1}$, respectively.
Denote $\Lambda_{\max} = \max_{k \in [K]} \Lambda_k$ and $\Lambda_{\min} = \min_{k \in [K]} \Lambda_k$, then for any 
\[
    \begin{aligned}
        s & \geq s_{a} (T) = \frac{144 \lambda^2 \sigma_1^2}{(1 + 2 \lambda)^2 \Delta_k^2} \log T +  \Bigg[ \big[\Lambda_{\max} + ( \mathfrak{a}_1 + \mathfrak{a}_2) + (\mathfrak{b}_1 + \mathfrak{b}_2) + \Lambda_{\max}  (\mathfrak{a}_1 + \mathfrak{a}_2) \big]  \\
        & \qquad \qquad \qquad \qquad \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi}}{2} \left( {\sqrt{\mathfrak{b}_1}} + \sqrt{\mathfrak{b}_2} \right) + \frac{\sqrt{2 \pi} \Lambda_{\max}  }{2}  \left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1^2} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2^2} \right) + 2\left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2} \right) \right]  \right\}  + 1\right) \\
        &  \qquad \qquad \qquad \qquad  \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T,
    \end{aligned}
\]
we have $s \geq \max_{j = 1, 2, 3} s_{a, j} (T)$ due to that
\[
    s_a(T) = s_{a, 1} (T) + \max_{j =2 , 3} s_{a, j} (T)  \geq \max_{j = 1, 2, 3} s_{a, j} (T).
\]
Hence, for any $s \geq s_{a} (T)$, we have 
\[
    a_{k, s} = a_{k, s, 1} + a_{k, s, 2} + a_{k, s, 3} \leq 3 T^{-1}.
\]
Finally, Lemma \ref{lem_bounding_a_k_s_constant} in Appendix \ref{sec_lem_components} ensures if we take $\lambda \geq \frac{4 \sigma_1}{\sigma_{\omega}}  + \sqrt{\frac{4\sigma_1}{\sigma_{\omega}} \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1\right)}$, the component $a_{k, s} \leq 2 + 4 e^{9 / 8}$ for any $s \geq 0$. Therefore, by setting $\lambda \geq \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1 \right) + \sqrt{\frac{4\sigma_1}{\sigma_{\omega}} \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1\right)}$, we have 
\[
    \begin{aligned}
    	a_{k} & = \sum_{s = 0}^{T - 1} a_{k, s}\\
            & \leq \sum_{s < s_{a} (T)} \max_{s \in \{ 0,1, \ldots, T - 1\}} a_{k, s} + \sum_{s_{a} (T) \leq s < T } 3 T^{-1} \\
            & = \max_{s \in \{ 0,1, \ldots, T - 1\}} a_{k, s} \times s_{a} (T) + 3 T^{-1} \times \big(T -  s_{a} (T) \big) \\
    
            & \leq 2 (1 + 2 e^{9 / 8}) \Bigg\{ \frac{144 \lambda^2 \sigma_1^2}{(1 + 2 \lambda)^2 \Delta_k^2} \log T +   \bigg[ \big[ \Lambda_{\max} + ( \mathfrak{a}_1 + \mathfrak{a}_2) + (\mathfrak{b}_1 + \mathfrak{b}_2) + \Lambda_{\max}  (\mathfrak{a}_1 + \mathfrak{a}_2) \big]  \\
        & \qquad \qquad \qquad \qquad \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi}}{2} \left( {\sqrt{\mathfrak{b}_1}} + \sqrt{\mathfrak{b}_2} \right) + \frac{\sqrt{2 \pi} \Lambda_{\max} }{2}  \left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1^2} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2^2} \right) + 2\left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2} \right) \right]  \right\}  + 1\right) \\
        &  \qquad \qquad \qquad \qquad  \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2 }{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \bigg]\Bigg\} \log T + 3
    \end{aligned}
\]
for any $T \geq 2$.
   
       
       
   

\textbf{Step 2: Bounding $b_k$.} 

Again, we will provide a roadmap 
for proving $a_k$ is bounded by a term of $O(\log T)$ order. Similar to Step 1, we set a fixed level $\tau_k$ such that $\tau_k > \frac{\mu_k + \lambda}{1 + 2\lambda}$, then the theory of large deviation gives
\[
    \lim_{s \rightarrow \infty} Q_{k, s} (\tau_k) = 0.
\]
Thus, given the time horizon $T$, there exists a constant $s_0(T)$ such that $Q_{k, s}(\tau_k) \leq T^{-1}$ for all $s$ over $s_0(T)$. As a result, the event $\left\{Q_{k, s} (\tau_k ) > T^{-1}\right\}$ is empty if the number of pulls $s$ is beyond $s_0(T)$. Consequently, 
\[
    b_k \leq \sum_{s=0}^{s_0(T)} \mathbb{P} \left(Q_{k, s}\left(\tau_k\right)>T^{-1}\right).
\]
The fact that the constant $s_0(T)$ is of $O(\log T)$ order will be shown in Lemma \ref{lem_bounding_b_k_s_1}, Lemma \ref{lem_bounding_b_k_s_2}, and Lemma \ref{lem_bounding_b_k_s_3}. For small number of pull $s \leq s_0(T)$, we apply a trivial bound $\mathbb{P} \left(Q_{k, s}\left(\tau_k\right)>T^{-1}\right) \leq 1$ that holds for any $s$. 
Therefore, it is sufficient to conclude that $b_k$ can be bounded by a term of $O(\log T)$ order.

Note that $b_{k, s}$ is naturally bounded by {the} constant $1$. Similar to Step 2, decompose $b_{k, s} = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1}\big)$ as $b_k = b_{k, s, 1} + b_{k, s, 2} + b_{k, s, 3}$ with
\begin{equation}\label{def_b_k_s_1}
    b_{k, s, 1} = {\mathbb{E}} \big[ \mathbb{I} ( Q_{k, s} (\tau_k) > T^{-1}) \mathbb{I} (A_{k, s}^{c} )\big],
\end{equation}
\begin{equation}\label{def_b_k_s_2}
    b_{k, s, 2} = {\mathbb{E}} \big[ \mathbb{I} ( Q_{k, s} (\tau_k) > T^{-1}) \mathbb{I} (A_{k, s}) \mathbb{I} (G_{k, s}^c)\big],
\end{equation}
and
\begin{equation}\label{def_b_k_s_3}
    b_{k, s, 3} = {\mathbb{E}} \big[ \mathbb{I} ( Q_{k, s} (\tau_k) > T^{-1}) \mathbb{I} (A_{k, s}) \mathbb{I} (G_{k, s})\big],
\end{equation}
Again, define $s_{b, j}: = \max \{ s: b_{k, s, j} \geq T^{-1}\}$ for $j = 1, 2, 3$, then Lemma \ref{lem_bounding_b_k_s_1} in Appendix \ref{sec_lem_components} guarantees that take $s \geq s_{b, 1} (T)$ with
\[
    s_{b, 1} (T) = \frac{72 \lambda^2 \sigma^2_k}{(1 + 2\lambda)^2 \Delta_k^2} \log T,
\]
then $b_{k, s, 1} \leq T^{-1}$. Consider $T \geq 2$ and let $C_1 = \frac{1}{6 \lambda}$ and $C_2 = \frac{1}{12 \lambda}$ with fixed $\lambda > 1$ as in Step 1, 
Lemma \ref{lem_bounding_b_k_s_2} and Lemma \ref{lem_bounding_b_k_s_3} in Appendix \ref{sec_lem_components} proves that:  if we take
\[
    \begin{aligned}
        s \geq s_{b, 2}(T) = \Bigg[ \big[ \Lambda_k + \mathfrak{a}_1 + \mathfrak{b}_1 + \Lambda_k \mathfrak{a}_1 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\mathfrak{b}_1}{4 \Lambda_k \mathfrak{a}_1} + \frac{\sqrt{2 \pi} \Lambda_k \mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right]  \right\}  + 1\right)\\
        &  \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T
    \end{aligned}
\]
and
\[
    \begin{aligned}
        s \geq s_{b, 3} (T) =  \Bigg[ \big[ \Lambda_k + \mathfrak{a}_2 + \mathfrak{b}_2 + \Lambda_k \mathfrak{a}_2 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\mathfrak{b}_2}{4 \Lambda_k \mathfrak{a}_2} + \frac{\sqrt{2 \pi} \Lambda_k \mathfrak{a}_2}{2 \mathfrak{b}_2^2} + \frac{2 \mathfrak{a}_2}{\mathfrak{b}_2} \right]  \right\}  + 1\right) \\
        & \vee \frac{18 \sigma_{\omega}^2 (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T,
    \end{aligned}
\]
then we have $b_{k, s, 2} \leq T^{-1}$ and $b_{k, s, 3} \leq T^{-1}$, respectively. Let 
\[
    \begin{aligned}
        s_{b}(T) & := \frac{72 \lambda^2 \sigma_k^2}{(1 + 2 \lambda)^2 \Delta_k^2} \log T  +  \Bigg[ \big[ \Lambda_{\max} + ( \mathfrak{a}_1 + \mathfrak{a}_2) + (\mathfrak{b}_1 + \mathfrak{b}_2) + \Lambda_{\max} (\mathfrak{a}_1 + \mathfrak{a}_2) \big]  \\
        & \qquad \qquad  \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi}}{2} \left( {\sqrt{\mathfrak{b}_1}} + \sqrt{\mathfrak{b}_2} \right) + \frac{\sqrt{2 \pi} \Lambda_{\max} }{2}  \left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1^2} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2^2} \right) + 2\left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2} \right) \right]  \right\}  + 1\right) \\
        &  \qquad \qquad \qquad \qquad  \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T,
    \end{aligned}
\]
then for any $s \geq s_b(T)$, we have $s \geq \max_{j = 1, 2, 3} s_{b, j} (T)$ due to $s_b(T) = s_{b, 1} (T) + \max_{j = 2, 3} s_{b, j} (T)$. Thus, $b_{k, s, 1} + b_{k, s, 2} + b_{k, s, 3} \leq 3 T^{-1}$ for any $s \geq s_b(T)$.
Note that $b_{k, s} = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1}\big) \leq 1$ for any $s \geq 0$, then 
\[
    \begin{aligned}
        b_{k} & = 1 + \sum_{s = 0}^{T - 1} b_{k, s, 1}\\
        & \leq 1 + 1 \times s_{b}(T) + 3 T^{-1} \times \big(T - s_{b}(T) \big) \\
        & \leq \Bigg\{ \frac{72 \lambda^2 \sigma_k^2}{(1 + 2 \lambda)^2 \Delta_k^2} + \bigg[ \big[ \Lambda_{\max} + ( \mathfrak{a}_1 + \mathfrak{a}_2) + (\mathfrak{b}_1 + \mathfrak{b}_2) + \Lambda_{\max} (\mathfrak{a}_1 + \mathfrak{a}_2) \big]  \\
        & \qquad \qquad  \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi}}{2} \left( {\sqrt{\mathfrak{b}_1}} + \sqrt{\mathfrak{b}_2} \right) + \frac{\sqrt{2 \pi} \Lambda_{\max} }{2}  \left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1^2} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2^2} \right) + 2\left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2} \right) \right]  \right\}  + 1\right) \\
        &  \qquad \qquad \qquad \qquad  \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \bigg] \Bigg\} \log T + 4
    \end{aligned}
\]
for any $T \geq 2$.

\textbf{Step 3: Aggregating results.}

Denote 
\[
    d_1 = \Delta_k^2 \big\{ ( \mathfrak{a}_1 + \mathfrak{a}_2) + (\mathfrak{b}_1 + \mathfrak{b}_2) + \Lambda_{\max}  (\mathfrak{a}_1 + \mathfrak{a}_2) \big\}
\]
and
\[
    d_2 = 3 \left[  1 + \frac{\sqrt{\pi}}{2} \left( {\sqrt{\mathfrak{b}_1}} + \sqrt{\mathfrak{b}_2} \right) + \frac{\sqrt{2 \pi} \Lambda_{\max} }{2}  \left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1^2} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2^2} \right) + 2\left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2} \right) \right],
\]
then the bounds in Step 1 and Step 2 imply
\[
    \begin{aligned}
        a_k  \leq 2 (1 + 2 e^{9 / 8}) & \Bigg\{ \frac{144 \lambda \sigma_1^2}{(1 + 2 \lambda)^2 \Delta_k^2} \\
        & + \bigg[ ( \Lambda_{\max} + d_1) \bigg( \frac{1}{3}\frac{\log  d_2}{\log 2} + 1 \bigg) \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg]  \Bigg\} \log T + 3.
    \end{aligned}
\]
and
\[
    \begin{aligned}
        b_k & \leq \Bigg\{ \frac{72 \lambda^2 \sigma_k^2}{(1 + 2 \lambda)^2 \Delta_k^2} \\
        & \qquad \qquad \qquad + \bigg[ ( \Lambda_{\max} + d_1) \bigg( \frac{1}{3}\frac{\log d_2}{\log 2} + 1 \bigg) \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg]  \Bigg\} \log T + 4
    \end{aligned}
\]
for any $T \geq 2$. Therefore, by applying Lemma \ref{lem.a.b}, we have 
\begin{equation}\label{reg_T_med}
    \begin{aligned}
        \operatorname{Reg}_T & \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big) \\
        &\leq \sum_{k = 2}^K \Delta_k \bigg[ 7 + \left\{ c_1(\mu_1, \sigma_1, \mu_k, \sigma_k, \lambda, \sigma_{\omega}) + c_2(\mu_1, \sigma_1, \mu_k, \sigma_k, \lambda, \sigma_{\omega})\Delta_k^{-2} \right\} \log T \bigg]
    \end{aligned}
\end{equation}
for any $T \geq 2$, where
\[
    c_1(\mu_1, \sigma_1, \mu_k, \sigma_k, \lambda) = (3 + 2 e^{9 / 8}) \bigg[ \bigg( \Lambda_{\max} \left( \frac{\log d_2}{3 \log 2} + 1\right) \bigg) \vee \frac{18 \sigma_{\omega}^2 \{ (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2 \}}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2}  \bigg],
\]
and
\[
    \begin{aligned}
        c_2(\mu_1, \sigma_1, \mu_k, \sigma_k, \lambda) & = \frac{72 \lambda^2 \sigma_k^2}{(1 + 2 \lambda)^2 } ( 5 + 4 e^{9 / 8}) \\
        & \qquad + (3 + 2 e^{9 / 8}) \bigg[ \bigg( d_1 \left( \frac{\log d_2}{3 \log 2} + 1\right) \bigg) \vee \frac{18 \sigma_{\omega}^2 \{ (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2 \} }{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2}  \bigg],
    \end{aligned}
\]
for $k = 2, \ldots, K$. When the total round $T = 1$, the bound \eqref{reg_T_med} still holds because $\operatorname{Reg}_T \leq \max_{k = 2, \ldots K} \Delta_k \leq 7 \sum_{k = 2}^K \Delta_k$. 
\vspace{2ex}

\textbf{Step 4: Simplifying bounds.}

Note that
\[
    \begin{aligned}
        d_1 = & \Delta_k^2 \big\{ ( \mathfrak{a}_1 + \mathfrak{a}_2) + (\mathfrak{b}_1 + \mathfrak{b}_2) + \Lambda_{\max}  (\mathfrak{a}_1 + \mathfrak{a}_2) \big\} \\
        \alignedoverset{\text{by }\Delta_k \leq 1}{\leq} \frac{1 + \Lambda_{\max}}{3} \sigma_{\omega}^2 \left[ \frac{192 \lambda^4}{(1 + 2 \lambda)^2} + \frac{36 \lambda^4}{(\lambda - 1)^2}\right]  \\
        & \qquad \qquad \qquad + \sigma_{\omega}^2 \left[ \frac{2 \left[96 \lambda^4 \sigma_1^2 + (1 + 2\lambda)^2 / (36 \lambda^2) \right]}{3 (1 + 2\lambda)^2} + \frac{72 \lambda^4 \sigma_1^2 + 25 (1 + 2\lambda)^2}{6 (\lambda - 1)^2}\right].
    \end{aligned}
\]
By $\lambda \geq \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1 \right) + \sqrt{\frac{4\sigma_1}{\sigma_{\omega}} \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1\right)}$, we have $(\lambda - 1)^2 \geq 16 \sigma_1^2 / \sigma_{\omega}^2$, and hence
\[
    \frac{192 \lambda^4}{(1 + 2 \lambda)^2} + \frac{36 \lambda^4}{(\lambda - 1)^2} \leq \frac{192 \lambda^4}{4 \lambda^2} + \frac{36 \lambda^4}{16 \sigma_1^2 / \sigma_{\omega}^2 } = 48 \lambda^2 + 3 \frac{\sigma_{\omega}^2}{\sigma_1^2}  \lambda^4.
\]
Similarly,
\[
     \begin{aligned}
         & \frac{2 \left[96 \lambda^4 \sigma_1^2 + (1 + 2\lambda)^2 / (36 \lambda^2) \right]}{3 (1 + 2\lambda)^2} + \frac{72 \lambda^4 \sigma_1^2 + 25 (1 + 2\lambda)^2}{6 (\lambda - 1)^2} \\
         \leq & \frac{2 \times 96 \lambda^4 \sigma_1^2}{3 \times 4 \lambda^2} + \frac{2 }{3 \times 36 \lambda^2} + \frac{72 \lambda^4 \sigma_1^2 + 25 (1 + 2\lambda)^2}{6 \times 16 \sigma_1^2 / \sigma_{\omega}^2 } \\
         \leq & 16 \lambda^2 \sigma_1^2 + \frac{1}{48} + \lambda^4 \sigma_{\omega}^2 + \frac{25 \times 9 \lambda^2}{6 \times 16 \sigma_1^2 / \sigma_{\omega}^2} \leq 16 \lambda^2 \sigma_1^2 + \frac{1}{48} + 3 \lambda^4 \sigma_{\omega}^2 + 3 \lambda^2 \sigma_{\omega}^2 / \sigma_1^2.
     \end{aligned}
\]
Thus, 
\begin{equation}\label{final_bound_1}
    \begin{aligned}
        d_1 = & \Delta_k^2 \big\{ ( \mathfrak{a}_1 + \mathfrak{a}_2) + (\mathfrak{b}_1 + \mathfrak{b}_2) + \Lambda_{\max}  (\mathfrak{a}_1 + \mathfrak{a}_2) \big\} \\
        \leq & \frac{(1 + \Lambda_{\max}) \sigma_{\omega}^2}{3} \left[ 48 \lambda^2 + 3 \frac{\sigma_{\omega}^2}{\sigma_1^2}  \lambda^4 \right] + \sigma_{\omega}^2 \left[ 16 \lambda^2 \sigma_1^2 + \frac{1}{48} + 3 \lambda^4 \sigma_{\omega}^2 + 3 \frac{ \sigma_{\omega}^2}{\sigma_1^2} \lambda^2 \right] \\
        = & \sigma_{\omega}^2 \bigg\{ \bigg[ 16 (1 + \Lambda_{\max}) + 16 \sigma_1^4 + 3 \frac{\sigma_{\omega}^2}{\sigma_1^2}  \bigg] \lambda^2 + \bigg[ (1 + \Lambda_{\max}) \frac{\sigma_{\omega}^2}{\sigma_1^2}  + 3 \sigma_{\omega}^2\bigg] \lambda^4 +\frac{1}{48} \bigg\}  \\
        \leq & \sigma_{\omega}^2 \bigg[ 16 (1 + \Lambda_{\max}) + 16 \sigma_1^4 + 3 \frac{\sigma_{\omega}^2}{\sigma_1^2}  + (1 + \Lambda_{\max}) \frac{\sigma_{\omega}^2}{\sigma_1^2}  + 3 \sigma_{\omega}^2 + 1 \bigg] \lambda^4 \\
        \leq &  \bigg[ (1 + \Lambda_{\max}) \bigg( 16 + \frac{\sigma_{\omega}^2}{\sigma_1^2} \bigg) + 16 \sigma_1^4 + 3 \frac{\sigma_{\omega}^2}{\sigma_1^2} + 3 \sigma_{\omega}^2 + 1 \bigg] \sigma_{\omega}^2 \lambda^4.
    \end{aligned}
\end{equation}
Define
\[
    D_1(\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega}) :=  \bigg[ (1 + \Lambda_{\max}) \bigg( 16 + \frac{\sigma_{\omega}^2}{\sigma_1^2} \bigg) + 16 \sigma_1^4 + 3 \frac{\sigma_{\omega}^2}{\sigma_1^2} + 3 \sigma_{\omega}^2 + 1 \bigg] \sigma_{\omega}^2 \lambda^4,
\]
then $d_1 \leq D_1(\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega})$.
   
       
       
   
On the other hand, for $d_2$, we have 
\[
    \begin{aligned}
        d_2 = & 3 \left[  1 + \frac{\sqrt{\pi}}{2} \left( {\sqrt{\mathfrak{b}_1}} + \sqrt{\mathfrak{b}_2} \right) + \frac{\sqrt{2 \pi} \Lambda_{\max} }{2}  \left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1^2} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2^2} \right) + 2\left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2} \right) \right] \\
        \leq & 3 \Bigg[ 1 +  \frac{\sqrt{\pi}}{2} \left( \sqrt{\frac{2 \sigma_{\omega}^2 (1 + 2 \lambda)^2  \Delta_k^2 / (144 \lambda^2) }{3 (1 + 2 \lambda)^2 \Delta_k^2}} + \sqrt{\frac{25 \sigma_{\omega}^2 (1 + 2\lambda)^2 \Delta_k^2}{6 (\lambda - 1)^2 \Delta_k^2}} \right)\\
        & \qquad \quad  +  \frac{\sqrt{2 \pi} \Lambda_{\max} }{2} \left( \frac{(1 + 2\lambda)^2}{64 \lambda^4 \sigma_1^4} + \frac{(\lambda - 1)^2}{12 \lambda^4 \sigma_1^2 \sigma_{\omega}^2} \right) \\
        & \qquad \quad + 2 \left( \frac{96 \lambda^4}{96 \lambda^4 \sigma_1^2 + (1 + 2 \lambda)^2 / (36 \lambda^2)} + \frac{72 \lambda^4}{72 \lambda^4 \sigma_1^2 + 25 (1 + 2\lambda)^2} \right)\ \Bigg].
    \end{aligned}
\]
Similarly, by the fact $\lambda > 1$,
\[
   \begin{aligned}
        \sqrt{\frac{2 \sigma_{\omega}^2 (1 + 2 \lambda)^2 \Delta_k^2 / (144 \lambda^2)}{3 (1 + 2 \lambda)^2 \Delta_k^2}} + \sqrt{\frac{25 \sigma_{\omega}^2 (1 + 2\lambda)^2 \Delta_k^2}{6 (\lambda - 1)^2 \Delta_k^2}} & \leq \frac{\sigma_{\omega}}{12 \lambda} + \frac{5 \sigma_{\omega} (1 + 2 \lambda)}{2(\lambda - 1)} \\
        & \leq \sigma_{\omega} + 3 \frac{\sigma_{\omega}^2}{\sigma_1} (1 + 2\lambda) \leq \frac{3\sigma_{\omega}^2}{\sigma_1} \left[ \frac{\sigma_1}{\sigma_{\omega}} + 3 \lambda \right]
   \end{aligned}
\]
\vspace{1ex}
\[
    \begin{aligned}
        \frac{(1 + 2\lambda)^2}{64 \lambda^4 \sigma_1^4} + \frac{(\lambda - 1)^2}{12 \lambda^4 \sigma_1^2 \sigma_{\omega}^2} & \leq \frac{9 \lambda^2}{64 \lambda^4 \sigma_1^4} + \frac{ 16 \sigma_1^2 / \sigma_{\omega}^2}{12 \lambda^4 \sigma_1^2 \sigma_{\omega}^2} \\
        & \leq  \frac{9 \lambda^2}{64 \lambda^4 \sigma_1^4} + \frac{ 16 \sigma_1^2 / \sigma_{\omega}^2}{12 \lambda^2  \sigma_1^2 \sigma_{\omega}^2} \\
        & \leq 2 \left[ \frac{1}{\sigma_1^2 \lambda^2} + \frac{1}{\sigma_{\omega}^2 \lambda^2}\right] \leq  2 \left[ \frac{\sigma_{\omega}^2}{16 \sigma_1^4} + \frac{1}{\sigma_{\omega}^2}\right],
    \end{aligned}
\]
and 
\[
    \begin{aligned}
        & \frac{96 \lambda^4}{96 \lambda^4 \sigma_1^2 + (1 + 2 \lambda)^2 / (36 \lambda^2)} + \frac{72 \lambda^4}{72 \lambda^4 \sigma_1^2 + 25 (1 + 2\lambda)^2} \\
        \leq &  \frac{96 \lambda^4}{96 \lambda^4 \sigma_1^2} + \frac{72 \lambda^4}{72 \lambda^4 \sigma_1^2} = \frac{2 \lambda^2}{\lambda^2 \sigma_1^2} \leq \frac{2 \lambda^2}{16 \sigma_1^4 / \sigma_{\omega}^2} = \frac{\lambda^2 \sigma_{\omega}^2}{8 \sigma_1^4}.
    \end{aligned}
\]
Thus,
\begin{equation}\label{final_bound_2}
    \begin{aligned}
        d_2 & = 3 \left[  1 +  \frac{\sqrt{\pi}}{2} \left( {\sqrt{\mathfrak{b}_1}} + \sqrt{\mathfrak{b}_2} \right) + \frac{\sqrt{2 \pi} \Lambda_{\max} }{2}  \left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1^2} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2^2} \right) + 2\left( \frac{\mathfrak{a}_1}{\mathfrak{b}_1} + \frac{\mathfrak{a}_2}{\mathfrak{b}_2} \right) \right] \\
        & \leq 3 \bigg[ 1 + \frac{3 \sqrt{\pi}\sigma_{\omega}^2}{2 \sigma_1} \left( \frac{\sigma_1}{\sigma_{\omega}} + 3 \lambda \right) + \sqrt{2 \pi} \Lambda_{\max} \left( \frac{\sigma_{\omega}^2}{16 \sigma_1^4} + \frac{1}{\sigma_{\omega}^2} \right) + \frac{\lambda^2 \sigma_{\omega}^2}{4 \sigma_1^4} \bigg].
    \end{aligned}
\end{equation}
Again, we define 
\[
    D_2 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega}) = 3 \bigg[ 1 +  \frac{3 \sqrt{\pi}\sigma_{\omega}^2}{2 \sigma_1} \left( \frac{\sigma_1}{\sigma_{\omega}} + 3 \lambda \right) + \sqrt{2 \pi} \Lambda_{\max} \left( \frac{\sigma_{\omega}^2}{16 \sigma_1^4} + \frac{1}{\sigma_{\omega}^2} \right) + \frac{\lambda^2 \sigma_{\omega}^2}{4 \sigma_1^4} \bigg],
\]
then $d_2 \leq D_2 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega})$.

The next step is giving a simply bounds for $c_1(\mu_1, \sigma_1, \mu_k, \sigma_k, \lambda, \sigma_{\omega})$ and $c_2(\mu_1, \sigma_1, \mu_k, \sigma_k, \lambda, \sigma_{\omega})$. We use the fact $a \vee b \leq a + b$ and $a \vee b \vee c = [ (a \vee b) \vee c ]$, then obtain that
\[
    \begin{aligned}
         & \bigg( \Lambda_{\max} \left( \frac{\log d_2}{3 \log 2} + 1\right) \bigg) \vee \frac{18 \sigma_{\omega}^2 \{ (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2 \}}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2}  \\
        \leq & \bigg( \Lambda_{\max} \left( \frac{\log d_2}{3 \log 2} + 1\right) \bigg) \vee \frac{72 \sigma_{\omega}^2 \{ (2 \mu_1 - 1)^2 + (2 \mu_k - 1)^2 \} }{(1 + 2\lambda)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \\
        \leq & \bigg( \Lambda_{\max} \left( \frac{\log d_2}{3 \log 2} + 1\right) \bigg) \vee \frac{72 \sigma_{\omega}^2 \times 2 + 2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \\
        \leq & \bigg( \Lambda_{\max} \left( \frac{\log d_2}{3 \log 2} + 1\right) \bigg) \vee \frac{144 \sigma_{\omega}^2 + 2 \sigma_{\omega}^2 + 4 \lambda^2 \sigma_{\omega}^2}{4 \lambda^2} \\
        \leq & \bigg[ \Lambda_{\max} \left( \frac{\log d_2}{3 \log 2} + 1\right) \bigg] + 38  \sigma_{\omega}^2 \leq \Lambda_{\max} \left( \frac{\log D_2 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega})}{3 \log 2} + 1\right) + 38  \sigma_{\omega}^2.
    \end{aligned}
\]
and 
\[
    \begin{aligned}
        & \bigg( d_1 \left( \frac{\log d_2}{3 \log 2} + 1\right) \bigg) \vee \frac{18 \sigma_{\omega}^2 \{ (2 \mu_1 - 1)^2 \vee (2 \mu_k - 1)^2 \} }{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \\
        \leq & D_1 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega}) \left( \frac{\log D_2 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega})}{3 \log 2} + 1\right) + 38  \sigma_{\omega}^2,
    \end{aligned}
\]
where the last steps in two inequalities above are by \eqref{final_bound_1} and \eqref{final_bound_2}.
Define 
\[
    C_1 (\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) = 10 \bigg[ \Lambda_{\max} \left( \frac{\log D_2 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega})}{3 \log 2} + 1\right) + 38  \sigma_{\omega}^2 \bigg],
\]
and
\[
    C_2 (\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) = 50 \lambda^2 \sigma_k^2 + 10 \bigg[ D_1 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega}) \left( \frac{\log D_2 (\sigma_1, \sigma_{k}, \lambda, \sigma_{\omega})}{3 \log 2} + 1\right) + 38  \sigma_{\omega}^2 \bigg].
\]
Note that 
\[
    3 + 2 e^{9 / 8} \leq 10, \qquad \frac{72 \lambda^2 \sigma_k^2}{(1 + 2 \lambda)^2}  (5 + 4 e^{9 / 8}) \leq \frac{72 \lambda^2 \sigma_k^2}{25}  (5 + 4 e^{9 / 8}) \leq 50 \lambda^2 \sigma_k^2,
\]
we will have $c_1 (\mu_1, \sigma_1, \mu_k, \sigma_k,\lambda, \sigma_{\omega}) \leq C_1 (\sigma_1, \sigma_k, \lambda, \sigma_{\omega})$ and $c_2 (\mu_1, \sigma_1, \mu_k, \sigma_k,\lambda, \sigma_{\omega}) \leq C_2 (\sigma_1, \sigma_k, \lambda, \sigma_{\omega})$.
Finally, by using the bound in \eqref{reg_T_med},  one has
\[
    \begin{aligned}
        \operatorname{Reg}_T & \leq \sum_{k = 2}^K \Delta_k \bigg[ 7 + \left\{ c_1(\mu_1, \sigma_1, \mu_k, \sigma_k, \lambda, \sigma_{\omega}) + c_2(\mu_1, \sigma_1, \mu_k, \sigma_k, \lambda, \sigma_{\omega})\Delta_k^{-2} \right\} \log T \bigg] \\
        & \leq \sum_{k = 2}^K \Delta_k \bigg[ 7 + \left\{ C_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) + C_2(\sigma_1, \sigma_k, \lambda, \sigma_{\omega})\Delta_k^{-2} \right\} \log T \bigg].
    \end{aligned}
\]
For the problem-dependent case. For the problem-independent case, denote
\[
    J_{k, T} := \sum_{t = 1}^T \mathbb{I}(I_t = k) = \sum_{k = 2}^K (a_k + b_k)
\]
then
\begin{equation}\label{pro_ind_tech}
    \begin{aligned}
        \operatorname{Reg}_T & = \sum_{\Delta_k : \Delta_k < \Delta} \Delta_k {\mathbb{E}} J_{k, T} + \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k {\mathbb{E}} J_{k, T} \\
        \alignedoverset{\text{by the bounds of } a_k, b_k}{\leq} T \Delta + \sum_{\Delta_k : \Delta_k \geq \Delta}  \Delta_k \bigg[ 7 + \left\{ C_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) + C_2(\sigma_1, \sigma_k, \lambda, \sigma_{\omega})\Delta_k^{-2} \right\} \log T \bigg] \\
       
        & = T \Delta + 7 \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k + \sum_{\Delta_k : \Delta_k \geq \Delta} C_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega})  \Delta_k \log T + \sum_{\Delta_k : \Delta_k \geq \Delta} \frac{C_2 (\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) }{\Delta_k} \log T \\
        \alignedoverset{\text{by } \Delta_k \leq \mu_1 \leq 1}{\leq} T \Delta + 7 K \mu_1 + \max_{k \in [K] \setminus \{ 1 \}}  C_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) K \log T + \max_{k \in [K] \setminus \{ 1 \}}  C_2 ( \sigma_1, \sigma_k, \lambda, \sigma_{\omega}) \frac{K \log T}{\Delta},
       
       
       
    \end{aligned}
\end{equation}
for any previously specified $\Delta \in (0, 1)$. Take $\Delta =  \sqrt{\max_{k \in [K] \setminus \{ 1 \}}  C_2 (\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) K \log T / T}$, we have 
\[
     \operatorname{Reg}_T \leq 7 K \mu_1 + \max_{k \in [K] \setminus \{ 1 \}}  C_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) K \log T + 2 \sqrt{\max_{k \in [K] \setminus \{ 1 \}}  C_2 (\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) K T \log T}.
\]
Therefore, we finish the proof of Theorem \ref{thm.MAB}.


For proving Theorem \ref{thm.MAB.1}. Set $\sigma_k \equiv 1$ for $k \in [K]$, we have 
\[
    \begin{aligned}
        D_1 (1, 1, \lambda, \sigma_{\omega}) & = \bigg[ \big(1 +  8 \sqrt{2} \big) \big( 16 + \sigma_{\omega}^2 \big) + 16 + 3 \sigma_{\omega}^2 + 3 \sigma_{\omega}^2 + 1 \bigg] \sigma_{\omega}^2 \lambda^4 \\
        & \leq \bigg[ 13 \big( 16 + 3 \sigma_{\omega}^2 \big) + 6 \sigma_{\omega}^2 + 17 \bigg] \sigma_{\omega}^2 \lambda^4 = 45 (3 +  \sigma_{\omega}^2) \lambda^4
    \end{aligned}
\]
and
\[
    \begin{aligned}
        D_2 (1, 1, \lambda, \sigma_{\omega}) &=  3 \bigg[ 1 +  \frac{3 \sqrt{\pi}\sigma_{\omega}^2}{2} \left( \frac{1}{\sigma_{\omega}} + 3 \lambda \right) + 8 \sqrt{ \pi}\left( \frac{\sigma_{\omega}^2}{16} + \frac{1}{\sigma_{\omega}^2} \right) + \frac{\lambda^2 \sigma_{\omega}^2}{4 } \bigg] \\
        & \leq  3 \bigg[ 1 +  \frac{3 \sqrt{\pi}\sigma_{\omega}^2}{2} \left( \frac{1}{\sigma_{\omega}} + 3 \right) + 8\sqrt{\pi}\left( \frac{\sigma_{\omega}^2}{16 } + \frac{1}{\sigma_{\omega}^2} \right) + \frac{\sigma_{\omega}^2}{4} \bigg] \lambda^2 \leq \big( 1 + 15 \sigma_{\omega}^{-2} + 3 \sigma_{\omega} + 10 \sigma_{\omega}^2\big) \lambda^2.
    \end{aligned}
\]
Therefore, we have 
\[
    \begin{aligned}
        C_1 (1, 1, \lambda, \sigma_{\omega}) & = 10 \bigg[ 8 \sqrt{2}  \left( \frac{\log D_2 (1, 1, \lambda, \sigma_{\omega})}{3 \log 2} + 1\right) + 38  \sigma_{\omega}^2 \bigg] \\
        & \leq 10 \bigg[ 8 \sqrt{2}  \left( \frac{\log \big[ ( 1 + 15 \sigma_{\omega}^{-2} + 3 \sigma_{\omega} + 10 \sigma_{\omega}^2)  \lambda^2\big]}{3 \log 2} + 1\right) + 38  \sigma_{\omega}^2 \bigg],
    \end{aligned}
\]
and
\[
    \begin{aligned}
        C_2( 1,1, \lambda, \sigma_{\omega}) & = 50 \lambda^2 + 10 \bigg[ D_1 (1, 1, \lambda, \sigma_{\omega}) \left( \frac{\log D_2 (1, ,1, \lambda, \sigma_{\omega})}{3 \log 2} + 1\right) + 38  \sigma_{\omega}^2 \bigg] \\
        & \leq 50 \lambda^2 + 10 \left[ 45( 3+ \sigma_{\omega}^2) \left( \frac{\log \big[ ( 1 + 15 \sigma_{\omega}^{-2} + 3 \sigma_{\omega} + 10 \sigma_{\omega}^2)  \lambda^2\big]}{3 \log 2} + 1\right) \lambda^4  + 38  \sigma_{\omega}^2 \right].
    \end{aligned}
\]
The above bounds together with the fact that
\[
    C_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) + C_2(\sigma_1, \sigma_k, \lambda, \sigma_{\omega})\Delta_k^{-2} \leq \frac{  C_1(\sigma_1, \sigma_k, \lambda, \sigma_{\omega}) + C_2(\sigma_1, \sigma_k, \lambda, \sigma_{\omega})}{\Delta_k^2}
\]
gives the first result in Theorem \ref{thm.MAB.1}. Finally, by using the same technique as in \eqref{pro_ind_tech}, we obtain the problem-independent regret in we require.
\end{proof}



\section{Proof of Theorem \ref{thm.MAB}}\label{proof_thm}


Before starting our proof, we first give the definition of sub-exponential variables. To be specific, we call a \textit{zero-mean} variable $X$ sub-exponential with parameters $\lambda$ and $\alpha$ if
\[
    {\mathbb{E}} \exp \{ t X \} \leq \exp \left\{ \frac{t^2 \lambda^2}{2}\right\}, \qquad |t| \leq \frac{1}{\alpha} ,
\]
denoted as $X \sim \operatorname{subE}(\lambda, \alpha)$. Specially, we denote $\operatorname{subE}(\lambda) := \operatorname{subE}(\lambda, \lambda)$. 
Similar to the sub-Gaussian variables, we have the following concentration {property}
\[
    \mathbb{P} (X \geq t) \leq \exp \left\{ - \frac{1}{2} \left( \frac{t^2}{\lambda^2} \wedge \frac{t}{\alpha}\right) \right\}, \qquad  {\forall} \, t \geq 0.
\]

   

   



\subsection{Basic Idea}

Our proof relies on the following decomposition of the cumulative regret from \cite{kveton2019garbage}. 
We will recap its proof in Appendix \ref{sec:proof_lemma} for completeness. Let the history of arm $k$ after $s$ pulls be a set $\mathcal{H}_{k, s}$ of cardinality $s$, $Q_{k, s} (\tau) := \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_{k, s} \big)$ be the tail probability that $\overline{Y}_{k, s}$ conditioned on history $\mathcal{H}_{k, s}$ is a least $\tau$ for some tunable parameter $\tau$, and $N_{k, s} (\tau) := 1 / Q_{k, s} (\tau) - 1$.
\begin{lemma}\label{lem.a.b}
Suppose in MAB we select arms according to the rule 
$A_t = \arg \max_{k \in [K]} \overline{Y}_{k, t}$. 
Then for any $\{ \tau_k \}_{k = 2}^K \subseteq \mathbb{R}^{K - 1}$, 
the expected $T$-round regret can be bounded above as
\[
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
\]
where
\[
    a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] =: \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \sum_{s = 0}^{T - 1} \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1 =: \sum_{s = 0}^{T - 1} b_{k, s} + 1.
\]
\end{lemma}
Recall the summation index $s$ is the time we pull the $k$-th arm in the total $T$ round{s}. 
The definition of $a_{k}$ and $b_{k}$ have important meaning, $a_{k}$ represents the expected number of rounds that optimal arm $1$ has been being underestimted, while $b_{k}$ is the probability that the suboptimal arm $k$ is being overestimated.
Here we only need to consider the lower bound of the tail of the distribution of the rewards from the optimal arm. 
The intuition is: (i) we only need the rewards from the optimal arm taking a relatively large value with a probability that is not too small; (ii) we do not care about the tiny probability of a large reward from suboptimal arms. 


In the following steps, we will bound $a_k$ and $b_k$ separately up to order $\log T$. Denote $\mathbb{P}_{\xi}(A)$ is the probability of event $A$ concerning the law of $\xi$ and taking other quantities as deterministic. Recall that we denote two functions $a(s, T) \lesssim b(s, T)$ if $a(s, T) \leq c \times b(s, T)$ for some constant $c$ free of $s$ and $T$. 
And we write $a(s, T) \asymp b(s, T)$ if both $a(s, T) \lesssim b(s, T)$ and $a(s, T) \gtrsim b(s, T)$. 
The proof of all lemmas can be found in Appendix \ref{sec:proof_lemma}. 

\subsection{Bounding $a_k$}

For bounding $a_{k}$ with $C\log T$ for some constant $C > 0$, we will need first prove a weaker statement: $a_{k, s}$ as a function of $T$ can be upper bounded by some constants free of $T$. As we can see later, this is a critical intermediate step for proving $a_{k, s}$ has an order $\log T$.
   

\subsubsection{Bounding $a_{k,s}$ by constant}
Note that if we take
\begin{equation}\label{requirement_0}
    \tau_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda},
\end{equation}
then we will have
\begin{equation}\label{constant_bound}
    \begin{aligned}
    	a_{k, s} & = {\mathbb{E}} \left[ \left( \frac{1}{Q_{1, s}(\tau_k)}  - 1\right) \wedge T \right] \\
    	(\text{by } Q_{1, s}(\cdot) \text{ is decreasing})&\leq {\mathbb{E}} \frac{1}{Q_{1, s}(\tau_k)} \leq {\mathbb{E}} Q_{1, s}^{-1} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right),
    \end{aligned}
\end{equation}
Hence, we need to find a lower bound of
the tail probability $Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right) = \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right)$. And throughout the proof, we will use the choice of \eqref{requirement_0} for $\tau_k$, and we can take advantage of the bound \eqref{constant_bound}. 

For further analyzing \eqref{constant_bound}, we write it as the probability with respect to the weighted random summation of Gaussian random variable $\{ \omega_i, \omega_i', \omega_i''\}$.

   
   
       
           
           
   
   
   
   
       
           
           
       
   
   

Let $x_i := (2 \lambda + 1) R_{1, i} - (\mu_1 + \lambda)$, $y_i := \lambda (\mu_1 - 1 - \lambda) $, $z_i :=  \lambda (\lambda + \mu_1) $ for $i \in [s]$, then
\[
    S_x = \sum_{i = 1}^s x_i = s \big( (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big), \quad S_y = \sum_{i = 1}^s y_i = s \lambda (\mu_1 - 1 - \lambda), \quad S_z = \sum_{i = 1}^s z_i = s \lambda (\lambda + \mu_1),
\]
with $S_x - S_y - S_z = s (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1)$ and
\[
    T_x = \sum_{i = 1}^s x_i^2 = \sum_{i = 1}^s \big[  (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big]^2, \quad T_y = \sum_{i = 1}^s y_i^2 = s \lambda^2 (\mu_1 - 1 - \lambda)^2, \quad T_z = \sum_{i = 1}^s z_i^2 = s \lambda^2 (\lambda + \mu_1)^2,
\]
with $T_x + T_y + T_z = \sum_{i = 1}^s \big[  (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big]^2  + s \big[ \lambda^2 (\mu_1 - 1 - \lambda)^2 + \lambda^2 (\lambda + \mu_1)^2\big]$. Let $Z$ be the standard Gaussian variable. For a standard Gaussian variable $Z$, we have the follow lemma.
\begin{lemma}\label{lemma:gaussian_weight}
	Let $Z$ be standard Gaussian variable, then the tail probability satisfies
	\[
	    \frac{1}{4} \exp (- x^2 ) < \mathbb{P}(Z > x) \leq \frac{1}{2} \exp ( - {x^2} / {2} )
	\]
	for any $x \geq 0$. 
\end{lemma}
Then we write the probability as
\[
    \begin{aligned}
        & Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right) \\
        = & \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \big( (2 \lambda + 1) R_{1, i} - (\mu_1 + \lambda) \big)\omega_i > \lambda (\mu_1 - 1 - \lambda) \sum_{i = 1}^s \omega_i' + \lambda (\lambda + \mu_1) \sum_{i = 1}^s \omega_i'' \right).
    \end{aligned}
\]
Hence, by applying the first inequality in Lemma \ref{lemma:gaussian_weight} and use the fact that $T_x + T_y + T_z \geq T_z \geq s \lambda^4$, one cannot give a valid tuning condition.
\[
    \begin{aligned}
        & Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s x_i \omega_i - \sum_{i = 1}^s y_i \omega_i' -   \sum_{i = 1}^s z_i \omega_i'' > 0 \right) \\
        = & \mathbb{P}_{Z} \left( Z > \frac{- (S_x - S_y - S_z)}{\sigma_{\omega} \sqrt{T_x + T_y + T_z}}\right) \\
        \geq & \frac{1}{2} \times \mathbb{I} \left( \frac{- (S_x - S_y - S_z)}{\sigma_{\omega} \sqrt{T_x + T_y + T_z}} < 0 \right) + \frac{1}{4} \exp \left\{ - \frac{(S_x - S_y - S_z)^2}{\sigma_{\omega}^2 (T_x + T_y + T_z)} \right\} \times  \mathbb{I} \left( \frac{- (S_x - S_y - S_z)}{\sigma_{\omega} \sqrt{T_x + T_y + T_z}} > 0 \right) \\
        \geq & \frac{1}{2} \times \mathbb{I} \left(S_x - S_y - S_z > 0 \right) + \frac{1}{4} \exp \left\{ - \frac{(S_x - S_y - S_z)^2}{\sigma_{\omega}^2 \cdot s \lambda^4} \right\} \times  \mathbb{I} \left( S_x - S_y - S_z \leq 0 \right) \\
        = & \frac{1}{2} \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > 0 \right) + \frac{1}{4} \exp \left\{ - \frac{s (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1 )^2}{\sigma_{\omega}^2\lambda^4}\right\} \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 \leq 0 \right).
    \end{aligned}
\]
where $Z$ again is the standard Gaussian variable and we will use this notation through this proof. Then this result combined with the fact \eqref{constant_bound} gives the upper bound for $a_{k, s}$ as
\[
    \begin{aligned}
    	a_{k, s} & \leq 2 \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 > 0 \right) + 4  {\mathbb{E}} \exp \left\{ \frac{s (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1 )^2}{\sigma_{\omega}^2\lambda^4}\right\} \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 \leq 0 \right) \\
    	& \leq 2 + 4  {\mathbb{E}} \exp \left\{ \frac{s (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1 )^2}{\sigma_{\omega}^2\lambda^4}\right\} 
    \end{aligned}
\]
   
       
       
       
   
\iffalse
\begin{equation}\label{a_k_constant_2}
    \begin{aligned}
        a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times\mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \\
        & ~~~~~~~~~~~~~~ + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \times \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 \leq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right) \\ 
        & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2  \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times\mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  \mathbb{I} \left( \mu_1 > \frac{1}{2} \right) \\
        & ~~~~ + {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right)  \mathbb{I} \left( \mu_1 < \frac{1}{2} \right)  + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \\
        & = : I + II + III, 
       
       
       
       
       
       
       
    \end{aligned}
\end{equation}
where $III = \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}$ is a constant. We will bound $I$ and $II$ as follows. 
\begin{itemize}
	\item Bounding $I$: 
	\[
	    \begin{aligned}
	    	I & = {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2  \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2  }\right\} \times\mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right)  \mathbb{I} \left(  \mu_1 > \frac{1}{2} \right) \\
	    	   \alignedoverset{\overline{R}_{1, s} > \mu_1 > \frac{\mu_1 + \lambda}{1 + 2 \lambda} \text{ when } \mu_1 > \frac{1}{2}}{\leq} {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2  \left(  \mu_1 +\frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 \mathbb{I} \left( \overline{R}_{1, s}  \geq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right) }\right\} \\
	    	  
	    	  
	    	 
	    	 
	    	 
	    	  & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ \times\mathbb{I} \left( \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right)  \mathbb{I} \left(  \mu_1 > \frac{1}{2} \right)\\
	    	  \alignedoverset{\text{by using } \mu_1 > \frac{1}{2}}{\leq} {\mathbb{E}} \exp \left\{ \frac{3 (2 \lambda + 1)^2 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\}  \mathbb{I} \left( \overline{R}_{1, s }- \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right) \mathbb{I} \left(  \mu_1 > \frac{1}{2} \right).
	    \end{aligned}
	\]
	
	\item Bounding $II$: 
	

	


	   
	    
	    
	    
	    
	    
	   

	
	

	We first note under the indicator $\mathbb{I}\left(\overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right)$, we have
	\[
	    \overline{R}_{1, s} > \mu_1 + \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} > \mu_1.
	\]
	The basic idea is the fact that ${\mathbb{E}} X = \int_0^{\infty} \mathbb{P} (X > r) \, \mathrm{d} r$ for any positive $X$. Hence, for any $r > 0$,  we consider set that 
	\[
	    \begin{aligned}
	    	E(r, s) & := \left\{ \overline{R}_{1, s} - \mu_1: \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} > r \right\} \\
	    	& = \left\{  \overline{R}_{1, s} - \mu_1 : \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2} > \log r\right\}.
	    \end{aligned}
	\]
	We need to divide the analysis into two cases: (i) $\overline{R}_{1, s} \geq \frac{\mu_1 + \lambda}{1 + 2 \lambda} $ and (ii)
	$\overline{R}_{1, s} < \frac{\mu_1 + \lambda}{1 + 2 \lambda} $ respectively. We first prove the case (i) since it is easier. Note that 
	\begin{equation}\label{a_k_constant_II_case1}
		\overline{R}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} > \mu_1
	\end{equation}
	when $\mu_1 < \frac{1}{2}$. Hence we can write $E(r, s)$ on condition \eqref{a_k_constant_II_case1} as
	\[
	    \begin{aligned}
	    	E(r, s) \alignedoverset{\text{by \eqref{a_k_constant_II_case1}}}{=} \left\{  \overline{R}_{1, s} - \mu_1 : \frac{\sqrt{3 s} (\overline{R}_{1, s} - \mu_1)}{\sqrt{2} \sigma_{\omega} \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)} > \sqrt{\log r}\right\} \\
	    	& = \left\{  \overline{R}_{1, s} - \mu_1 : \sqrt{2 \log r} \sigma_{\omega} \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} > \left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right) (\overline{R}_{1, s} - \mu_1) \right\} \\
	    	& = \left\{ \begin{array}{ll}
	    		\left\{  \overline{R}_{1, s} - \mu_1 : \sqrt{2 \log r} \sigma_{\omega} \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} > 0 >  \left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right) (\overline{R}_{1, s} - \mu_1) \right\}, & r \leq \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}, \\
	    		\left\{  \overline{R}_{1, s} - \mu_1 : \frac{\sqrt{2 \log r} \sigma_{\omega} \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda}}{\left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right)} > (\overline{R}_{1, s} - \mu_1) \right\}, & r > \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\} 
	    	\end{array}\right. \\
	    	\alignedoverset{\text{by \eqref{a_k_constant_II_case1} again}}{=} \left\{ \begin{array}{ll}
	    		\left\{ \overline{R}_{1, s} - \mu_1 \in \mathbb{R} \right\}, & r \leq \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}, \\
	    		\left\{  \overline{R}_{1, s} - \mu_1 : \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} \frac{\sqrt{2 \log r} \sigma_{\omega}} {\left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right)} > (\overline{R}_{1, s} - \mu_1) \right\}, & r > \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\} 
	    	\end{array} \right. 
	    \end{aligned}
	\]
	Then,
	\[
	    \begin{aligned}
	    	II & = \int_0^{\infty} \mathbb{P} \left( E(r, s) \, \cap \, \left\{ \overline{R}_{1,s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right\} \right) \, \mathrm{d} r \\
	    	& = \left[ \int_0^{ \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}} + \int_{\exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}}^{\infty} \right] \mathbb{P} \left( E(r, s) \, \cap \, \left\{ \overline{R}_{1,s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right\} \right) \, \mathrm{d} r \\
	    	& = II_1 + II_2.
	    \end{aligned} 
	\]
	We first bound $II_2$. Write $II_2$ as
	\[
	    II_2 = \int_{\exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}}^{\infty} \mathbb{P} \left( \frac{\lambda(1 - \mu_1)}{1 + 2 \lambda} < \overline{R}_{1,s} - \mu_1 <  \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} \frac{\sqrt{2 \log r} \sigma_{\omega}} {\left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right)} \right) \, \mathrm{d} r.
	\]
	By letting $s \longrightarrow \infty$,
	\[
	    \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\} \, \uparrow \, \infty, \qquad \left\{ \frac{\lambda(1 - \mu_1)}{1 + 2 \lambda} < \overline{R}_{1,s} - \mu_1 <  \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} \frac{\sqrt{2 \log r} \sigma_{\omega}} {\left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right)} \right\}  \, \downarrow \, \varnothing,
	\]
	hence we have $II_2 \longrightarrow 0$. This implies  there exist $T_1 \in \mathbb{N}$ such that $II_2 \leq 1$ for any $s \geq T_1$. And for $II_1$, we take advantage of sub-Gaussian inequality,
	\[
	    \begin{aligned}
	    	II_1 & = \int_0^{ \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}} \mathbb{P} \left( E(r, s) \mathbb{I} \left\{ \overline{R}_{1,s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right\} \right) \, \mathrm{d} r \\
	    	& \leq \int_0^{ \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}} \mathbb{P} \left( \overline{R}_{1,s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right) \, \mathrm{d} r \\
	    	& \leq \int_0^{ \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}} \mathbb{P} \left( \overline{R}_{1,s} - \mu_1 > \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} \right) \, \mathrm{d} r \\
	    	\alignedoverset{\text{by sub-Gaussian inequality}}{\leq} \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}  \cdot \exp \left\{ - \frac{\lambda^2 (1 - 2 \mu_1)^2 s}{2 (1 + 2 \lambda)^2 \sigma_1^2}\right\} \\
	    	& = \exp \left\{ \left[ \frac{3}{2 \sigma_{\omega}^2} - \frac{\lambda^2 (1 - 2 \mu_1)^2}{2 (1 + 2 \lambda)^2 \sigma_1^2} \right] s \right\}.
	    \end{aligned}
	\]
	Therefore, for making $II_1$ have a upper a constant    free of $s$, a sufficient condition is
	\[
	    \frac{3}{2 \sigma_{\omega}^2} - \frac{\lambda^2 (1 - 2 \mu_1)^2}{2 (1 + 2 \lambda)^2 \sigma_1^2} \geq 0
	\]
	i.e.
	\begin{equation}\label{tuning_case_1}
		\lambda > \frac{\sqrt{3}\sigma_1}{(1 - 2\mu_1) \sigma_{\omega} - 2 \sqrt{3}\sigma_1} \qquad  \text{ with } \qquad \sigma_{\omega} > \frac{2 \sqrt{3} \sigma_1}{1 - 2\mu_1}.
	\end{equation}
	Combining these results, we conclude that if take \eqref{tuning_case_1}, we have $II \leq 2$ for any $s \geq T_1$. Now switching to case (ii). In case (ii), we will have:
	\begin{equation}\label{a_k_constant_II_fact1}
		\frac{\mu_1 + \lambda}{1 + 2 \lambda} - \overline{R}_{1, s} > 0 \qquad \text{ and } \qquad \frac{\sqrt{2 \pi}}{s(2 \lambda + 1)} < \overline{R}_{1, s} - \mu_1 < \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda}.
	\end{equation}
	\[
	    \begin{aligned}
	    	E(r, s) & = \left\{  \overline{R}_{1, s} - \mu_1 : \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2} > \log r\right\} \\
	    	(\text{use \eqref{a_k_constant_II_fact1}}) & \subseteq \left\{  \overline{R}_{1, s} - \mu_1 : \frac{3 s (\overline{R}_{1, s} - \mu_1)^{2 - \epsilon}}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2} > \log r \left[\frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda}\right]^{-\epsilon}\right\} \\
	    	& =  \left\{  \overline{R}_{1, s} - \mu_1 : \left[ \frac{(3 s)^{\frac{1}{2 - \epsilon}} (\overline{R}_{1, s} - \mu_1)}{\log^3 r}\right]^{2 - \epsilon} > \left[ \frac{\sqrt{2} \sigma_{\omega}\left( \frac{1 + 2 \lambda}{\lambda(1 - 2 \mu_1)}\right)^{\frac{\epsilon}{2}}\left( \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \overline{R}_{1, s}\right)}{\log^{\frac{5 - 3\epsilon}{2}} } \right]^2\right\} \\
	    	& =  \left\{  \overline{R}_{1, s} - \mu_1 : \left[ \frac{(3 s)^{\frac{1}{2 - \epsilon}} (\overline{R}_{1, s} - \mu_1)}{\log^3 r}\right]^{1 - \frac{\epsilon}{2}} > \frac{\sqrt{2} \sigma_{\omega}\left( \frac{1 + 2 \lambda}{\lambda(1 - 2 \mu_1)}\right)^{\frac{\epsilon}{2}}\left( \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \overline{R}_{1, s}\right)}{\log^{\frac{5 - 3\epsilon}{2}} } \right\}
	    \end{aligned}
	\]
	For any $0 < \epsilon < 2$. Specially, we take $\epsilon = 1$, and 

	\[
	    F(r, s) := \left\{ \frac{(3 s)^{\frac{1}{2 - \epsilon}} (\overline{R}_{1, s} - \mu_1)}{\log^{3} r} \geq 1 \right\},
	\]
	then
	\[
	    \begin{aligned}
	    	II & = \int_0^{\infty} \mathbb{P} \left\{ E (r, s)  \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right) \right\} \, \mathrm{d} r \\
	    	& \leq \int_0^{\infty} \left[\mathbb{P} \left\{ \big[ E (r, s) \, \cap  \, F(r, s) \big] \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right) \right\} + \mathbb{P} \left\{ \big[ F(r, s)^c \big] \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right)\right\} \right]\, \mathrm{d} r \\
	    	& =: II_1 + II_2.
	    \end{aligned}
	\]
	For $II_2$, we write that
	\[
	    \begin{aligned}
	    	II_2 & \leq \int_0^{\infty} \mathbb{P} \left\{ \big[ F(r, s)^c \big] \, \cap \, \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right)\right\}\, \mathrm{d} r  \\
	    	& = \int_0^{\infty} \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \, \mathrm{d}r \\
	    	& = \left[ \int_0^{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}} + \int_{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}}^{\infty} \right] \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \, \mathrm{d}r \\
	    	& = \int_{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}}^{\infty}  \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right)  \, \mathrm{d}r.
	    \end{aligned}
	\]
	For $II_2$, we write that
	\[
	    \begin{aligned}
	    	II_2 & = \int_0^{\infty} \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \, \mathrm{d}r \\
	    	& = \left[ \int_0^{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}} + \int_{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}}^{\infty} \right] \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \, \mathrm{d}r \\
	    	& = \int_{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}}^{\infty}  \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right)  \, \mathrm{d}r.
	    
	    
	    
	    
	    \end{aligned}
	\]
	By letting $s \longrightarrow \infty$, we have 
	\[
	    \exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\} \longrightarrow \infty, \qquad \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \longrightarrow 0.
	\]
	Thus, $II_2$ shows we must have $II_2 \leq 1$ bounded by a constant free of $s$ for enough big $s$, say $s \geq T_2$.
	And for $II_1$, we take advantage that $x \geq x^{1 - \frac{\epsilon}{2}}$ for $x > 1$ and $0 < 1 - \frac{\epsilon}{2}< 1$, 
	\[
	    \begin{aligned}
	    	& E(r, s) \, \cap \, F(r, s) \\
	    	\subseteq & \left\{  \overline{R}_{1, s} - \mu_1 :  \frac{(3 s)^{\frac{1}{2 - \epsilon}} (\overline{R}_{1, s} - \mu_1)}{\log^3 r} > \frac{\sqrt{2} \sigma_{\omega}\left( \frac{1 + 2 \lambda}{\lambda(1 - 2 \mu_1)}\right)^{\frac{\epsilon}{2}}\left( \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \overline{R}_{1, s}\right)}{\log^{\frac{5 - 3\epsilon}{2}} } \right\} \\
	    	= & \left\{  \overline{R}_{1, s} - \mu_1 : \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r}\right\}.
	    \end{aligned}
	\]
	and in turn, we have
	\[
	    \begin{aligned}
	    	 & \big[ E(r, s) \, \cap \, F(r, s) \big] \mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right\} \\
	    	
	    	 \subseteq & \left\{ \begin{array}{ll}
	    	 	\varnothing, & r \leq \overline{r}_1 (s) \\
	    	 	\left\{ \overline{R}_{1, s} - \mu_1 : \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r}\right\}, & r > \overline{r}_1(s) 
	    	 \end{array}\right..
	    \end{aligned}
	\]
	where
	\[
	    \overline{r}_1 (s) := \exp \left\{ \left[ \frac{\sqrt{2 \pi}}{\left[ \sqrt{2} \lambda (1 - 2 \mu_1) - 2 \sqrt{\pi} / s\right] \sigma_{\omega}}\right]^{\frac{2}{1 + 3 \epsilon}} \left( \frac{\lambda (1 - 2\mu_1)}{1 + 2\lambda} \right)^{\frac{\epsilon}{1 + 3 \epsilon}} (3s)^{\frac{2(\epsilon - 1)}{(2 - \epsilon)(1 + 3 \epsilon)}}  \right\}.
	\]
	Therefore, 
	\[
	    \begin{aligned}
	    	II_1 & = \left[ \int_{0}^{\overline{r}_1(s)} + \int_{\overline{r}_1(s)}^{\infty} \right] \mathbb{P} \left( \big[ E(r, s) \, \cap \, F(r, s) \big] \mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right\} \right) \, \mathrm{d} r \\
	    	& \leq \int_{0}^{\overline{r}_1(s)} \mathbb{P} \left(  \varnothing \right) \, \mathrm{d} r + \int_{\overline{r}_1(s)}^{\infty} \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r} \right) \, \mathrm{d} r \\
	    	& = \int_{\overline{r}_1(s)}^{\infty} \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r} \right) \, \mathrm{d} r.
	    \end{aligned}
	\]
	Then, by sub-Gaussian inequality,
	\[
	    \begin{aligned}
	    	II_1 & \leq \int_{\overline{r}_1(s)}^{\infty} \exp \left( - \frac{s}{2 \sigma_1^2} \left[ \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r} \right]^2\right) \, \mathrm{d} r \\
	    	\alignedoverset{\text{Cauchy inequality}}{\leq}\int_{\overline{r}_1(s)}^{\infty}  \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{3^{\frac{2}{2-\epsilon}} s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r + 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r\\ 
	    	& = \left[ \int_{\overline{r}_1(s)}^{\overline{r}_2(s)} + \int_{\overline{r}_2(s)}^{\infty} \right] \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{3^{\frac{2}{2-\epsilon}} s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r + 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r \\
	    	& =: II_{11} + II_{12},
	    \end{aligned}
	\]
	where
	\[
	    \overline{r}_2(s) := \exp \left\{ \left[ \left( \frac{\lambda(1 - 2\mu)}{1 +2 \lambda} \right)^{\epsilon}\frac{3^{\frac{2}{2 - \epsilon}}}{2 \sigma_1^2}\right]^{\frac{1}{1 + 3 \epsilon}} s^{\frac{2}{(2 - \epsilon)(1 + 3 \epsilon)}}\right\} 
	\]
	is a higher order of $\overline{r}_1(s)$ due to $0 < 2 (\epsilon - 1) < 2$. For $II_{12}$, 
	\[
	    \begin{aligned}
	    	II_{12} & \leq \int_{\overline{r}_2(s)}^{\infty} \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{2 \cdot 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r \\
	    	& = \int_{\overline{r}_2(s)}^{\infty} \exp \left\{ - \frac{s}{8 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2} \right\} \, \mathrm{d} r \, \longrightarrow \, 0
	    \end{aligned}
	\]
	when $s \, \longrightarrow \, \infty$. Thus, there exists a $T_3$ such that, $II_{12} \leq 1$ whenever $s \geq T_3$. Finally, for $II_{11}$, 
	\[
	    \begin{aligned}
	    	II_{11} & \leq \int_{\overline{r}_1(s)}^{{\overline{r}_2(s)}} \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{2 \cdot 3^{\frac{2}{2-\epsilon}} s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r}\right) \, \mathrm{d} r \\
	    	& = \int_{\overline{r}_1(s)}^{{\overline{r}_2(s)}} \exp \left\{  - \frac{\sigma_{\omega}^2}{4 \cdot 3^{\frac{2}{2-\epsilon}}  \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \log^{1 + 3\epsilon} r s^{\frac{- \epsilon}{2 - \epsilon}} \right\} \\
	    	& \leq \overline{r}_2(s) \exp \left\{  - \frac{\sigma_{\omega}^2}{4 \cdot 3^{\frac{2}{2-\epsilon}}  \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} s^{\frac{- \epsilon}{2 - \epsilon}} \big[ \log \overline{r}_2(s) \big]^{1 + 3\epsilon} \right\} \\
	    	& = \exp \left\{ \left[ \left( \frac{\lambda(1 - 2\mu)}{1 +2 \lambda} \right)^{\epsilon}\frac{3^{\frac{2}{2 - \epsilon}}}{2 \sigma_1^2}\right]^{\frac{1}{1 + 3 \epsilon}} s^{\frac{2}{(2 - \epsilon)(1 + 3 \epsilon)}} - \frac{\sigma_{\omega}^2}{4 \cdot 3^{\frac{2}{2-\epsilon}}  \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} s^{\frac{- \epsilon}{2 - \epsilon}} \big[ \log \overline{r}_2(s) \big]^{1 + 3\epsilon}  \right\} \\
	    	& = \exp \left\{C_1 \cdot s^{\frac{2}{(2 - \epsilon)(1 + 3 \epsilon)}} - C_2 \cdot s \right\} \, \longrightarrow \, 0
	    \end{aligned}
	\]
	when $s \, \longrightarrow \, \infty$, where $C_1$ and $C_2$ are some positive constants free of $s$. Again, when $s \, \longrightarrow \, \infty$. Thus, there exists a $T_4$ such that, $II_{11} \leq 1$ whenever $s \geq T_4$.
	\iffalse
	\red{
	Therefore, by sub-Gaussian inequality,
	\[
	    \begin{aligned}
	    	& \mathbb{P} \left( E(r, s) \, \cap \, F(r, s)  \, \cap \,\left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right\} \right)\\
	    	& = \mathbb{P} \\
	    	& \leq 1 + \int_1^{\infty} \exp \left( - \frac{s}{2 \sigma_1^2} \left[ \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r}\right]^2 \right) \, \mathrm{d}r \\
	    \alignedoverset{\text{Cauchy inequality}}{\leq} 1 + \int_1^{\infty} \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{3 s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r + 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r\\
	    	& = 1 + \left[ \int_1^{\overline{r}(s)} +\int_{\overline{r}(s)}^{\infty}\right] \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{3 s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r + 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r \\
	    	& = : 1 + II_{21} + II_{22}:
	    \end{aligned}
	\]
	where $\overline{r}(s)$ is defined as 
	\[
	     \overline{r}(s) := \exp \left\{ \left[ \left( \frac{\lambda(1 - 2\mu)}{1 +2 \lambda} \right)^{\epsilon}\frac{3 s^{\frac{2}{2 - \epsilon}}}{2 \sigma_1^2}\right]^{\frac{1}{1 + 3 \epsilon}}\right\}.
	\]
	For $r \leq \overline{r}(s)$, we have $3 s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r \geq 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}$, hence
	\[
	    \begin{aligned}
	    	II_{21} & \leq \int_1^{\overline{r}(s)} \exp \left(  - \frac{\sigma_{\omega}^2}{12 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \log^{1 + 3\epsilon} rs^{\frac{2(1 - \epsilon)}{2 - \epsilon}}\right) \, d r \\
	    	& \leq \int_1^{\infty} \exp \left(  - \frac{\sigma_{\omega}^2}{12 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \log^{1 + 3\epsilon} r s^{\frac{2(1 - \epsilon)}{2 - \epsilon}}\right) \, d r \\
	    	& \overset{\text{take } s = 1}{\leq}  \int_1^{\infty} \exp \left(  - \frac{\sigma_{\omega}^2}{6 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \log^{1 } r\right) \, d r \\
	    	& = \int_1^{\infty} r^{-  \frac{\sigma_{\omega}^2}{12 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon}} \, d r
	    \end{aligned}
	\]
	Therefore, for making $II_{21}$ have a upper bound free of $s$, a sufficient condition is 
	\[
		\frac{\sigma_{\omega}^2}{12 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda} \right]^1 > 1
	\]
	i.e.
	\begin{equation}\label{tuning_sigma}
	    \lambda > \frac{12 \sigma_1^2}{(1 - 2\mu_1) \sigma_{\omega}^2 - 24 \sigma_1^2} \qquad \text{ and } \qquad \sigma_{\omega} > \frac{2 \sqrt{6} \sigma_1}{1 - 2\mu_1}
	\end{equation}	
	For $II_{22}$, we have
	\[
	    \begin{aligned}
	    	II_{22} \leq \int_{\overline{r}(s)}^{\infty} \exp \left\{ - \frac{1}{4 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2} s^2\right\} \, dr \longrightarrow 0
	    \end{aligned}
	\]
	when $s \longrightarrow \infty$, {there exist a $T_1$ such that $s \geq T_1$, $II_{22} < 1$.}}
	\fi
\end{itemize}
\fi
From the above expression, for proving $a_{k, s}$ is bounded by a constant free of $s$, it remains to show the expectation
\begin{equation}\label{a_k_constant_3}
	{\mathbb{E}} \exp \left\{ \frac{s (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1 )^2}{\sigma_{\omega}^2\lambda^4}\right\} 
\end{equation}

	

We need the following lemma.

\begin{lemma}\label{lem_subGtosubE}
    Suppose $\{X_i\}_{i = 1}^n$ are i.i.d. zero-mean sub-Gaussian with variance proxy $\sigma^2$, and $\overline{X} = \frac{1}{n} \sum_{i = 1}^n X_i$, then
    \[
        {\mathbb{E}} \exp\big( \lambda\overline{X}^2 \big) \leq e^{9 / 8}
       
    \]
    for any $|\lambda| \leq \frac{n}{8 \sigma^2}$.
\end{lemma}
\iffalse
Lemma \ref{lem_subGtosubE} ensures the expectation in \eqref{a_k_constant_3}
   
    
    
   
\[
    \begin{aligned}
        & {\mathbb{E}} \exp \left\{ \frac{3 (2 \lambda + 1)^2 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} \mathbb{I} \left( \mu_1 > {1} / {2} \right) \\
        = & {\mathbb{E}} \exp \left\{ \frac{3 (2 \lambda + 1)^2 s }{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2} \times(\overline{R}_{1, s} - \mu_1)^2 \right\} \mathbb{I} \left( \mu_1 > {1} / {2} \right)  \leq e^{9/8} \times\mathbb{I} \left( \mu_1 > {1} / {2} \right)
    \end{aligned}
\]
as long as we take
   
\begin{equation}
\label{cond.1}
    \frac{3 s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2} \leq \frac{s}{8 \sigma_1^2}.
\end{equation}
   
  
The condition \eqref{cond.1} is equal to 
   

   
   
\begin{equation}\label{tuning_condition}
    \frac{\sqrt{3} (1 + 2 \lambda)}{\lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s } \leq \frac{\sigma_{\omega}}{2 \sigma_1}.
\end{equation}
Hence it is enough to ask $\lambda (2 \mu_1 - 1) > {2 \sqrt{3} \sigma_1} (1 + 2 \lambda) / \sigma_{\omega}$, i.e.
    \[
        \lambda \left[ 2 \mu_1 - 1 - \frac{4 \sqrt{3} \sigma_1}{\sigma_{\omega}}\right] >  \frac{2 \sqrt{3} \sigma_1}{\sigma_{\omega}}.
	\]
Solving the above inequality for positive $\lambda$ and $\sigma_{\omega}$ and we get that 
	\[
	    \lambda > \frac{2 \sqrt{3} \sigma_1}{(2 \mu_1  - 1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {2 \mu_1 - 1}.
	\]
\iffalse
\begin{itemize}
	\item If $\mu_1 > 1 / 2$, the above inequality is equal to 
	\[
	    \lambda (2 \mu_1 - 1) + \frac{\sqrt{2\pi}}{s} > \frac{2 \sqrt{3} \sigma_1}{\sigma_{\omega}} (1 + 2 \lambda).
	\]
	And hence it is enough to ask $\lambda (2 \mu_1 - 1) > {2 \sqrt{3} \sigma_1} (1 + 2 \lambda) / \sigma_{\omega}$, i.e.
    \[
        \lambda \left[ 2 \mu_1 - 1 - \frac{4 \sqrt{3} \sigma_1}{\sigma_{\omega}}\right] >  \frac{2 \sqrt{3} \sigma_1}{\sigma_{\omega}}.
	\]
	Solving the above inequality for positive $\lambda$ and $\sigma_{\omega}$ and we get that 
	\[
	    \lambda > \frac{2 \sqrt{3} \sigma_1}{(2 \mu_1  - 1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {2 \mu_1 - 1}.
	\]
	\item If $\mu_1 < 1 / 2$, the above inequality is equal to 
	\[
	    \lambda (1 - 2\mu_1) + \frac{\sqrt{2\pi}}{s} > \frac{2 \sqrt{3} \sigma_1}{\sigma_{\omega}} (1 + 2 \lambda).
	\]
	Similar to the case of $\mu_1 > 1 / 2$, the solution is 
	\[
	    \lambda > \frac{2 \sqrt{3} \sigma_1}{(1 - 2 \mu_1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {1 - 2 \mu_1}.
	\]
\end{itemize}
\fi

   
  
  
  
  
   
   
  
  
Hence, in the case $\mu_1 \neq \frac{1}{2}$, we can take the tuning condition as 
\begin{equation}\label{tuning_condition_exact}
    \begin{aligned}
    	& \lambda > \frac{2 \sqrt{3} \sigma_1}{(2 \mu_1  - 1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {2 \mu_1 - 1}, \qquad \text{ for } \qquad \mu_1 > \frac{1}{2} \\
    	& \lambda > \frac{\sqrt{3}\sigma_1}{(1 - 2\mu_1) \sigma_{\omega} - 2 \sqrt{3}\sigma_1} \qquad  \text{ with } \qquad \sigma_{\omega} > \frac{2 \sqrt{3} \sigma_1}{1 - 2\mu_1}, \qquad \text{ for } \qquad \mu_1 < \frac{1}{2}.
    \end{aligned}
\end{equation}	

\iffalse
\begin{equation}\label{tuning_condition_exact}
	\begin{aligned}
		 & \text{if the distribution of reward is symmetric:} \\
		 & ~~~~~~~~~~~~~~~~~~~~~ \lambda > \frac{2 \sqrt{3} \sigma_1}{|2 \mu_1 - 1| \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {|2 \mu_1 - 1|}  \\
	    & \text{if the distribution of reward is not symmetric} \\
	    & ~~~~~~~~~~~~~~~~~~~~~ \lambda > \frac{2 \sqrt{3} \sigma_1}{(2 \mu_1 - 1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {2 \mu_1 - 1} \qquad \text{ for } \mu_1 > \frac{1}{2} \\
	    & ~~~~~~~~~~~~~~~~~~~~~ \lambda >  \frac{12 \sigma_1^2}{(1 - 2\mu_1) \sigma_{\omega}^2 - 24 \sigma_1^2} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{2 \sqrt{6} \sigma_1}{1 - 2\mu_1} \qquad \text{ for } \mu_1 < \frac{1}{2}
	\end{aligned}
\end{equation}
\fi
   
   

For the case $\mu_1 = \frac{1}{2}$, we analyze it separately, $Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right)$ will become
\[
    \begin{aligned}
        Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right) & = Q_{1, s} \left(\frac{1}{2}\right) \\
        & = \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \left( R_{1, i} - \frac{1}{2} \right) \omega_i > \frac{\lambda}{2} \sum_{i = 1}^s (\omega_i'' - \omega_i')  \right) \\
        & = \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \big( R_{1, i} - \mu_1 \big) \omega_i > \frac{\lambda}{2} \sum_{i = 1}^s (\omega_i'' - \omega_i')  \right) \\
        & = \mathbb{P}_Z \left( Z > \frac{- s(\overline{R}_{1, s} - \mu_1)}{\sigma_{\omega}\sqrt{\sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + \lambda^2 s  / 2}} \right) \\
        & \geq \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 \geq 0 \right) \times\frac{1}{2} + \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 < 0 \right) \times\mathbb{P}_Z \left( Z > \frac{ s(\mu_1 - \overline{R}_{1, s})}{\sigma_{\omega}\sqrt{\sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + \lambda^2 s  / 2}} \right).
    \end{aligned}
\]
where $Z \sim \mathcal{N}(0, 1)$ is the standard Gaussian variable. 
And then by the inequality $\mathbb{P} (Z \geq x) \geq e^{-x^2} / 4$ for any $x > 0$ in \cite{zhang2022sharper}, we will have
\[
     \begin{aligned}
        & \mathbb{P}_Z \left( Z > \frac{s(\mu_1 - \overline{R}_{1, s})}{\sigma_{\omega}\sqrt{\sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + \lambda^2 s  / 2}} \right) \\
        (\text{by Cauchy inequality})\geq & \mathbb{P} \left( Z \geq \frac{s(\mu_1 - \overline{R}_{1, s})}{\sigma_{\omega} \sqrt{s (\mu_1 - \overline{R}_{1, s})^2 + \lambda^2 s  / 2}} \right)  \\
        \geq & \frac{1}{4} \exp\left\{ - \frac{s(\mu_1- \overline{R}_{1, s})^2}{4 (\mu_1- \overline{R}_{1, s})^2 + 2 \lambda^2} \right\}.
     \end{aligned}
\]
Therefore, by the fact \eqref{constant_bound} and Lemma \ref{lem_subGtosubE} again,
\[
    \begin{aligned}
        a_{k, s} & \leq Q_{1, s} \left(\frac{1}{2}\right) \\
        & \leq 2 \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 \geq 0 \right) + 4 {\mathbb{E}} \exp \left\{ \frac{s(\mu_1- \overline{R}_{1, s})^2}{4 (\mu_1- \overline{R}_{1, s})^2 + 2 \lambda^2} \right\} \\
        & \leq 2 + 4 {\mathbb{E}} \exp \left\{ \frac{s(\mu_1- \overline{R}_{1, s})^2}{ 2 \lambda^2} \right\} \\
        \alignedoverset{\text{apply Lemma \ref{lem_subGtosubE}}}{\leq} 2 \left( 1 + 2 e^{9 / 8} \right),
    \end{aligned}
\]
as long as we take
\[
    \frac{s}{2 \lambda^2} \leq \frac{s}{8 \sigma_1^2}
\]
which is equivalent to $\lambda \geq 2 \sigma_1$. 
   
\fi

By applying Lemma \ref{lem_subGtosubE}, we know that \eqref{a_k_constant_3} can be upper bounded by $e^{9 / 8}$, if we take
\[
    \frac{s(2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^4} \leq \frac{s}{8 \sigma_1^2}.
\]
A sufficient condition for this is
\[
    \frac{s(2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^4} \leq \frac{s}{16 \sigma_1^2}, \qquad \lambda^2 - \frac{8\sigma_1}{\sigma_{\omega}} \lambda - \frac{4 \sigma_1}{\sigma_{\omega}} \geq 0,
\]
i.e.
\begin{equation}\label{tuning_condition_exact}
	\lambda \geq \frac{4 \sigma_1}{\sigma_{\omega}} + \sqrt{\frac{4\sigma_1}{\sigma_{\omega}} \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1\right)}.
\end{equation}

Therefore, if we take the tuning parameters $\lambda$ and $\sigma_{\omega}$ as in \eqref{tuning_condition_exact}, then $a_{k, s}$ will be bounded by a constant as 
\begin{equation}\label{a_k_constant_final}
	a_{k, s} \leq 2 + 4 e^{9 / 8}.
\end{equation}
   
       
   
Note that this step derives the restriction conditions of the tuning parameters in Theorem \ref{thm.MAB}. 







  

  


\subsubsection{Decomposition for bounding $a_k$}
\todorw{
The paragraphs below belong to the roadmap of this section and I would suggest moving them to the beginning of this section. 
I would suggest always provide the big picture first, and then zoom into detailed components, so readers will not be lost. 
Otherwise, e.g., readers will be curious about why we need to bound a by constant. Few people has the patience to finish what above without knowing the big picture. 
}


Next, we decompose $a_{k, s}$ as $a_{k, s} = a_{k, s, 1} + a_{k, s, 2} + a_{k, s, 3}$, and we can focus on these simpler components $a_{k, s, j}$ for $j = 1, 2, 3$. 

Following the idea in \cite{kveton2019garbage}, we are going to try to bound 
\[
    s_{a, j} = \max \{ s: a_{k, s, j} \geq T^{-1}\}, \qquad j = 1, 2, 3
\]
with a constant times $\log T$. Indeed, if we successfully do so, then 
\[
    \begin{aligned}
    	a_{k} & \leq \max_{s \in [T]} a_{k, s} \times s_{a, j} + T^{-1} \times (T - s_{a, j}) \\
    	& \asymp O(1) \times O(\log T) + O(1) = O  (\log T),
    \end{aligned}
\]
which does bound $a_{k} = \sum_{s = 0}^{T - 1} a_{k, s} \lesssim \log T$.




For the decomposition, a common approach is using indicators on "\textit{good}" events to decompose. Denote the shifted version mean reward defined as 
\[
    \overline{R}_{k, s}^* := \frac{\sum_{i = 1}^s R_{k, i} + \lambda s}{s(1 + 2 \lambda)} = \frac{\lambda}{1 + 2 \lambda} + \frac{1}{1 + 2 \lambda} \overline{R}_{k, s}.
\]
Then we can define the following "good" events for $k$-th arm as
\[
    A_{k, s} := \left\{ - C_1 \Delta_k < \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2 \lambda}  < C_1 \Delta_k \right\},
\]
and
\[
    G_{k', s} := \left\{ - C_2 \Delta_k \leq \overline{Y}_{k', s} - \overline{R}_{k', s}^* \leq C_2 \Delta_k \right\}.
\]
The definitions of $A_{k, s}$ and $G_{k, s}$ are intuitive: $A_{k, s} \, \cap \, A_{k, s}'$ represents the sample mean does not derivative from the true mean too far; and events on $G_{k', s}$ means $\overline{Y}_{k', s}$ is not too far away from the scaled-shifted sample mean $\overline{R}_{k, s}^*$. Here $0 < C_1, C_2 \leq 1$ are some constants.

Therefore, by using these good events, we decompose $a_{k, s}$ as the following three parts:
\[
    a_{k, s, 1} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}^c)\Big],
\]
\[
    a_{k, s, 2} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s}^c)\Big], \qquad a_{k, s, 3} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s})\Big].
\]
As we can see, $a_{k, s, 1}$ and $a_{k, s, 2}$ are defined on complement sets of the good events or say "bad" events,
and we will bound them by bounding the probability $\mathbb{P} (A_{1, s}^c)$ and $\mathbb{P} (G_{1, s}^c)$; while $a_{k, s, 3}$ defined on the "good" events, we will need extra steps to bound them, which we will specify later. 

\subsubsection{Bounding $a_{k, s, 1}$ by analyzing $s_{a, 1}$}

We will bound $a_{k, s, 1}$ by bounding the probability of $A_{k, s}^c$. Write
\begin{equation}\label{a_k_1_bound}
    a_{k, s, 1} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}^c)\Big] \leq {\mathbb{E}} \Big[ T  \mathbb{I}(A_{1, s}^c)\Big] = T \mathbb{P} (A_{1, s}^c) 
\end{equation}
Since the summation of independent sub-Gaussian variables is still sub-Gaussian, \eqref{lem_subGtosubE_1} in Lemma \ref{lem_subGtosubE} gives $\overline{R}_{1, s} - \mu_1 \sim \operatorname{subG} (\sigma^2 /  s)$. And by the concentration inequality of the sub-Gaussian variable,
\[
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \left( \big| \overline{R}_{1, s} - \mu_1 \big| \geq (1 + 2\lambda) C_1 \Delta_k \right) \\
        & \leq 2 \exp \left\{ - \frac{(1 + 2\lambda)^2 C_1^2 \Delta_k^2 s}{2 \sigma_1^2}\right\} \\
        \alignedoverset{\text{take } s \geq s_{a, 1} :=  \begin{matrix} \frac{\sigma_1^2}{C_1^2 (1 + 2 \lambda)^2 \Delta_k^2} \times\log (2 T) \end{matrix}}{\leq} \frac{1}{T^2}
    \end{aligned}
\]
Hence, we obtain that
\[
    a_{k, s, 1} \leq  T \mathbb{P}(A_{1, s}^c) \leq T^{-1}, \qquad \text{ for any } s \geq s_{a, 1} \asymp \log T.
\]

  

\subsubsection{Bounding $a_{k, s, 2}$ by tackling $s_{a, 2}$}
For bounding the probability of $G_{1, s}^c$ conditioning on $\mathcal{H}_{1, s}$, we use the fact of the following lemma for Gaussian variable.
\begin{lemma}\label{lem_gau_frac}
	Suppose $X$ and $Y$ are Gaussian variables with means ${\mathbb{E}} X = 0$ and ${\mathbb{E}} Y > 0$, then
	\[
	    \mathbb{P} \left(\frac{|X|}{|Y|} > c \right) \leq 2 \mathbb{P} (X > c Y) + \mathbb{P} \left( Y < 0 \right)
	\]
	for any $c > 0$.
\end{lemma} 
\todorw{This section is too long and readers will be lost. Please either provide a roadmap, or better organize, or move some details to later as a lemma. }
\todohw{I agree, I am thinking to add a extra lemma part including all technique details for bounding the inter-step terms in these section. }
By the above lemma, note that 
\[
    \begin{aligned}
    	& {\mathbb{E}}_{\omega, \omega', \omega''} \left[ \sum_{i = 1}^s \big[ (1 + 2\lambda) R_{1, i} - (\overline{R}_{1, s} + \lambda)\big] \omega_i + \sum_{i = 1}^s \lambda (1 + \lambda - \overline{R}_{1, s}) \omega_i' - \sum_{i = 1}^s \lambda (\overline{R}_{1, s} + \lambda) \omega_i'' \right] \\
    	 = & s \big( (1 + 2\lambda) \overline{R}_{1, s} - (\overline{R}_{1, s} + \lambda) + \lambda (1 + \lambda - \overline{R}_{1, s}) - \lambda (\overline{R}_{1, s} + \lambda) \big) = 0,
    \end{aligned}
\]
we can upper bound the tail probability $\mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s})$ as, 
\[
    \begin{aligned}
        & \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s})\\
        = & \mathbb{P} \big( |\overline{Y}_{k', s} - \overline{R}^*_{k', s}| > C_3 \Delta_k \mid \mathcal{H}_{1, s}\big)\\
        = & 2 \mathbb{P}_{\omega, \omega', \omega''}  \left\{ \left| \frac{\sum_{i = 1}^s \big[ (1 + 2\lambda) R_{1, i} - (\overline{R}_{1, s} + \lambda)\big] \omega_i + \sum_{i = 1}^s \lambda (1 + \lambda - \overline{R}_{1, s}) \omega_i' - \sum_{i = 1}^s \lambda (\overline{R}_{1, s} + \lambda) \omega_i''}{(1 + 2 \lambda)(\sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i'')} \right| > C_2 \Delta_k \right\} \\
        \alignedoverset{\text{by Lemma \ref{lem_gau_frac}}}{\leq} 2 \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s \big[ (1 + 2\lambda) R_{1, i} - (\overline{R}_{1, s} + \lambda)\big] \omega_i + \sum_{i = 1}^s \lambda (1 + \lambda - \overline{R}_{1, s}) \omega_i'  \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - \sum_{i = 1}^s \lambda (\overline{R}_{1, s} + \lambda) \omega_i'' - C_2 (1 + 2\lambda) \Delta_k \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] > 0\Bigg\} \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + \mathbb{P} \left\{ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i'' < 0\right\}  =: I^{(1)} + II^{(1)}. 
    \end{aligned}
\]
To analyze the above probability, we first study $II^{(1)}$. We write it as
\[
    \begin{aligned}
    	II^{(1)} & = \mathbb{P} \left( \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i'' < 0\right) \\
    	& = \mathbb{P} \left( \frac{\sum_{i = 1}^s (\omega_i - 1) + \lambda \sum_{i = 1}^s (\omega_i' - 1) + \lambda \sum_{i = 1}^s (\omega_i'' - 1)}{\sqrt{s \sigma_{\omega}^2 (1 +2 \lambda^2)}} < - \frac{(1 + 2\lambda) s}{\sqrt{s \sigma_{\omega}^2 (1 +2 \lambda^2)}} \right) \\
    	& = \mathbb{P} \left( Z < - \frac{(1 + 2\lambda) }{\sigma_{\omega} \sqrt{1 + 2 \lambda^2}} \times \sqrt{s} \right) \\
    	\alignedoverset{\text{by Lemma \ref{lemma:gaussian_weight}}}{\leq} \frac{1}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\}.
    \end{aligned}
\]
For studying $I^{(1)}$, we define the following functions of $\{ R_{1,i}\}_{i = 1}^s$:
\[
   f \big(\{ R_{1,i}\}_{i = 1}^s\big) := f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) + f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) - f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big)
\]
with 
\[
    \begin{aligned}
    	f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) & := \sum_{i = 1}^s \big[ (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) - (1 + 2 \lambda ) C_2 \Delta_k  \big] \omega_i,\\
    	f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) & := \sum_{i = 1}^s \big[ \lambda (\lambda + 1  - \overline{R}_{1, s}) - (1 + 2 \lambda ) C_2 \Delta_k  \big] \omega_i', \\
    	f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big) & = \sum_{i = 1}^s \big[ \lambda (\overline{R}_{1, s} + \lambda) + (1 + 2 \lambda ) C_2 \Delta_k  \big] \omega_i''.
    \end{aligned}
\]
   
   
   
Then we can write $I^{(1)} = 2 \mathbb{P}_{\omega, \omega', \omega'} (f_1 + f_2 - f_3 > 0)$. Note that $f_1, f_2$, and $f_3$ are mutually independent. 
\todorw{Few decomposition are "by luck". You must have a reason to do so (e.g., is "independent" the motivation for such a decomposition? and they are independent for a reason.). Always share the motivation with readers. It also helps us to think more clearer about our proof strategy. }
The expectation is
\[
    {\mathbb{E}} \big[ f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big] = - 3 C_2 (1 + 2 \lambda) \Delta_k s,
\] 
and the variance is
\begin{equation}\label{bound_var_f}
    \begin{aligned}
        & \operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s} \big) \\
        = & \sigma_{\omega}^2 \Bigg [ \sum_{i = 1}^s \big[ (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) - (1 + 2 \lambda ) C_2 \Delta_k  \big]^2 \\
        & \qquad \qquad \qquad + \sum_{i = 1}^s \big[ \lambda (\lambda + 1  - \overline{R}_{1, s}) - (1 + 2 \lambda ) C_2 \Delta_k  \big]^2 + \sum_{i = 1}^s \big[ \lambda (\overline{R}_{1, s} + \lambda) + (1 + 2 \lambda ) C_2 \Delta_k  \big]^2 \Bigg] \\ 
        =: & \sigma_{\omega}^2 \Big[ I^{(2)} + II^{(2)} + III^{(2)} \Big].
    \end{aligned}
 \end{equation}

 For bounding the conditional variance above. We calculate the components are:
 \begin{equation}\label{bound_I}
 	\begin{aligned}
 	    I^{(2)} & = \sum_{i = 1}^s  \big[ (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) \big]^2  + s \lambda^2 (\lambda + 1  - \overline{R}_{1, s})^2 + s \lambda^2 (\overline{R}_{1, s} + \lambda)^2 \\
 	    & = \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s})^2 + 2 s \lambda^2 \big( 3 \overline{R}_{1, s}^2 - 3 \overline{R}_{1, s} + \lambda^2 + \lambda + 1 \big) \\
 	    & = \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + (6 \lambda^2 - 1)s (\overline{R}_{1, s} - \mu_1)^2 + 6  s \lambda^2 (2 \mu_1 - 3)  (\overline{R}_{1, s} - \mu_1) + 2 s \lambda^2 \big( \lambda^2 + \lambda + 1 + 3 \mu (\mu - 1) \big) \\
 	    \alignedoverset{\text{Cauchy inequality}}{\leq} 6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2  + 6  s \lambda^2 (2 \mu_1 - 1)  (\overline{R}_{1, s} - \mu_1) + 2 s \lambda^2 \big( \lambda^2 + \lambda + 1 - \mu_1 (1 - \mu_1) \big) \\
 	    & = 6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + 6  s \lambda^2 (2 \mu_1 - 1)  (\overline{R}_{1, s} - \mu_1)  + 2 s \lambda^2 \left( \lambda^2 + \lambda + 1 - \mu_1 (1 - \mu_1)\right),
 	\end{aligned}
 \end{equation}
 \[
     \begin{aligned}
         II^{(2)} & = - 2 (1 + 2\lambda) C_3 \Delta_k \bigg[ \sum_{i = 1}^s  \big( (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) \big) + s \lambda (\lambda + 1  - \overline{R}_{1, s}) -  s \lambda (\overline{R}_{1, s} + \lambda) \bigg] \\
         & = - 2 s (1 + 2\lambda) C_3 \Delta_k \Big[ \big(\overline{R}_{1, s} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) + \lambda (\lambda + 1  - \overline{R}_{1, s}) \big) -  \lambda (\overline{R}_{1, s} + \lambda) \Big] = 0,
     \end{aligned}
 \]
 and $III^{(2)} = 3 s (1 + 2\lambda)^2 C_3^2 \Delta_k^2$. Therefore, the conditional variance in \eqref{bound_var_f} is bounded by
 \begin{equation}\label{bound_var_f_new}
 	\begin{aligned}
 		 \sigma_{\omega}^{-2}\operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s} \big) & \leq I^{(2)} + II^{(2)} + III^{(2)} \\
 		 & \leq  \underbrace{6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + 6 s \lambda^2 (2\mu_1 - 1) \left( \overline{R}_{1, s} - \mu_1\right)}_{\text{the random part}} \\
 		 & ~~~~~~~~~~~~~~~~~~~ + \underbrace{2 s \lambda^2 \left( \lambda^2 + \lambda + 1 - 3 \mu_1 (1 - \mu_1)\right) + 3 s (1 + 2\lambda)^2 C_2^2 \Delta_k^2}_{\text{the determined part}} \\
 		 & =: \mathfrak{R}_1 + \mathfrak{R}_2 + \mathfrak{D},
 	\end{aligned}
 \end{equation}
 where
 \[
     \mathfrak{R}_1 := 6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2, \qquad \mathfrak{D} = 3 s (1 + 2\lambda)^2 C_2^2 \Delta_k^2
 \]
 are strictly positive. For the random quantity,
 \[
     \mathfrak{R}_2 = 6 s \lambda^2 (2\mu_1 - 1) \left( \overline{R}_{1, s} - \mu_1\right) + 2 s \lambda^2 \left( \lambda^2 + \lambda + 1 - 3 \mu_1 (1 - \mu_1)\right),
 \]
 it is non-positive with a high probability. Indeed, note that $0 \leq 3 \mu_1 (1 - \mu_1) \leq \frac{3}{4}$
 \begin{equation}\label{R_2_pr}
     \begin{aligned}
     	\mathbb{P} ( \mathfrak{R}_2 > 0) & = \mathbb{P} \left( 3 (2 \mu_1 - 1)\left( \overline{R}_{1, s} - {\mu_1} / {2} \right) > \lambda^2 + \lambda + 1 - 3 \mu_1 (1 - \mu_1)\right) \\
     	& \leq \mathbb{P} \left( \Big| 3 (2 \mu_1 - 1)\left( \overline{R}_{1, s} - {\mu_1} \right)\Big| > \Big| \lambda^2 + \lambda + 1 - 3 \mu_1 (1 - \mu_1) \Big| \right) \mathbb{I}\left( \mu_1 \neq  \frac{1}{2}\right) + \mathbb{P} (\varnothing) \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \\
     	\alignedoverset{0 \leq 3 \mu_1 (1 - \mu_1) \leq \frac{3}{4}}{\leq} \mathbb{P} \left( \Big| \overline{R}_{1, s} - {\mu_1} \Big| > \frac{\lambda^2 + \lambda + 1 / 4}{|3 (2 \mu_1 - 1)|} \right) \mathbb{I}\left( \mu_1 \neq  \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \\
     	\alignedoverset{\text{by sub-Gaussian inequality}}{\leq} \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right).
     \end{aligned}
 \end{equation}


    
       
       
       
       
   
   
   
   
       
       
       
       
       
       
       
       
       
       
       
       
   

   
       
       
       
       
       
       
       
       
       
       
   
   
       
       
       
       
       
       
       
       
       
       
       
       
       
   
   
   
       
       
       
   


   
       
       
       
       
       
   
Then by applying Lemma \ref{lemma:gaussian_weight} again,
\[
    \begin{aligned}
        \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s}) & = 2 \mathbb{P} \left(\frac{f_1 + f_2 - f_3 - {\mathbb{E}} \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{\sqrt{\operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}} > - \frac{{\mathbb{E}} \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{\sqrt{\operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}} \right) \\
        & \leq \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)} \right\}.
       
    \end{aligned}
\]
Therefore, combining with \eqref{bound_var_f_new}, 
$a_{k, s, 2}$ will be bounded by 
\[
    \begin{aligned}
        a_{k, s, 2} & \leq {\mathbb{E}} \Big[ T  \mathbb{I}(G_{1, s}^c)\Big] =  T {\mathbb{E}} \Big[ \mathbb{P}(G_{1, s}^c \mid \mathcal{H}_{1, s}) \Big] \\
        & \leq T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)} \right\} \\
        & \leq T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)} \right\} \big( \mathbb{I} (\mathfrak{R}_2 \leq 0) + \mathbb{I}(\mathfrak{R}_2 > 0) \big) \\
        \alignedoverset{\text{by the decomposition \eqref{bound_var_f_new}}}{\leq} T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \sigma_{\omega}^2 \big(\mathfrak{R}_1 + \mathfrak{R}_2 + \mathfrak{D} \big)}\right\} \mathbb{I} (\mathfrak{R}_2 \leq 0) + T \mathbb{P} (\mathfrak{R}_2 > 0) \\
        & \leq T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \sigma_{\omega}^2 \big(\mathfrak{R}_1 + \mathfrak{D} \big)}\right\} + T \mathbb{P} (\mathfrak{R}_2 > 0)
    \end{aligned}
\]
Recall $R_{1, i} - \mu_1 \sim \operatorname{subG}(\sigma_1^2)$, and then $(R_{1, i} - \mu_1)^2 - \operatorname{var} (R_{1, i}) := \xi_i \sim \operatorname{subE}(8 \sqrt{2} \sigma_1^2)$.
   
Therefore, for furthermore bounding $\mathfrak{R}_1 + \mathfrak{D}$, we have
\begin{equation}\label{G_function_bound}
	\begin{aligned}
		\mathfrak{R}_1 + \mathfrak{D} & = 6 \lambda^2 \left[ \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 - s \operatorname{var} (R_{1, i}) \right] + \left[ s \operatorname{var} (R_{1, i}) + 3 s(1 + 2\lambda)^2 C_2^2 \Delta_k^2\right] \\
             & = 6 \lambda^2 \sum_{i = 1}^s \xi_i + s \left[ 6 \lambda^2\operatorname{var} (R_{1, i}) + 3 (1 + 2\lambda)^2 C_2^2 \Delta_k^2\right] \\
	
		\alignedoverset{\text{by $\operatorname{var}(R_{1, i}) \leq \sigma_1^2$}}{\leq}  6\lambda^2 \sum_{i = 1}^s \xi_i + 3 s \left[ 2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2\right]  \\
        & = 3 s \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right],
	\end{aligned}
\end{equation}
   
       
       
       
       
       
   
   
       
       
   
where $\overline{\xi} = \frac{1}{s} \sum_{i = 1}^s \xi_i$ is the mean of $\{ \xi_i \}_{i = 1}^s$, Then applying Lemma \ref{lem_subE_ineq} in \eqref{G_function_bound} will furthermore give
\begin{equation}\label{a_2_bound}
	\begin{aligned}
		a_{k, s, 2} & \leq T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \sigma_{\omega}^2 \big(\mathfrak{R}_1 + \mathfrak{D} \big)}\right\} + T \mathbb{P} (\mathfrak{R}_2 > 0) \\
		\alignedoverset{\text{by \eqref{G_function_bound} and \eqref{R_2_pr}}}{\leq} T {\mathbb{E}}\exp \left\{ - \frac{9 C_2^2 (1 + 2\lambda)^2 \Delta_k^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right] }\right\} \\
		 & ~~~~~~~~~~~~~~~ + T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \right] \\
		 & = T {\mathbb{E}}\exp \left\{ - \frac{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2 s }{ 2 \sigma_{\omega}^2  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right] }\right\} \\
		 & ~~~~~~~~~~~~~~~ + T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \right].
	\end{aligned}
\end{equation}
The next step is applying Lemma \ref{lem_subE_ineq}. We denote $h_{\ell} = h_{\ell} (\cdot, \cdot, \cdot, \cdot) > 0$ as the different functions with different $\ell \in \mathbb{N}$, then
\[
    \log {\mathbb{E}}\exp \left\{ - \frac{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2 s }{ 2 \sigma_{\omega}^2  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right] }\right\} \leq - s \times h_1 (\sigma_1, \sigma_{\omega},\lambda, \Delta_k, C_2).
\]
For bounding \eqref{a_2_bound} with $T^{-1}$, we solve 
\[
    T {\mathbb{E}}\exp \left\{ - \frac{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2 s }{ 2 \sigma_{\omega}^2  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right] }\right\} \leq \frac{1}{2T}, \qquad \text{i.e.} \qquad s \geq \frac{\log 2 + 2 \log T}{h_1(\sigma_1, \sigma_{\omega},\lambda, \Delta_k, C_2)},
\]
and
\[
    T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \right] \leq \frac{1}{2T}, \quad \text{i.e.} \quad s \geq \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \left[ \log 2 + 2 \log T \right].
\]
Therefore, if we take 
\[
    s_{a, 2} := \left[ \frac{1}{h_1(\sigma_1, \sigma_{\omega},\lambda, \Delta_k, C_2)} \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \right] \times 3 \log T,
\]
we will have $a_{k, s, 2} \leq \frac{1}{T}$.
   
       
       
       
       
       
   
   
       
       
       
       
       
       
       
   

   
   
   
       
   
   

   
   
       
           
           
           
           
           
           
       
   
   
   
       
   
   
   
   
       
           
           
       
   
   
   
       
       
           
       
       
       
           
       
       
       
           
       
        
   
   
   
       
   
   

\begin{lemma}[sub-Exponential concentration]\label{lem_subE_ineq}
    Suppose zero-mean independent random $ X_i \sim \operatorname{subE}(\lambda_i)$, and positive constants $a$ and $b$ satisfy $a \overline{X} + b > 0$ with the mean $\overline{X} := \frac{1}{s} \sum_{i = 1}^s X_i$. Then,
    \[
        \log {\mathbb{E}} \exp \left\{ - \frac{s}{a \overline{X} + b}\right\} \, \lesssim \, - s,
    \]
    for $s \in \mathbb{N}$.
   
   
\end{lemma}

   
       
   
   
       
       
   

\subsubsection{Bounding $a_{k, s, 3}$ by analyzing $s_{a, 3}$}

In the previous section, we control the probability of "bad" events to bound $a_{k, s, 1}$ and $a_{k, s, 2}$. But since $a_{k, s, 3}$ is defined on good events, we need a different technique to deal with $a_{k, s, 3}$.

Different from dealing with $a_{k, s, 1}$ and $a_{k, s, 2}$, in which we control the probability of the bad events with $O(T^{-2})$,
$a_{k, s, 3}$ is defined on "good" events. Hence, 
We need a special technique to deal with $a_{k, s, 3}$. Note that 
\begin{equation}\label{a_k_3_sw}
    \begin{aligned}
        \{N_{1, s}(\tau_{k})<T^{-1}\} & \supseteq \{N_{1, s}(\tau_{k}) < (T^{2}-1)^{-1}\} \\
        \alignedoverset{\text{by } (T^2 - 1)^{-1} \leq T^{-1}}{=} \big\{Q_{1, s}\left(\tau_{k}\right)^{-1}<1+\left(T^{2}-1\right)^{-1}\big\} \\
        & = \big\{Q_{1, s}(\tau_{k})^{-1} < (1-T^{-2})^{-1}\big\} = \big\{\left[1-Q_{1, s}(\tau_{k})\right]<T^{-2}\big\},
    \end{aligned}
\end{equation}
and $a_{k, s, 3} \leq {\mathbb{E}} \big[ N_{1, s} (\tau_k) \mathbb{I} (A_{1, s} \cap G_{1, s}) \big]$. 
Thus, similar to 
\cite{wang2020residual}, it is sufficient to find $s \geq s_{a,3}$ satisfying $\big\{[1-Q_{1,s}(\tau_{k})]<T^{-2}\big\}$ on the event $\{A_{1, s} \cap G_{1, s}\}$.

We write 
\[
    \big[1-Q_{1, s}(\tau_{k})\big]  \mathbb{I} (A_{1, s})\mathbb{I}  (G_{1,s}) = \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Gamma_1 \, | \, \mathcal{H}_{1, s} \big)  \mathbb{I} (A_{1, s} \, \cap \, G_{1, s})
\]
with $\Gamma_1 := \Gamma_1(\tau_1) = \tau_1 - \overline{R}_{1, s}^*$ measuring the difference between $\overline{Y}_{1, s}$ and $\overline{R}_{1,s}^*$. And on the event $A_{1, s} \, \cap \, G_{1, s}$, $\Gamma_1$ can be bounded in an interval.
To see this, we write 
\[
    \begin{aligned}
    	A_{1, s} & = \bigg\{ \overline{R}_{1, s}^* - \frac{\mu_1 + \lambda}{1 + 2\lambda} > - C_1 \Delta_k \bigg\} \, \cap \, \bigg\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2 \lambda}  < C_1 \Delta_k \bigg\} \\
    	& = \bigg\{ \Gamma_1 < C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} + \tau_1 \bigg\} \, \cap \, \bigg\{ \Gamma_1 > - C_2 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} + \tau_1 \bigg\} \\
    	& = \bigg\{ \tau_1 - C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda}  < \Gamma_1 < \tau_1 + C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda}  \bigg\}
    \end{aligned}
\]
and
\[
    \begin{aligned}
        G_{1, s} = \big\{ - C_2 \Delta_k \leq \overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq C_2 \Delta_k \big\} & = \big\{ \tau_1 -C_2 \Delta_k \leq \overline{Y}_{1, s} + \Gamma_1 \leq \tau_1 + C_2 \Delta_k \big\} \\
        & = \big\{ \tau_1 - C_2 \Delta_k - \overline{Y}_{1, s} \leq \Gamma_1 \leq \tau_1 + C_2 \Delta_k - \overline{Y}_{1, s}\big\} \\
        \alignedoverset{\text{by } - C_2 \Delta_k + \overline{R}_{1, s}^* \leq \overline{Y}_{1, s}  \leq C_2 \Delta_k + \overline{R}_{1, s}^*}{\subseteq} \big\{ \tau_1 - 2 C_2 \Delta_k - \overline{R}_{1, s}^* \leq \Gamma_1 \leq \tau_1 + 2 C_2 \Delta_k - \overline{R}_{1, s}^* \big\}.
       
       
    \end{aligned}
\]
Then combine the above two result, if we take $C_1 = 2 C_2$, we will have
   
       
       
       
       
   
\[
    \begin{aligned}
    	& A_{1, s} \, \cap \, G_{1, s} \subseteq \bigg\{ \tau_1 - C_1 \Delta_k - \overline{R}_{1, s}^* \wedge \frac{\mu_1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 \leq \tau_1 + C_1 \Delta_k - \overline{R}_{1, s}^* \vee \frac{\mu_1 + \lambda}{1 + 2 \lambda}  \bigg\} \\
    	\alignedoverset{\text{by} \frac{\mu_1 +  \lambda}{1 + 2 \lambda} - C_1 \Delta_k  < \overline{R}_{1, s}^* < \frac{\mu_1 +  \lambda}{1 + 2 \lambda} + C_1 \Delta_k }{\subseteq} \bigg\{\tau_1 - 2 C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 \leq \tau_1 + 2 C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \bigg\} \\
    	\alignedoverset{\text{by } \mu_1 < 1 + \lambda \text{ and } \mu_1 < \frac{2 \lambda}{2 \lambda - 1}}{\subseteq} \bigg\{ \tau_1 -  2 C_1 \Delta_k - 1 \leq \Gamma_1 \leq \tau_1 + 2 C_1 \Delta_k - \frac{\mu_1}{2}\bigg\}.
    
    
    \end{aligned}
\]
    
   
   
Now take 
\[
    \tau_1 := \frac{\mu_1}{2} - 3 C_1 \Delta_k
\]
satisfying \eqref{requirement_0} as $ \tau_1  < \frac{\mu_1}{2} < \frac{\mu_1 + \lambda}{1 + 2 \lambda}$. We furthermore get on $A_{1, s} \, \cap \, G_{1, s}$, 
\[
    \Gamma_1 \leq \tau_1 + 2 C_1 \Delta_k - \frac{\mu_1}{2} \leq - C_1 \Delta_k < 0
\]
and
\[
    0 > \Gamma_1 \geq \tau_1 - 2 C_1 \Delta_k - 1 \geq \frac{\mu_1}{2} - 1 - 3 C_1 \Delta_k \geq 0 - 1 - 3 = - 4.
\]
Thus, we get $ (- \Gamma_1) \in (C_1 \Delta_k, 4) \subseteq (0, + \infty)$. Returning to the quantity we care about, $ \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s})$, we write it as 
\[
    \begin{aligned}
        & \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s}) \\
        = & \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Gamma_1 \, | \, \mathcal{H}_{1, s} \big)  \mathbb{I} (A_{1, s} \, \cap \, G_{1, s})\\
        = & {\mathbb{E}} \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) \omega_i + \lambda \sum_{i = 1}^s (2 R_{1, i} \omega_i - \overline{R}_{1, s} \omega_i' - \overline{R}_{1, s} \omega_i'') 
        \\
        & \qquad \qquad + \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s (\omega_i' - \omega''_i) - \Gamma_1 (1 + 2\lambda) \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] < 0\Bigg\} \mathbb{I} (A_{1, s} \, \cap \, G_{1, s}).
    \end{aligned}
\]
Similar to the technique we use to bound conditional probability $\mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s})$, we just need to replace $C_2 \Delta_k$ by $(-\Gamma_1)$ in \eqref{a_2_bound}. We get that 
\begin{equation}\label{Gamma_1_bound_next}
    \begin{aligned}
         & \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I} (A_{1, s}) \mathbb{I}(G_{1, s}) \\
         \alignedoverset{\text{apply steps in  \eqref{bound_I}}}{\leq} T {\mathbb{E}}\exp \left\{ - \frac{9 (- \Gamma_1)^2 \Delta_k^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 +(- \Gamma_1)^2   \right] }\right\} \\
         \alignedoverset{\text{by the bound of } \Gamma_1}{\leq} T {\mathbb{E}}\exp \left\{ - \frac{256 \Delta_k^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 +C_1^2 \Delta_k^2  \right] }\right\}  \\
         \alignedoverset{\text{Lemma \ref{lem_subE_ineq}}}{\leq} T \exp \left\{ - s \times h_2 (\sigma_1, \sigma_{\omega},\lambda, \Delta_k, C_1)\right\}.
    \end{aligned}
\end{equation}
   
       
       
       
       
       
   
Therefore, to find $s$ such that $[1 - Q_{1, s} (\tau_k)] < T^{-2}$ on $\{ A_{1, s} \, \cap \, G_{1, s}\}$, let
\[
   \begin{aligned}
       s \geq s_{a, 3} := \frac{2 \log T}{h_2(\sigma_1, \sigma_{\omega},\lambda, \Delta_k, C_1)}.
   \end{aligned}
\]

\todorw{People will think the below is part of C.2.5, which makes the structure confusing. If the decomposition has already been given at the begining and readers are clear that you just want to deal with components in each subsection, then we can omit the below. Otherwise, maybe make it a separate section.}
Now, combining all the results in Section C.2, we get that 
\[
    a_{k} \leq \underbrace{\max_{s = 0, \ldots, T -  1} a_{k, s}}_{\text{bound by a constant in \eqref{a_k_constant_3}}} \times\overbrace{\max _{j = 1, 2, 3} s_{a, j}}^{\text{with order } \log T} + 3 T^{-1} \cdot\big(T - \underbrace{\max_{j = 1, 2, 3} s_{a, j}}_{\text{with order } \log T}\big) \lesssim \log T.
\]
\begin{remark}
   Here we do not give the explicit expression of the bound as it involves several complicated integrals, but its order is exactly $\log T$. However, for some explicit distributions, the expectation in Lemma \ref{lem_subE_ineq} can be bounded by some simpler displays, and hence we can get explicit expressions for the bounds in these cases. See Corollary \ref{cor_ber} and Corollary \ref{cor_gau}.
\end{remark}

\subsection{Bounding $b_k$}

Similar to the previous sections, for bounding $b_k = \sum_{s = 0}^{T - 1} b_{k, s}$, we decompose $b_{k, s} = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1}\big) + 1$ as $b_k = b_{k, s, 1} + b_{k, s, 2} + b_{k, s, 3} + 1$ with
\[
    b_{k, s, 1} := {\mathbb{E}} \big[ \mathbb{I} ( Q_{k, s} (\tau_k) > T^{-1}) \mathbb{I} (A_{k, s}^{c} )\big],
\]
\[
    b_{k, s, 2} := {\mathbb{E}} \big[ \mathbb{I} ( Q_{k, s} (\tau_k) > T^{-1}) \mathbb{I} (A_{k, s}) \mathbb{I} (G_{k, s}^c)\big],
\]
and
\[
    b_{k, s, 3} := {\mathbb{E}} \big[ \mathbb{I} ( Q_{k, s} (\tau_k) > T^{-1}) \mathbb{I} (A_{k, s}) \mathbb{I} (G_{k, s})\big].
\]
Since $b_{k, s}$ is naturally bounded {the} constant $2$, it only needs to bound $s_{b, j}: = \max \{ s: b_{k, s, j} \geq T^{-1}\}$ with the $C \log T$ for $j = 1, 2, 3$ with some positive constant $C$.




It is easy to see that $\mathbb{P} (A^{c}_{k, s}) \leq T^{-2} \leq T^{-1}$ when $s \geq s_{a, 1}$. Next, similar as bounding $a_{k, s, 2}$, we take
\[
    s_{b, 2} := \frac{3 \log T}{h_1(\sigma_{k}, \sigma_{\omega}, \lambda, \Delta_k, C_2)}
\]
we have $b_{k, s, 2} \leq {\mathbb{E}} \big[ \mathbb{I}(G_{k, s}^c) \big] \leq T^{-2} \leq T^{-1}$ (by just replacing $\sigma_1$ with $\sigma_k$ in $s_{a, 2}$).
It remains to deal with $b_{k, s, 3}$. The basic idea is the same as bounding $a_{k, s, 3}$. We will first bound $\Gamma_k = \overline{Y}_{k, s}  - \overline{R}_{k, s}^*$ on the event $A_{k, s} \, \cap \, G_{k, s}$.
   
   
       
       
       
   
   
       
       
       
   
Exactly the same as before, if we take
\[
    \tau_k := \frac{\mu_k}{2} - 3 C_1 \Delta_k,
\]
   
   
then we will obtain that a bound for $\Gamma_k$ is $- \Gamma_k \in (C_1\Delta_k, 4) \subseteq (0, \infty)$, then 
   
\[
    \begin{aligned}
         &  Q_{k,s} (\tau_k)\big] \mathbb{I} (A_{k, s}) \mathbb{I}(G_{k, s}) \\
         \alignedoverset{\text{apply steps in  \eqref{bound_I}}}{\leq} T {\mathbb{E}}\exp \left\{ - \frac{9 (- \Gamma_k)^2 \Delta_k^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\zeta^{(k)}} +  2 \lambda^2 \sigma_1^2 +(- \Gamma_k)^2   \right] }\right\} \\
         \alignedoverset{\text{by the bound of } \Gamma_1}{\leq} T {\mathbb{E}}\exp \left\{ - \frac{256 \Delta_k^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\zeta^{(k)}} +  2 \lambda^2 \sigma_1^2 +C_1^2 \Delta_k^2  \right] }\right\}  \\
         \alignedoverset{\text{Lemma \ref{lem_subE_ineq}}}{\leq} T \exp \left\{ - s \times  h_3 (\sigma_k, \sigma_{\omega},\lambda, \Delta_k )\right\}.
    \end{aligned}
\]
   
       
       
       
       
       
       
       
       
   
where $h_3$ is defined in Lemma \ref{lem_subE_ineq} by replacing the corresponding parameters again and $\overline{\zeta^{(k)}} = \frac{1}{s} \sum_{i = 1}^s \zeta^{(k)}_i$ with $\zeta^{(k)}_i$ are independent sub-Exponential variable such that $\zeta^{(k)}_i \sim \operatorname{subE}(8 \sqrt{2} \sigma_k^2)$. Therefore, one can just take 
\[
    \begin{aligned}
         s \geq s_{b, 3} := \frac{2 \log T}{h_3(\sigma_k, \sigma_{\omega},\lambda, \Delta_k, C_1)}
    \end{aligned}
\]
to ensure $b_{k, s, 3} \leq T^{-1}$.
And then, by using a similar argument, we have 
\[
    b_{k} \leq 2 \times\overbrace{\max _{j = 1, 2, 3} s_{b, j}}^{\text{with order } \log T} + 3 T^{-1} \cdot\big(T - \underbrace{\max_{j = 1, 2, 3} s_{b, j}}_{\text{with order } \log T}\big) \lesssim \log T.
\]
\begin{remark}
   Similarly, for specific reward distributions, we can bound $b_k$ in $\log T$ with some simple and specific coefficients. See Corollary \ref{cor_ber} and Corollary \ref{cor_gau}.
\end{remark}

   
        
        
       
       
       
       
   
   
Up to now, we have finished the first part in Theorem \ref{thm.MAB}. For the second part, 
\begin{equation*}
    \begin{aligned}
        \operatorname{Reg}_T & = \sum_{\Delta_k : \Delta_k < \Delta} \Delta_k {\mathbb{E}} J_{k, T} + \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k {\mathbb{E}} J_{k, T} \\
        & \lesssim T \Delta + \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k  +  \sum_{\Delta_k : \Delta_k \geq \Delta} \frac{\log T}{\Delta_k} \\
        & \lesssim T \Delta + \frac{ K \log T}{\Delta} \lesssim \sqrt{K T \log T}.
    \end{aligned}
\end{equation*}
for any previously specified $\Delta \in (0, 1)$ by Cauchy inequality. Therefore, we finish the proof of Theorem \ref{thm.MAB}.





\section{Proof of Corollary \ref{cor_ber}}

TBA.

\iffalse
The similar as stated in Section \ref{proof_thm}. Take advantage of Bernoulli rewards are bounded in $[0, 1]$, one can get that 
\[
    \begin{aligned}
        a_{k, s} & \leq \big( e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \big) \mathbb{I} ( \mu_1 \neq 1 / 2 )  + 2 \big( 1 + 2 e^{9 / 8} \big) \mathbb{I} ( \mu_1 = 1/ 2 ).
    \end{aligned}
\]
\todohw{Need to revise later}
as long as the parameters satisfying the restrictions in Theorem \ref{thm.MAB}. Then we need to control $a_{k, s, j}$ with $\log T$ for $j \in \{ 1, 2, 3\}$. These steps will be completed in the following.
\begin{itemize}
    \item For $a_{k, s, 1}$, recall the inequality \eqref{a_k_1_bound}. We use the fact that the rewards are bounded in $[0,1]$, Hoeffding inequality will give
    \[
        \mathbb{P} (A_{1, s}^c) = \mathbb{P} (\mu_1 - \overline{R}_{1, s} \geq (1 +2  \lambda) C_1 \Delta_k) \leq \exp \left\{ - 2 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2 s\right\}.
    \]
    Hence, take
    \[
        s \geq s_{a, 1} := \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2} \log T,
    \]
    we have $\mathbb{P} (A_{1, s}^c) \leq T^{-2}$ , and $a_{k, s, 1} \leq T^{-1}$ then.
    
    \item For $a_{k, s, 2}$, one can {follow} the same steps as in the proof of Theorem \ref{thm.MAB}, while constants in the upper bound can be simpler and sharper due to the distribution of rewards being Bernoulli.
    To be specific, the variance $\sigma_{\omega}^{-2} \operatorname{var}(f_1 + f_2 - f_3)$ in \eqref{bound_var_f} can be bound by a deterministic quantity:
    
   
    \[
        \begin{aligned}
            & \mathbb{G}_1 (s, \lambda) + \mathbb{G}_2 (s, \lambda) + g_3 (s, \lambda) \\
            \overset{\text{by the same steps in} \eqref{G_function_bound_2}}{\leq} & 6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big] \\
            \overset{\text{use the fact } R_{1, i} \in [0, 1]}{\leq} & 6 \lambda^2 s + s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big].
        \end{aligned}
    \]
    Then use \eqref{a_2_bound} again, 
    \[
        \begin{aligned}
            a_{k, s, 2} & \leq 2 T  {\mathbb{E}} \exp \left\{ - \frac{9 C_3^2 (1 + 2\lambda)^2 \Delta_k^2 s^2 }{2 \sigma_{\omega}^{2} \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + g_3 (s, \lambda) \big]}\right\} \\
            & \leq 2 T \exp \left\{ - \frac{9 \sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2 s}{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }\right\}.
        \end{aligned}
    \]
    Hence, if we take 
    \[
        \begin{aligned}
            s \geq s_{a, 2} & := \frac{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{3 \sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2} \log T \\
            (
            \text{require } T \geq 2) & \geq \frac{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{9\sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2} \log (2 T^2)
        \end{aligned}
    \]
    and then $a_{k, s, 2} \leq T^{-1}$.
    \iffalse
    Then if we denote 
    \[
        f(x) := \exp \left\{ - \frac{64^{-1} (1 + 2 \lambda)^4 \Delta_k^2 s}{4 \Big[ s^{- 1 / 2} (2 \lambda + 1) (x - s \mu_1) + \lambda \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big] \Big]^2 + 2 s^{-1} g_5(s, \lambda) \big]^2} \right\},
    \]
    then $f(x)$ is a decreasing function on $0 \leq x \leq s \mu_1$ and an increasing function on $x \geq s \mu_1$. By symmetric technique of Binomial distribution and Lemma 2 in \cite{kveton2019garbage}, we have
    \[
        a_{k, s, 2} \leq {\red{4}} T \sum_{x = 0}^{\lfloor s \mu_1 \rfloor} B(x ; s, \mu_1) f(x) \leq 4 T \left[ \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[-2 i^{2}\right] f(\mu_1 s -(i+1) \sqrt{s})+\exp \left[-2 s^{2} \mu_1^2 \right] f(0) \right].
    \]
    Similarly to \cite{kveton2019garbage}, we can show that 
    \[
        \begin{aligned}
            & \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[-2 i^{2}\right] f(\mu_1 s -(i+1) \sqrt{s}) +\exp \left[-2 s^{2} \mu_1^2 \right] f(0) \\
            \leq & \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[-2 i^{2}\right] \exp \left\{- \frac{64^{-1} (1 + 2 \lambda)^4 \Delta_k^2 s}{4 \left[ \lambda \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big] - (2 \lambda + 1) (i + 1)\right]^2 + 2 s^{-1} g_5(s, \lambda)} \right\} \\
            & \qquad +\exp \left[-2 s^{2} \mu_1^2 \right] f(0) \\
            \leq & \int_1^{s \mu_1 + 1} \exp \{ - 2 (u - 1)^2 \} \exp \left\{ - \frac{c_1 s}{4 (c_2 - (2 \lambda + 1) u )^2 + 2 c_3}\right\}  \, du \\
            = & \int_1^{s \mu_1 + 1} \exp \left\{ - \left[ 2 (u - 1)^2 + \frac{c_1 s}{4 (c_2 - (2 \lambda + 1) u )^2 + 2 c_3} \right] \right\}  \, du \\
            \leq & \int_1^{s \mu_1 + 1} \exp \left\{- \left[ 2 s^2 \mu_1^2 + \frac{c_1 s}{4(c_2 - (2 \lambda + 1) (s \mu_1 + 1))^2}\right] \right\} \, du \\
            \leq & \int_1^{s \mu_1 + 1} \exp \left\{ - 2 s \mu_1^2 \right\} \, du \leq T \mu_1 \exp \left\{ - 2 s \mu_1^2 \right\}
           
           
           
        \end{aligned}
    \]
    with $c_1 = 64^{-1} (1 + 2 \lambda)^4 \Delta_k^2$, $c_2 = \lambda \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big]$, and $c_3 = \lambda^2 \big\{ \big[ (2 \mu_1 - 1) - C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \big\}$. Therefore, if $T \geq 2$, by taking 
    \[
        \red{s_{a, 2} \geq \frac{5 \log T}{\mu_1^2}\geq \frac{\log 4 T^3 \mu_1}{\mu_1^2}}
    \]
    one will have 
    \[
        a_{k, s, 2} \leq T^{-1}
    \]
    \fi
    \item To deal with $a_{k, s, 3}$,
   
    by taking the same technique as $a_{k, s, 2}$, we have
    \[
        \begin{aligned}
             \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I}(A_{1, s}) \mathbb{I}(G_{1, s}) & \leq {\mathbb{E}} \exp \left\{ - \frac{9 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 (- \Gamma_1)^2 s^2 }{ s \big[ 4 \lambda^2 (\lambda^2 + \lambda + 4)  + 6  (1 + 2\lambda)^2 (- \Gamma_1)^2  \big]}\right\} \\
             & \leq \exp \left\{ - \frac{144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 s }{ 4 \lambda^2 (\lambda^2 + \lambda + 4)  + 6  (1 + 2\lambda)^2 C_1^2 \Delta_k^2 }\right\} \leq T^{-2}
        \end{aligned}
    \]
    as long as 
    \[
        s \geq s_{a, 3} := \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2}{36 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} \log T.
    \]
    \iffalse
    \[
        g(x) = \exp \left\{ - \frac{64^{-1} (1 + 2 \lambda)^4 \Delta_k s}{4 \Big[ s^{- / 2} (2 \lambda + 1) (x - s \mu_1) + \lambda \big[ (\lambda + \mu_1) + 4 (1 + 2 \lambda)\big] \Big]^2 + 2 s^{-1} g_5(s, \lambda)}\right\}
    \]
    one can show that
    \[
        \begin{aligned}
            \big[ 1 - Q_{1, s}(\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \leq 2 \sum_{x = 0}^{\lfloor s \mu_1 \rfloor} B(x; s, \mu_1) g(x) \leq 2 s \mu_1 \exp \{ - s \mu_1\} \leq T^{-1}
        \end{aligned}
    \]
    as long as $s \geq s_{a, 3} := s_{a, 2}$. And hence, in this case, one will have $a_{k, s, 3} \leq T^{-1}$.
    \fi
\end{itemize}

Combining these results above with 
\[
    \begin{aligned}
        &\max \left\{ \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2}, \frac{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{3\sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2}, \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2}{36 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} \right\} \\
        \leq & \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2} + \frac{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{3 \sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2} + \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2}{36 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2}\\
        = & \frac{36 (1 + 2 \lambda)^2 + \lambda^2 (\lambda^2 + \lambda + 4)}{18 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} + \frac{32^2 \lambda^2 \sigma_{\omega}^{2}  (\lambda^2 + \lambda + 4) + 12}{3 (1 + 2 \lambda)^2 \Delta_k^2} + \frac{\sigma_{\omega}^{2} \Delta_k^2}{3 \times 16^2},
    \end{aligned}
\]
we obtain that 
\[
    \begin{aligned}
        a_k & \leq \max_{s = 0, \ldots, T -  1} a_{k, s} \times\max _{j = 1, 2, 3} s_{a, j} + 3 T^{-1} \cdot\big(T - \max_{j = 1, 2, 3} s_{a, j}\big) \\
        & \leq \Big[\big( e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \big) \mathbb{I} ( \mu_1 \neq 1 / 2 )  + 2 \big( 1 + 2 e^{9 / 8} \big) \mathbb{I} ( \mu_1 = 1/ 2 ) \Big] \\
        & \qquad \qquad \times \left[  \frac{36 (1 + 2 \lambda)^2 + \lambda^2 (\lambda^2 + \lambda + 4)}{18 \sigma_{\omega}^{- 2}  (1 + 2 \lambda)^2} + \frac{32^2 \lambda^2 \sigma_{\omega}^{2}  (\lambda^2 + \lambda + 4) + 12}{3 (1 + 2 \lambda)^2 \Delta_k^2} + \frac{\sigma_{\omega}^{2} \Delta_k^2}{3 \times 16^2} \right] 
        \log T + 3.
    \end{aligned}
\]
   
By using the same technique, one can show that 
\[
    \begin{aligned}
        b_k & \leq 2 \times\max _{j = 1, 2, 3} s_{b, j} + 3 T^{-1} \cdot\big(T - \max_{j = 1, 2, 3} s_{b, j}\big) \\
        & \leq 2 \times \left\{ \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2}, \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{3 \sigma_{\omega}^{- 2}  C_3^2 (1 + 2 \lambda)^2 \Delta_k^2}, \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_2^2 \Delta_k^2}{36 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} \right\} \log T + 3 \\
        & \leq  2 \left[  \frac{18(1 + 2 \lambda)^2 + \lambda^2 (\lambda^2 + \lambda + 4)}{18 \sigma_{\omega}^{- 2}  (1 + 2 \lambda)^2} + \frac{2 \times16^2  \sigma_{\omega}^{2}  \lambda^2 (\lambda^2 + \lambda + 4) + 12}{3 (1 + 2 \lambda)^2 \Delta_k^2} + \frac{\sigma_{\omega}^{2} \Delta_k^2}{3 \times 16^2} \right] 
        \log T + 3
    \end{aligned}
\]
These results together with Lemma \ref{lem.a.b} give Corollary \ref{cor_ber}.
\fi

\section{Proof of Corollary \ref{cor_gau}}

TBA.

\iffalse
The basic proof idea is the same as the proof of Corollary \ref{cor_ber}. Since the Gaussian variable is still sub-Gaussian, the constant bound in \eqref{a_k_constant_3} is still available for this case. And next we will bound $s_{a,  j} :=  \min \{ s: a_{k, s, j} \leq T^{-1}\}$ with $\log T$ under the Gaussian case.
\iffalse
\begin{itemize}
    \item[1.] By using the fact that $s^2 (\overline{R}_{1, s} - \mu_1) \sim \chi^2(s)$,
    \[
        \begin{aligned}
            a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \times1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \\
            & \leq 
        \end{aligned}
    \]
    \[
        \begin{aligned}
            a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \times1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  \\
            (\text{by using the MGF of } \chi^2(s))& = \left[ 1 - \frac{3 (2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^2 [(\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2]s^2}\right]^{- \frac{s}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \\
            (\text{take } s = 1) & \leq \left[ 1 - \frac{3 (2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^2 [(\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2]}\right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}.
        \end{aligned}
    \]
    since $s = 0$ naturally satisfies the above inequality. This further implies we will need
    \[
        \frac{3 (2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^2 [(\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2]} < 1
    \]
    with its sufficient condition as 
    \[
        \frac{3(2 \lambda + 1)^2}{\lambda^2[\lambda^2 + (\lambda + 1)^2]} \leq 1
    \]
    i.e. $\lambda \geq 1 + \sqrt{2}$. In a word, under the assumptions in Corollary \ref{cor_gau}, one will have
    \[
        \begin{aligned}
            a_{k, s} & \leq \left[ 1 - \frac{3 (2 \lambda + 1)^2}{\sigma_{\omega}^2 [(\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2]}\right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \\
            & \leq \left[ 1 - \frac{3(2 \lambda + 1)^2}{\lambda^2[\lambda^2 + (\lambda + 1)^2]} \right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}. 
        \end{aligned}
    \]
    with $\sigma_{\omega}^2 \geq 1$ for $s \in \{ 0, \ldots, T - 1\}$.
    
    \item[2.] For controlling $a_{k}$, one can easily show that 
    \[
        a_k \leq \left\{ \left[ 1 - \frac{3(2 \lambda + 1)^2}{\lambda^2[\lambda^2 + (\lambda + 1)^2]} \right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \right\} \left[ \frac{65}{(1 + 2 \lambda)^2 \Delta_k^2} + \frac{6}{\log 2} \right] \log T + 3.
    \]
    by the following statements.
    \begin{itemize}
        \item[2.1] Still use sub-Gaussian inequality to get that we have $a_{k, s, 1} \leq T^{-1}$ when
        \[
            s \geq s_{a, 1} = \frac{64 }{(1 + 2 \lambda)^2 \Delta_k^2} \log T.
        \]
        
        \item[2.2] For dealing with $a_{k, s, 2}$, we again use the fact that 
        \[
            \Big[ \sqrt{s} (\overline{R}_{1, s} - \mu_1) + \lambda \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big] \Big]^2 \, \sim \, \chi^2(s; s^{-1}\lambda^2 \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big]^2 )
        \]
        is a noncentral chi-squared distribution, then it will lead to the result that 
        \[
            \begin{aligned}
                a_{k, s, 2} & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{64^{-1}(1 + 2 \lambda)^4 \Delta_k^2 s}{4\big[ \chi^2(s; s^{-1}\lambda^2 \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k) + 2 s^{-1}g_5(s, \lambda)\big]}\right\} \\
                & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{64^{-1} (1 + 2\lambda)^4 \Delta_k^2 s}{4  \big[ \chi^2(s) + s^{-1}g_5(s, \lambda)\big]}\right\} \\
                & \leq 2 T \int_0^{\infty} \frac{2^{- s / 2}}{\Gamma(s / 2)} u^{-s / 2-1} e^{-1 /(2 u)} e^ { - 256^{-1} (1 + 2\lambda)^4 \Delta_k^2 s u} \, du \\
                (\text{by Cauchy inequality})& \leq  2 T \frac{2^{- s / 2}}{\Gamma(s / 2)}  \int_0^{\infty} u^{- s / 2 - 1} e^{- 1 / u} e^{- 16 (1 + 2 \lambda)^4 \Delta_k \sqrt{s}} \, du  \\
                & =  2 T \frac{2^{- s / 2}}{\Gamma(s / 2)}  \Gamma(s /2 ) e^{- 16 (1 + \lambda)^2 \Delta_k \sqrt{s}} \\
                & \leq 2 T \exp \left\{ - \left[ \frac{\log 2}{2} s + 16 (1 + 2 \lambda)^2 \Delta_k \sqrt{s}\right]\right\} \leq T^{-1}
            \end{aligned} 
        \]
        if 
        \[
            s \geq s_{a, 2} :=  \left[ \frac{1}{8(1 + 2 \lambda)^2 \Delta_k} + \frac{4}{\log 2}\right] \log T
        \]
        \item[2.3] Just by replacing $C_3 \Delta_k$ by $(- \Gamma_1)$ which is bounded in $(C_1 \Delta_k, 4)$, and we will get that
        \[
            \begin{aligned}
                \big[ 1 - Q_{1, s}(\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \leq  \exp \left\{ - \left[ \frac{\log 2}{2} s + 16 (1 + 2 \lambda)^2 \Delta_k \sqrt{s}\right]\right\} \leq T^{-1}
            \end{aligned}
        \]
        i.e. $a_{k, s, 3} \leq T^{-1}$ whenever 
        \[
            s \geq s_{a, 3} := \left[ \frac{1}{16(1 + 2 \lambda)^2 \Delta_k} + \frac{2}{\log 2}\right] \log T.
        \]
    \end{itemize}
    
    \item[3.] Exactly the same as 2., we can get that 
    \[
        b_{k} \leq 2 \left[ \frac{33}{(1 + 2 \lambda)^2 \Delta_k^2} + \frac{6}{\log 2}\right] \log T + 3.
    \]
\end{itemize}
These results give Corollary \ref{cor_gau}.
\fi

\begin{itemize}
    \item For $a_{k, s, 1}$, we still use the same steps of proof in Theorem \ref{thm.MAB} directly: take
    \[
        s \geq s_{a, 1} = \frac{\sigma_1^2}{C_1^2 (1 + 2 \lambda)^2 \Delta_k^2} \log T,
    \]
    we have $a_{k, s, 1} \leq T^{-1}$.
    
    \item For $a_{k, s, 2}$, we note that
    \begin{equation}\label{chi_square}
    	\sum_{i = 1}^s (R_{1, i} - \mu_1)^2 = \sigma_1^2 \sum_{i = 1}^s \left( \frac{R_{1, i} - \mu_1}{\sigma_1} \right)^2 \, \sim \, \sigma_1^2 \chi^2(s).
    \end{equation}
    is a chi-square distribution.
    By using the density of chi-square distribution, we have a bound of \eqref{a_2_bound} as
    \[
        \begin{aligned}
            a_{k, s, 2} & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{9 C_3^2 (1 + 2 \lambda)^2 \Delta_k^2 s^2}{2 \sigma_{\omega}^{2} \big[ \mathbb{G}_{1} (s, \lambda) + \mathbb{G}_{2} (s, \lambda) + g_3(s, \lambda)\big]}\right\} \\
            & \overset{\text{use the bound in \eqref{G_function_bound_2} and \eqref{chi_square}}}{\leq} 2 T {\mathbb{E}} \exp \left\{ - \frac{9 \sigma_{\omega}^{- 2}  C_3^2 (1 + 2 \lambda)^2 \Delta_k^2 s^2}{12 \lambda^2 \sigma_1^2 \chi^2(s) + 2 s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2\big] }\right\} \\
            & =  2 T \int_0^{\infty} \exp \left\{ - \frac{D_1 s^2}{D_2 u + D_3}\right\} \frac{1}{2^{s / 2} \Gamma(s / 2)} u^{{s} / {2} - 1} e^{- u / 2} \, du \\
            & = 2 T \int_0^{\infty} \exp \left\{ - \left[ \frac{D_1 s^2}{D_2 u + D_3} + \frac{u}{4} \right]\right\} \frac{1}{2^{s / 2} \Gamma(s / 2)} u^{{s} / {2} - 1} e^{- u / 4} \, du
        \end{aligned}
    \]
   
    with
    \[
        D_1 = 9 \sigma_{\omega}^{- 2}  C_3^2 (1 + 2 \lambda)^2 \Delta_k^2, \qquad D_2 = 12 \lambda^2 \sigma_1^2, \qquad D_3 = 2 \big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2\big].
    \]
    Write 
    \[
        \frac{D_1 s^2}{D_2 u + D_3} + \frac{u}{4} = D_4 s^2 + \frac{D_2 u^2 + (D_3 - 4 D_2 D_4 s^2) u + 4 (D_1 - D_3 D_4) s^2}{4 (D_2 u + D_3) }
    \]
    with $D_4 > 0$ is a constant making
    \begin{equation}\label{gau_D_4}
        D_2 u^2 + (D_3 - 4 D_2 D_4 s^2) u + 4 (D_1 - D_3 D_4) s^2 \geq 0
    \end{equation}
    strictly holds on $u \in (0, \infty)$. A sufficient condition for \eqref{gau_D_4} is
    \[
         \begin{aligned}
             \boldsymbol\Delta & = (D_3 - 4 D_2 D_4 s^2)^2 - 4 D_2 \big[ 4 (D_1 - D_3 D_4) s^2 \big] \\
             & = 16 D_2^2 D_4^2 s^4 + 8 D_2 (D_3 D_4 - 2) s^2 + D_3^2 > 0, \qquad s \in [T].
         \end{aligned}
    \]
    For ensuring the above holds, one only needs to ask
    \[
        D_3 D_4 -  2 \geq 0.
    \]
    Therefore, we choose 
    \[
        D_4 := \frac{2}{D_3} = \frac{1}{\big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2\big]}.
    \]
    and hence 
    \[
        \frac{D_1 s^2}{D_2 u + D_3} + \frac{u}{4} \geq \frac{2}{D_3} s^2, \qquad \forall \, u > 0.
    \]
    It will furthermore give
    \[
        \begin{aligned}
            a_{k, s, 2} & \leq 2 T \int_0^{\infty} \exp \left\{ - \left[ \frac{D_1 s^2}{D_2 u + D_3} + \frac{u}{4} \right]\right\} \frac{1}{2^{s / 2} \Gamma(s / 2)} u^{{s} / {2} - 1} e^{- u  / 4}\, du \\
            & \leq 2 T \int_0^{\infty} \exp \left\{ - \frac{2 s^2 }{D_3} \right\} \frac{1}{2^{s / 2} \Gamma(s / 2)} u^{{s} / {2} - 1} e^{- u /4} \, du \\
            (\text{use the density of Gamma distribution})& = 2 T \exp \left\{ - \frac{2 s^2}{D_3} \right\} 2^{s / 2} \\
            & = 2 T \exp \left\{ -  \frac{2 s^2}{D_3} + \frac{s \log 2} {2} \right\} \leq 2 T \exp \left\{ -  \frac{2 s^2}{D_3} \right\} .
        \end{aligned}
    \]
    Hence, we can take
    \[
        s \geq s_{a, 2} := \frac{3 D_3 \log T}{2} = 3 \sigma_{\omega}^{2} \big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2\big] \log T.
    \]
    to guarantee $a_{k, s, 2} \leq T^{-1}$ for any $T \geq 2$.
    \item The approach to deal with $a_{k, s, 3}$ is the same as above, and we can get we have $a_{k, s, 3} \leq T^{-1}$ whenever
    \[
        s \geq s_{a, 3} = 3  \sigma_{\omega}^{2}  \big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2\big] \log T.
    \]

   
\end{itemize}

Combining the expressions of $a_{k, s, j}$, $j \in 1, 2, 3$, we obtain 
    \[
        \begin{aligned}
            a_{k} & \leq \Big[\big( e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \big) \mathbb{I} ( \mu_1 \neq 1 / 2 )  + 2 \big( 1 + 2 e^{9 / 8} \big) \mathbb{I} ( \mu_1 = 1/ 2 ) \Big] \\
            & \qquad \qquad \times \left[  \frac{16 \sigma_1^2}{(1 + 2 \lambda)^2 \Delta_k^2} + 12 \lambda^2  \sigma_{\omega}^{2} (\lambda^2 + \lambda + 4)  + \frac{153}{256} (1 + 2 \lambda)^2  \sigma_{\omega}^{2}  \Delta_k^2 \right] 
        \log T + 3.
        \end{aligned}
    \]
And similarly the bound of $b_k$ can be also proven as follows:
\[
     b_k \leq 2 \left[  \frac{16 \sigma_k^2}{(1 + 2 \lambda)^2 \Delta_k^2} + 9 \lambda^2  \sigma_{\omega}^2 (\lambda^2 + \lambda + 4)  + \frac{297}{512} (1 + 2 \lambda)^2 \sigma_{\omega}^{2} \Delta_k^2 \right] \log T + 3.
\]
\fi
\section{Additional Method Details}\label{sec:appendix_details}
\subsection{\name{MBE} for MAB}\label{sec:MBTS_MAB}
In this section, we present the concrete form of \name{MBE} when being applied to MAB. 
Recall that $\mathbf{x}_t$ is null, $A_t \in [K]$, and $r_k$ is the mean reward of the $k$-th arm. 
We define $f(\mathbf{x}_t, A_t; \boldsymbol{r}) = r_{A_t}$, where the parameter vector $\boldsymbol{r} = (r_1, \dots, r_K)^{\top}$. 
We define the loss function as 
\begin{align*}\label{eqn:MAB_form}
   
   
   
   
    \frac{1}{t'}\sum_{t=1}^{t'} \omega_t (r_{A_t} - R_t)^2. 
   
\end{align*}
The solution 
is then $(\widehat{r}_1, \dots, \widehat{r}_K)^{\top}$ with $\widehat{r}_k = 
(\sum_{t:A_t = k} \omega_t)^{-1}
\sum_{t:A_t = k} \omega_t R_t $, i.e., the arm-wise weighted average. After adding the pseudo rewards, we can give algorithm for MAB in Algorithm \ref{alg:MBTS-MAB-2}. 

Next, we provide intuitive explanation on why Algorithm \ref{alg:MBTS-MAB-2} works. Indeed, denote $s:= |\mathcal{H}_{k, T}|$, where $\mathcal{H}_{k, T}$ is the set of observed rewards for the $k$-th arm up to round $T$. 
Let $R_{k, l}$ be the $l$-th element in $\mathcal{H}_{k, T}$. 
Then
\[
    \begin{aligned}
        \overline{Y}_{k, s} & = \frac{\sum_{i = 1}^s \omega_i R_{k, i} + \lambda \sum_{i = 1}^s \omega_i'}{\sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''} \\
        & = \frac{s^{-1} \sum_{i = 1}^s \omega_i ( R_{k, i} - \mu_k) + s^{-1} \sum_{i = 1}^s (\omega_i - 1) + \lambda s^{-1} \sum_{i = 1}^s (\omega_i' - 1) + \mu_k + \lambda}{s^{-1} \sum_{i = 1}^s (\omega_i - 1) + \lambda s^{-1} \sum_{i = 1}^s (\omega_i' - 1)  + \lambda s^{-1} \sum_{i = 1}^s (\omega_i'' - 1) + 1 +  2 \lambda} \, \xlongrightarrow{\mathbb{P}} \, \frac{\mu_k + \lambda}{1 + 2 \lambda} 
    \end{aligned}
\]
by using the law of large numbers. 
Then, by Slutsky's theorem, 
 \[
     \begin{aligned}
         \sqrt{s} \left[ \overline{Y}_{k, s} - \frac{\mu_k + \lambda}{1 + 2 \lambda} \right] = \frac{1}{1 + 2\lambda} \left[ \frac{1}{\sqrt{s}} \sum_{i = 1}^s \omega_i ( R_{k, i} - \mu_k) + \frac{1}{\sqrt{s}} \sum_{i = 1}^s (\omega_i - 1) + \frac{\lambda}{\sqrt{s}} \sum_{i = 1}^s (\omega_i' - 1) \right] + o_p(1)
     \end{aligned}
 \]
will weakly converge to a mean-zero Gaussian distribution $\mathcal{N}\left(0, \frac{\sigma_k^2 + 2}{(1 + 2 \lambda)^2}  \sigma_{\omega}^2 \right)$. Therefore, our algorithm  preserves the order
of the arms for any $\lambda > 0$.






\input{Alg/alg_MAB}


\subsection{\name{MBE} for stochastic linear bandits}\label{sec:MBTS_LB}



\input{Alg/alg_LB}

In this section, we derive the form of \name{MBE} when applied to stochastic linear bandits. 
We focus on the setup where $\mathbf{x}_t$ is empty and $A_t \in \mathbb{R}^p$ is a linear feature vector, and other setups of linear bandits can be formulated similarly. 
In this case, $f(\mathbf{x}_t, A_t ; \boldsymbol\theta) = A_t^{\top} \boldsymbol\theta$ where the parameter vector is $\boldsymbol\theta \in \mathbb{R}^{p}$. 
Then, the weighted loss function is 
\[
   
    \sum_{t = 1}^T \omega_t \big( A_t^{\top} \boldsymbol\theta - R_t\big)^2 + \frac{\xi}{2} \| \boldsymbol\theta\|_2^2,
\]
where $\xi \geq 0$ is a penalty tuning parameter. 
The solution is the standard weighted ridge regression estimator and can be updated in the following way:
\begin{itemize}
    \item[0.] Initialization: $A_0 = \mathbf{0}$, $\widehat{\boldsymbol\theta}_0 = \mathbf{0}$ with $b_0 = \mathbf{0}$, and $V_0 = (\xi + 1 ) I_{{\operatorname{dim}(A_t)}}$.
    \item[1.] $\widehat{\boldsymbol\theta}_t = V_{t}^{-1} A_{t}$;
    \item[2.] $V_{t + 1} = V_{t} + \omega_{t} A_{t} A_{t}^{\top}$, $b_{t + 1} = b_{t} + \omega_t R_t A_t$, and hence update 
    \[
        \begin{aligned}
            \widehat{\boldsymbol\theta}_{t + 1} = V_{t + 1}^{-1} b_{t +1} & = \big(V_{t} + \omega_{t} A_{t} A_{t}^{\top} \big)^{-1} \big( b_{t} + \omega_t R_t A_t \big) \\
            & = V_t^{-1}  - V_t^{-1} A_{t} (\omega_{t}^{-1} + A_{t}^{\top} V_t^{-1}A_t)^{-1} A_t^{\top} V_t^{-1}
        \end{aligned}
    \]
    \item[3.] Take the action $A_{t + 1} = \argmax_{\mathbf{a} \in \mathcal{A}_t} \mathbf{a}^{\top} \boldsymbol\theta_{t + 1}$. 
   
   
\end{itemize}
   
   
   
   
   
   
The \name{MBE} algorithm for linear bandits is presented in Algorithm \ref{alg:MBTS-LB}.

   
   
   
   
   
   





\subsection{Naive Adaptation of the Multiplier Bootstrap}\label{sec:naive}
We present the naive multiplier bootstrap-based exploration algorithm in Algorithm \ref{alg:MBTS-naive}. 
Specifically, there is no pseudo-rewards added. 

\input{Alg/alg_naive}

\subsection{ReBoot}\label{sec:ReBoot}
For completeness, we introduce the details of \name{ReBoot} in this section and discuss its generalizability. 
More details can be found in the original papers \citep{wang2020residual, wu2022residual}. 

Consider a stochastic bandit problem with a fixed and finite set of arm $\mathcal{A}$. 
Every arm $a \in \mathcal{A}$ may have a fixed feature vector (which with slight overload of notation, we also denote as $a$). 
The mean reward of arm $a$ is $f(a)$. 
At each round $t'$, \name{ReBoot} first fit the model $f$ as $\widehat{f}$ using all data.
Then for each arm $a$, \name{ReBoot} first computes the corresponding residuals using rewards related to that arm as $\{\epsilon_t = R_t - \widehat{f}(a)\}_{t:A_t = a, t \le t'}$, 
then perturbs these residuals with random weights as $\{\omega_t \epsilon_t = R_t - \widehat{f}(a)\}_{t:A_t = a, t \le t'}$ (\name{ReBoot} also adds pseudo-residuals, which we omit for ease of notations), 
and finally use $\widehat{f}(a) + |\{t:A_t = a, t \le t'\}|^{-1} \sum_{t:A_t = a, t \le t'} \omega_t \epsilon_t$ as the perturbed estimation of the mean reward of arm $a$.  
By design, it can be seen that \name{ReBoot} critically relies on the reward history of each \textit{fixed} arm. 
Therefore, to the best of our understanding, it is not easy to extend \name{ReBoot} to problems with either changing (e.g., contextual problems) or infinite arms. 







\section{Proof of Lemmas}\label{sec:proof_lemma}

\subsection{Proof of Lemma \ref{lem.a.b}}
\begin{proof}
   
    First, we note that
    $\operatorname{Reg}_T = \sum_{k = 2}^K \Delta_k {\mathbb{E}} \sum_{s = 1}^T \mathbb{I}(I_t = k)$. If we denote $E_{k, t} := \{ \overline{Y}_{k, t} \leq \tau_k \}$, then we have the following decomposition,
    \[
        {\mathbb{E}} J_{k, T} := {\mathbb{E}} \sum_{t = 1}^T \mathbb{I}(I_t = k) = {\mathbb{E}} \sum_{t = 1}^T \mathbb{I}(I_t = k, E_{k, t} \text{ occurs}) + {\mathbb{E}} \sum_{t = 1}^T \mathbb{I}(I_t = k, E_{k, t}^c \text{ occurs}).
    \]
    Then, we can have $a_k$ and $b_k$ bound the two quantities above respectively. Indeed, we have
    \[
        \sum_{t = 1}^T \mathbb{I}(I_t = k, E_{k, t} \text{ occurs}) \leq \sum_{s = 0}^{T - 1} \sum_{t = 1}^T \mathbb{I} \left\{ \max_{j} \overline{Y}_{j, t} \leq \tau_k, J_{1, t - 1} =  s \right\} ,
    \]
    then apply the Doob extension theory (see Theorem 5.2 in Chapter II of \cite{doob1953stochastic}), we have got the above can be bounded by $N_{1, s}(\tau) \wedge T$, i.e. ${\mathbb{E}} \sum_{t = 1}^T \mathbb{I}(I_t = k, E_{k, t} \text{ occurs})  \leq a_k$. Here $a_k$ can be understood as the anti-concentration property on pre-specific level $\tau$ of the rewards from the optimal arm. 
    
   
    
    For obtaining $b_k$, we let $\mathcal{T} = \{ t \in [T] : Q_{k, J_{k, t - 1}} (\tau_k )\geq 1 / T \}$,
   
    then, we have
    \[
        \begin{aligned}
            {\mathbb{E}} \sum_{t = 1}^T \mathbb{I}(I_t = k, E_{k, t}^c \text{ occurs}) & \leq {\mathbb{E}} \sum_{t \in \mathcal{T}} \mathbb{I}(I_t = k) + {\mathbb{E}} \sum_{t \notin \mathcal{T}} \mathbb{I} (E_{k, t}^c) \\
            & \leq \sum_{s = 0}^{T - 1} \mathbb{P} \big( Q_{k, s} (\tau_k) \geq 1 / T \big) + 1 =: b_k. 
        \end{aligned}
    \]
    It is obvious that $b_k$ is the summation of the probability of tail probability larger than $T^{-1}$. 
    
    
   
\end{proof}

\subsection{Proof of Lemma \ref{lemma:gaussian_weight}}


\begin{proof}
	Denote $Q(x) := \mathbb{P} (Z > x)$ for $x > 0$ as the tail probability for standard Gaussian variable $Z$. Let $Z_1$ and $Z_2$ are two independent standard Gaussian random variables, then
	\[
	    \begin{aligned} 
	        \mathbb{P} (Z_1 \leq x, \, Z_2 \leq x) &=\int_{-x}^x \int_{-x}^x \frac{1}{2 \pi} \exp \left[\left(-z_1^2-z_2^2\right) / 2\right] \, \mathrm{d} z_1 \mathrm{d} z_2 \\ 
	        & \leq \int_0^{\sqrt{2} x} \int_0^{2 \pi} \frac{1}{2 \pi} \exp \left(-r^2 / 2\right) r \, \mathrm{d} \theta \, \mathrm{d} r \\ 
	        &=1-\exp (-x^2)
	    \end{aligned}
	\]
	giving $[1-2 Q(x)]^2 \leq 1-\exp (-x^2)$ for $x \geq 0$, or, equivalently, $\exp (-x^2) \leq 4 Q(x)-4 Q^2(x)$. But, since $4 Q^2(x)>0$ for all $x$, we get that
	\[
	    Q(x) > \frac{1}{4} \exp (-x^2)
	\]
    which is a lower bound of the form desired. For the upper bound, we note that
    \[
        \begin{aligned}
            \exp \left(x^2 / 2\right) Q(x) &=\exp \left(x^2 / 2\right) \int_x^{\infty}(2 \pi)^{-1 / 2} \exp \left(-t^2 / 2\right) \mathrm{d} t \\
            &=\int_x^{\infty}(2 \pi)^{-1 / 2} \exp \left(-\left(t^2-x^2\right) / 2\right) \mathrm{d} t \\
            &<\int_x^{\infty}(2 \pi)^{-1 / 2} \exp \left(-(t-x)^2 / 2\right) \mathrm{d} t =\frac{1}{2},
         \end{aligned}
    \]
    which gives the upper bound.
\end{proof}

   
   
       
           
           
           
           
           
       
   
   
   
       
   
   

   
   
   
         
        
       
      
     
    
   
   
   
   
   
   
   
   

\subsection{Proof of Lemma \ref{lem_subGtosubE}}

\begin{proof}
    Note that $X - \mu \sim \operatorname{subG} (\sigma^2)$ implies
    \begin{equation}\label{lem_subGtosubE_1}
        \overline{X} - \mu \sim \operatorname{subG} (\sigma^2 / n)
    \end{equation}
    by the fact that
    \[
        \begin{aligned}
        	{\mathbb{E}} \exp \left\{ s (\overline{X} - \mu) \right\} & = \prod_{i = 1}^n \exp \left\{ \frac{s}{n} ({X}_i - \mu) \right\} \\
        	(\text{by } X_i - \mu \text{ is sub-Gaussian}) & \leq \prod_{i = 1}^n \exp \left\{ \frac{s^2}{n^2} \frac{\sigma^2}{2}\right\} = \exp \left\{ \frac{s^2 (\sigma^2 / n)}{2}\right\}.
        \end{aligned}
    \]
    And then by Proposition 4.3 in \cite{zhang2020concentration}, $(\overline{X} - \mu)^2 - n^{-1} \operatorname{var} (X) \sim \operatorname{subE} (8\sqrt{2}\sigma^2 / n, 8\sigma^2 / n)$. Therefore,
    \[
        \begin{aligned}
            {\mathbb{E}} \exp\left\{ \lambda(\overline{X} - \mu)^2 \right\} & = {\mathbb{E}} \exp\left\{ \lambda \big( (\overline{X} - \mu)^2 - n^{-1}\operatorname{var}(X) \big) \right\} {\mathbb{E}} \exp \{ \lambda n^{-1}  \operatorname{var}(X)\} \\
            & \leq \exp \left\{ \frac{(8 \sqrt{2} \sigma^2 / n)^2 \lambda^2}{2}\right\} \cdot \exp \left\{ \lambda n^{-1} \sigma^2 \right\} = \exp \left\{ \frac{64 \lambda^2 \sigma^4}{n^2} + \frac{\lambda \sigma^2}{n}\right\} \leq e^{9 / 8}.
        \end{aligned}
    \]
    for any $\lambda \leq \frac{n}{8 \sigma^2}$.
\end{proof}

\subsection{Proof of Lemma \ref{lem_gau_frac}}

\begin{proof}
	Write the probability as
	\[
	    \begin{aligned}
	    	\mathbb{P} \left(\frac{|X|}{|Y|} > c \right) & = \mathbb{P} \left(|X| > c |Y| \right) \\
	    	& = \mathbb{P} \left( \{ X > c Y \} \, \cap \, \{ X > 0\} \, \cap \, \{ Y > 0 \} \right) + \mathbb{P} \left( \{ - X > c Y \} \, \cap \, \{ X \leq 0\} \, \cap \, \{ Y > 0 \} \right) \\
	    	& ~~~~~~~~~~ + \mathbb{P} \left( \{ X >-  c Y \} \, \cap \, \{ X > 0\} \, \cap \, \{ Y \leq 0 \} \right) + \mathbb{P} \left( \{ - X > - c Y \} \, \cap \, \{ X \leq 0\} \, \cap \, \{ Y \leq 0 \}\right) \\
	    	& = \mathbb{P} \left( \{ X > c Y \} \, \cap \, \{ X > 0\} \, \cap \, \{ Y > 0 \} \right)  + \mathbb{P} \left( \{ - X > c Y \} \, \cap \, \{ - X \geq 0\} \, \cap \, \{ Y > 0 \} \right) + \mathbb{P} (Y \leq 0) \\
	    	\alignedoverset{\text{by using $X$ is symmetric about $0$.}}{\leq} 2 \mathbb{P} (  X > c Y ) + \mathbb{P} (Y \leq 0) 
	    \end{aligned}
	\]
	for any $c > 0$.
\end{proof}

\subsection{Proof of Lemma \ref{lem_subE_ineq}}
   
   
       
           
           
       
   
   
   
       
   
\begin{proof}
   
    Denote $Y := a \overline{X} + b$ is a strictly positive random variable, recall for any non-negative strictly increasing function $f$, we have
    \[
        {\mathbb{E}} f(Y) = \int_0^{\infty} \mathbb{P} \left( f(Y) > r\right) \, \mathrm{d} r = \int_0^{\infty} \mathbb{P} \left( Y > f^{-1} (r)\right) \, \mathrm{d} r.
    \]
    Then for any fixed $s \in \mathbb{N}$,
    \[
        \begin{aligned}
        	{\mathbb{E}} \exp \left\{ - \frac{s}{a\overline{X} + b}\right\} & = {\mathbb{E}} \exp \left\{ - \frac{s}{Y} \right\} = \int_0^{\infty} \mathbb{P} \left( \exp \left\{ - \frac{s}{Y}\right\} > r \right) \, \mathrm{d} r \\
        	& = \int_0^1 \mathbb{P} \left( Y > \frac{s}{\log r^{-1}} \right) \, \mathrm{d} r \\
        	& = \int_0^1 \mathbb{P} \left( \overline{X} > \frac{1}{a} \left[ \frac{s}{\log r^{-1}} - b \right]\right) \, \mathrm{d} r \\
        	& = \int_0^{\exp \left\{ -\frac{s}{b}\right\}} \mathbb{P} \left( \overline{X} > \underbrace{\frac{1}{a} \left[ \frac{s}{\log r^{-1}} - b \right]}_{\leq 0}\right) \, \mathrm{d} r + \int_{\exp \left\{ -\frac{s}{b}\right\}}^{1} \mathbb{P} \left( \overline{X} > \underbrace{\frac{1}{a} \left[ \frac{s}{\log r^{-1}} - b \right]}_{> 0}\right) \, \mathrm{d} r \\
        	& \leq \exp \left\{ -\frac{s}{b}\right\} \times 1 + \int_{\exp \left\{ -\frac{s}{b}\right\}}^{1} \mathbb{P} \left( \overline{X} > \underbrace{\frac{1}{a} \left[ \frac{s}{\log r^{-1}} - b \right]}_{> 0}\right) \, \mathrm{d} r \\
        	\alignedoverset{\text{by letting $u = \frac{s}{\log r^{-1}}$}}{=} \exp \left\{ -\frac{s}{b}\right\} + \int_b^{\infty} \mathbb{P} \left( \overline{X} > \underbrace{\frac{u - b}{a}}_{>0} \right) \exp\left\{ - \frac{s}{u}\right\} \frac{s}{u^2} \, \mathrm{d} r.
        \end{aligned} 
    \]
    Recall $\overline{X}$ is the mean of some independent zero mean sub-exponential distribution, denote $\lambda := \left(\frac{1}{s} \sum_{i=1}^s \lambda_i^2\right)^{1 / 2}$, then 
    \[
        \mathbb{P} \left( \overline{X} > t \right) \leq \exp \left\{ -\frac{1}{2}\left(\frac{s t^2}{{\lambda}^2} \wedge \frac{s t}{\lambda}\right) \right\}
    \]
    for any $t > 0$ by Corollary 4.2.(c) in \cite{zhang2020concentration}. Therefore, we can furthermore bound the expectation as
    \[
        \begin{aligned}
            {\mathbb{E}} \exp \left\{ - \frac{s}{a\overline{X} + b}\right\} \alignedoverset{\text{by sub-Exponential inequality}}{\leq} \exp \left\{ -\frac{s}{b}\right\} +\int_b^{\infty} \frac{s}{u^2} \exp \left\{ - \frac{s}{u}\right\} \exp \left\{- \frac{1}{2} \frac{s (u - b)^2}{\lambda^2 a^2} \wedge \frac{s (u - b)}{\lambda a} \right\} \, \mathrm{d} u \\
        	& \leq \exp \left\{ -\frac{s}{b}\right\} +\frac{s}{b^2} \int_b^{\infty} \exp \left\{- \left[ \frac{s}{u} + \frac{1}{2} \frac{s(u - b)^2}{\lambda^2 a^2} \wedge \frac{s(u - b)}{\lambda a} \right] \right\} \, \mathrm{d} u \\
        	& = \exp \left\{ -\frac{s}{b}\right\} + \left[ \int_b^{b + \lambda  a} + \int_{b + \lambda a }^{\infty} \right] \frac{s}{u^2} \exp \left\{- \left[ \frac{s}{u} + \frac{1}{2} \frac{s(u - b)^2}{\lambda^2 a^2} \wedge \frac{s(u - b)}{\lambda a} \right] \right\} \, \mathrm{d} u \\
        	& =: \exp \left\{ -\frac{s}{b}\right\} + \big[ I + II\big].
        \end{aligned}
    \]
    The next step is bounding both $I$ and $II$. For $I$, we obtain
    \[
        \begin{aligned}
        	I & = \int_b^{b + \lambda  a} \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)^2}{2 \lambda^2 a^2}\right]\right\} \, \mathrm{d} u \\
        	& \leq \int_b^{b + \lambda  a} \frac{s}{b^2} \exp \left\{ - \left[ \frac{s}{b + \lambda  a} + \frac{s(u - b)^2}{2 \lambda^2 a^2}\right]\right\} \, \mathrm{d} u \\
        	& = \frac{s}{b^2} \exp \left\{ - \frac{s}{b + \lambda a}\right\} \int_b^{b + \lambda  a} \exp \left\{ - \frac{ s (u - b)^2}{2 \lambda^2 a^2}\right\} \, \mathrm{d} u \\
        	& \leq \frac{s}{b^2} \exp \left\{ - \frac{s}{b + \lambda a}\right\} \int_b^{\infty} \exp \left\{ - \frac{(u - b)^2}{2 \lambda^2 a^2 / s}\right\} \, \mathrm{d} u = \frac{\sqrt{2 \pi} \lambda a }{2 b^2} \exp \left\{ -\frac{s}{b + \lambda a}\right\}
        \end{aligned}
    \]
    For $II$, we need to be more careful to decompose it as
    \[
        \begin{aligned}
            II &= \int_{b + \lambda  a}^{\infty} \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u \\
            & = \left[ \int_{b + \lambda a}^{\frac{2 \lambda a s}{b}} + \int_{\frac{2 \lambda a c}{b}}^{\infty} \right] \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u \\
            & = \int_{b + \lambda a }^{\frac{2 \lambda a s}{b}} \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u + \exp \left\{ -\frac{s}{b}\right\} \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{s}{u^2} \exp \left\{ - \frac{s(u - b) ( u - \frac{2 \lambda a s}{b})}{2 \lambda a u}\right\} \, \mathrm{d} u \\
            & =: II_1 + II_2 .
        \end{aligned}
    \] 
    And for $II_1$, 
    \[
        \begin{aligned}
            II_1 & = \int_{b + \lambda a}^{\frac{2 \lambda a s}{b}} \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u \\
            & = \int_{b + \lambda a}^{\frac{2 \lambda a s}{b}} \frac{s}{u^2}  \exp \left\{ - s \times \left[ \frac{1}{u} + \frac{(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u \\
            \alignedoverset{g(u) = \frac{1}{u} + \frac{(u - b)}{2 \lambda a } \text{ is increasing on } u \geq \sqrt{2 \lambda a}}{\leq} \int_{b + \lambda a}^{\frac{2 \lambda a s}{b}} \frac{s}{u^2}  \exp \left\{ - s \times \left[ \frac{1}{\sqrt{2 \lambda a} \vee (b + \lambda a )} + \frac{(\sqrt{2 \lambda a} \vee (b + \lambda a ) - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u  \\
            & \leq \frac{s}{(b + \lambda a) } \left[ \frac{2 \lambda a s}{b} - \big( b + \lambda a \big) \right] \exp \left\{ - \frac{s}{\sqrt{2 \lambda a} \vee (b + \lambda a )}\right\}.
        \end{aligned}
    \]
   
   
       
        
        
        
       
   
   
   
       
        
        
        
       
   
   
   
       
        
        
       
   
   
    For $II_2$, we obtain that
    \[
        \begin{aligned}
        	II_2 & = \exp \left\{ -\frac{s}{b}\right\}  \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{s}{u^2} \exp \left\{ - \frac{s(u - b) ( u - \frac{2 \lambda a s}{b})}{2 \lambda a u}\right\} \, \mathrm{d} u \\
        	& \leq \exp \left\{ -\frac{s}{b}\right\}  \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{s}{u^2} \exp \left\{ - \frac{s(u - \frac{2 \lambda a s}{b})^2}{2 \lambda a u}\right\} \, \mathrm{d} u \\
                & \leq \exp \left\{ -\frac{s}{b}\right\}  \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{s}{u^2} \exp \left\{ - \frac{(u - \frac{2 \lambda a s}{b})^2}{2 \lambda a u}\right\} \, \mathrm{d} u \\
        	& \leq c \sqrt{\frac{b}{2 \lambda a s}} \exp \left\{ -\frac{s}{b}\right\} \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{1}{u^{3 / 2}} \exp \left\{ - \frac{(u - \frac{2 \lambda a s}{b})^2}{2 \lambda a u}\right\} \, \mathrm{d} u \\
        	& = \sqrt{\frac{b s}{2 \lambda a}} \cdot \frac{b}{2s}\sqrt{\frac{\pi}{2 \lambda a}} \exp \left\{ -\frac{s}{b}\right\}  \left[ 1 - e^{4 s / b} \frac{2}{\sqrt{\pi}} \int_{2 \sqrt{s / b}}^{\infty} e^{-t^2} \,  \mathrm{d} t \right] \\
        	\alignedoverset{\text{by Lemma \ref{lem_erfc}}}{\leq}\sqrt{\frac{b s}{2 \lambda a}} \cdot \frac{b}{2s}\sqrt{\frac{\pi}{2 \lambda a}} \exp \left\{ -\frac{s}{b}\right\}  \cdot (1 - 0) \\
            & = \sqrt{\frac{\pi b}{s}} \frac{b}{4 \lambda a} \exp \left\{ -\frac{s}{b}\right\} .
        \end{aligned}
    \]
    Combining these results, we obtain that
    \[
        \begin{aligned}
        	{\mathbb{E}} \exp \left\{ - \frac{s}{aX + b}\right\} & \leq \exp \left\{ -\frac{s}{b}\right\} + I +  II_1 + II_2 \\
        	& \leq \exp \left\{ -\frac{s}{b}\right\} + \frac{\sqrt{2 \pi} \lambda a s}{2 b^2} \exp \left\{ -\frac{s}{b + \lambda^2 a / \alpha}\right\} \\
        	& \qquad + \frac{s}{(b + \lambda a) } \left[ \frac{2 \lambda a s}{b} - \big( b + \lambda a \big) \right] \exp \left\{ - \frac{s}{\sqrt{2 \lambda a} \vee (b + \lambda a )}\right\} + \sqrt{\frac{\pi b}{s}} \frac{b}{4 \alpha a} \exp \left\{ -\frac{s}{b}\right\} .
        \end{aligned}
    \]
    By taking $\log$ transform, and we get the result.
\end{proof}

\begin{lemma}\label{lem_erfc}
	For any $x \geq 0$,
	\[
	    \frac{2 e^{x^2}}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} \, \mathrm{d} t \leq 1.
	\]
\end{lemma}

\begin{proof}
	\[
	    \begin{aligned}
	    	\frac{2 e^{x^2}}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} \, \mathrm{d} t & = \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{- (t^2 - x^2)} \, \mathrm{d} t \\
	    	& = \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{- (t^2 - x^2)} \, \mathrm{d} t\\
	    	& \leq \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{- (t - x)^2} \, \mathrm{d} t \\
	    	& = \frac{2}{\sqrt{\pi}} \int_0^{\infty} \frac{e^{- t^2}}{2 \times \frac{1}{2}} \, \mathrm{d} t = \frac{2}{\sqrt{\pi}} \frac{\sqrt{2 \pi \times \frac{1}{2}}}{2} = 1.
	    \end{aligned}
	\]
\end{proof}


\section{Justification of Optimistic and Special Cases of Rewards under MAB}\label{sec_spe_case}

\section{Special Cases of Rewards under MAB}\label{sec_spe_case}



   
   
       
       
   
   
   
       
   
  
       
       
   
   
       
       
       
       
       
   
   
   

As said previously, due to undetermined structure of a general sub-Gaussian distribution, Theorem \ref{thm.MAB} can only give the rate of $\log T$ without the specific coefficients. However, for specific distributions, the coefficients can be derived with relatively simple expressions. See the following two corollaries. 

\todohw{All need to revise the coefficients.}

{\color{red}{
\begin{corollary}[Bernoulli]
\label{cor_ber}
    Consider a $K$-armed bandit with Bernoulli rewards. 
    With the tuning parameter $(\lambda, \sigma_{\omega}^2)$ satisfying the restriction in Theorem \ref{thm.MAB}, then 
    the expected $T$-round regret of Algorithm \ref{alg:MBTS-MAB-2} is bounded from above as
    \[
        \begin{aligned}
            \operatorname{Reg}_T & \leq \sum_{k = 2}^K \Delta_k \left\{ B_1 (\lambda, \sigma_{\omega}) \left[ 2 B_2 (\lambda, \sigma_{\omega}) + \frac{\sigma_{\omega}^2 \Delta_k^2}{3 \times 16^2}\right] 
            \log T + 3 \right\} \\
            & \qquad \qquad \qquad  \qquad \qquad  + \sum_{k = 2}^K \Delta_k \left\{ 2 \left[  B_2 (\lambda, \sigma_{\omega}) + \frac{\sigma_{\omega}^2 \Delta_k^2}{3 \times 16^2}\right] \log T + 3 \right\}
        \end{aligned}
    \]
    where 
    \[
       \begin{aligned}
           B_1  (\lambda, \sigma_{\omega}) :=  \Big[\big( e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \big) \mathbb{I} ( \mu_1 \neq 1 / 2 )  + 2 \big( 1 + 2 e^{9 / 8} \big) \mathbb{I} ( \mu_1 = 1/ 2 ) \Big]
       \end{aligned}
    \]
    \[
        B_2 (\lambda, \sigma_{\omega}) := \frac{36 (1 + 2 \lambda)^2 + \lambda^2 (\lambda^2 + \lambda + 4)}{18 \sigma_{\omega}^{- 2} (1 + 2 \lambda)^2} + \frac{32^2 \sigma_{\omega}^{ 2} \lambda^2 (\lambda^2 + \lambda + 4) + 12}{3 (1 + 2 \lambda)^2 \Delta_k^2},
    \]
    for $T \geq 2$.
\end{corollary}




\begin{corollary}[Gaussian]
\label{cor_gau}
    Consider a $K$-armed bandit with the rewards sampled from $\mathcal{N}(\mu_k, \sigma_k^2)$. With the tuning parameter $(\lambda, \sigma_{\omega}^2)$ satisfying the restriction in Theorem \ref{thm.MAB},  the expected $T$-round regret of Algorithm \ref{alg:MBTS-MAB-2} is bounded from above as

   
    \[
      \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \left[ 6 + \left\{  C_1(\lambda, \sigma_{\omega}) +C_2(\lambda, \sigma_k) \Delta_k^{-2} \right\} \log T\right],
    \]
    where the constants $C_{1}(\lambda, \sigma_{\omega})$ and $C_{2}(\lambda, \sigma_k)$ are defined as 
    \[
        C_1 (\lambda, \sigma_{\omega}) = \big[ C_3 (\lambda, \sigma_{\omega}) + 2 \big] \sigma_{\omega}^2 \big[ 12 \lambda^2 (\lambda^2 + \lambda + 4) + (1 + 2 \lambda)^2\big], \quad  C_2 (\lambda, \sigma_{k}) = 16 \big[ C_3 (\lambda, \sigma_{\omega}) + 2 \big] (1 + 2 \lambda)^{-2} (\sigma_1^2 + \sigma_k^2)
    \]
    \[
       \begin{aligned}
           C_3 (\lambda, \sigma_{\omega}) & = \Big[\big( e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \big) \mathbb{I} ( \mu_1 \neq 1 / 2 )  + 2 \big( 1 + 2 e^{9 / 8} \big) \mathbb{I} ( \mu_1 = 1/ 2 ) \Big]
       \end{aligned}
    \]
     for any $T \geq 2$.
   
\end{corollary}
}}
\section{Lemmas on Bounding Regret Components}\label{sec_lem_components}

\subsection{Lemmas on bounding $a_k$.}

\begin{lemma}[Bounding $a_{k, s}$ for any $s>0$]\label{lem_bounding_a_k_s_constant}
    Set 
    \[
        \lambda \geq \frac{4 \sigma_1}{\sigma_{\omega}} + \sqrt{\frac{4\sigma_1}{\sigma_{\omega}} \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1\right)},
    \]
    then 
    \[
        a_{k, s} \leq 2 + 4 e^{9 / 8}
    \]
    for any $s \geq 0$.
\end{lemma}

\begin{proof}
Note that if we take
\begin{equation}\label{requirement_0}
    \tau_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda},
\end{equation}
then we will have
\begin{equation}\label{constant_bound}
    \begin{aligned}
    	a_{k, s} & = {\mathbb{E}} \left[ \left( \frac{1}{Q_{1, s}(\tau_k)}  - 1\right) \wedge T \right] \\
        \alignedoverset{\text{by } Q_{1, s}(\cdot) \text{ is decreasing}}{\leq}{\mathbb{E}} \frac{1}{Q_{1, s}(\tau_k)} \leq {\mathbb{E}} Q_{1, s}^{-1} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right),
    \end{aligned}
\end{equation}
Hence, we need to find a lower bound of the tail probability $Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right) = \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right)$. And throughout the proof, we will use the choice of \eqref{requirement_0} for $\tau_k$, and we can take advantage of the bound \eqref{constant_bound}. 

For further analysis \eqref{constant_bound}, we write it as the probability with respect to the weighted random summation of the Gaussian random variables $\{ \omega_i, \omega_i', \omega_i''\}$.

Let $x_i := (2 \lambda + 1) R_{1, i} - (\mu_1 + \lambda)$, $y_i := \lambda (\mu_1 - 1 - \lambda) $, $z_i :=  \lambda (\lambda + \mu_1) $ for $i \in [s]$, then
\[
    S_x = \sum_{i = 1}^s x_i = s \big( (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big), \quad S_y = \sum_{i = 1}^s y_i = s \lambda (\mu_1 - 1 - \lambda), \quad S_z = \sum_{i = 1}^s z_i = s \lambda (\lambda + \mu_1),
\]
with $S_x - S_y - S_z = s (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1)$ and
\[
    T_x = \sum_{i = 1}^s x_i^2 = \sum_{i = 1}^s \big[  (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big]^2, \quad T_y = \sum_{i = 1}^s y_i^2 = s \lambda^2 (\mu_1 - 1 - \lambda)^2, \quad T_z = \sum_{i = 1}^s z_i^2 = s \lambda^2 (\lambda + \mu_1)^2,
\]
with $T_x + T_y + T_z = \sum_{i = 1}^s \big[  (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big]^2  + s \big[ \lambda^2 (\mu_1 - 1 - \lambda)^2 + \lambda^2 (\lambda + \mu_1)^2\big]$. For a standard Gaussian variable $Z$, we have Lemma \ref{lemma:gaussian_weight}, and
then we write the probability as
\[
    \begin{aligned}
        & Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right) \\
        = & \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \big( (2 \lambda + 1) R_{1, i} - (\mu_1 + \lambda) \big)\omega_i > \lambda (\mu_1 - 1 - \lambda) \sum_{i = 1}^s \omega_i' + \lambda (\lambda + \mu_1) \sum_{i = 1}^s \omega_i'' \right).
    \end{aligned}
\]
Hence, by applying the first inequality in Lemma \ref{lemma:gaussian_weight} and using the fact that $T_x + T_y + T_z \geq T_z \geq s \lambda^4$, we can get that
\[
    \begin{aligned}
        & Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s x_i \omega_i - \sum_{i = 1}^s y_i \omega_i' -   \sum_{i = 1}^s z_i \omega_i'' > 0 \right) \\
        = & \mathbb{P}_{Z} \left( Z > \frac{- (S_x - S_y - S_z)}{\sigma_{\omega} \sqrt{T_x + T_y + T_z}}\right) \\
        \geq & \frac{1}{2} \times \mathbb{I} \left( \frac{- (S_x - S_y - S_z)}{\sigma_{\omega} \sqrt{T_x + T_y + T_z}} < 0 \right) + \frac{1}{4} \exp \left\{ - \frac{(S_x - S_y - S_z)^2}{\sigma_{\omega}^2 (T_x + T_y + T_z)} \right\} \times  \mathbb{I} \left( \frac{- (S_x - S_y - S_z)}{\sigma_{\omega} \sqrt{T_x + T_y + T_z}} > 0 \right) \\
        \geq & \frac{1}{2} \times \mathbb{I} \left(S_x - S_y - S_z > 0 \right) + \frac{1}{4} \exp \left\{ - \frac{(S_x - S_y - S_z)^2}{\sigma_{\omega}^2 \cdot s \lambda^4} \right\} \times  \mathbb{I} \left( S_x - S_y - S_z \leq 0 \right) \\
        = & \frac{1}{2} \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > 0 \right) + \frac{1}{4} \exp \left\{ - \frac{s (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1 )^2}{\sigma_{\omega}^2\lambda^4}\right\} \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 \leq 0 \right).
    \end{aligned}
\]
where $Z$ again is the standard Gaussian variable and we will use this notation throughout this proof. Then this result combined with the fact \eqref{constant_bound} gives the upper bound for $a_{k, s}$ as follows:
\[
    \begin{aligned}
    	a_{k, s} & \leq 2 \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 > 0 \right) + 4  {\mathbb{E}} \exp \left\{ \frac{s (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1 )^2}{\sigma_{\omega}^2\lambda^4}\right\} \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 \leq 0 \right) \\
    	& \leq 2 + 4  {\mathbb{E}} \exp \left\{ \frac{s (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1 )^2}{\sigma_{\omega}^2\lambda^4}\right\}.
    \end{aligned}
\]
   
       
       
       
   
From the above expression, for proving $a_{k, s}$ is bounded by a constant free of $s$, it remains to show the expectation
\begin{equation}\label{a_k_constant_3}
	{\mathbb{E}} \exp \left\{ \frac{s (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1 )^2}{\sigma_{\omega}^2\lambda^4}\right\} 
\end{equation}
can be bounded below some constants free of $s$.

	



   
   
Applying Lemma \ref{lem_subGtosubE}, we know that if we take
\[
    \frac{s(2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^4} \leq \frac{s}{8 \sigma_1^2},
\]
then \eqref{a_k_constant_3} can be upper bounded by $e^{9 / 8}$. A sufficient condition for the above inequality is
\[
    \frac{s(2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^4} \leq \frac{s}{16 \sigma_1^2}, \qquad \lambda^2 - \frac{8\sigma_1}{\sigma_{\omega}} \lambda - \frac{4 \sigma_1}{\sigma_{\omega}} \geq 0,
\]
or say,
\begin{equation}\label{tuning_condition_exact}
	\lambda \geq \frac{4 \sigma_1}{\sigma_{\omega}} + \sqrt{\frac{4\sigma_1}{\sigma_{\omega}} \left( \frac{4 \sigma_1}{\sigma_{\omega}} + 1\right)}.
\end{equation}

Therefore, if we take the tuning parameters $\lambda$ and $\sigma_{\omega}$ as in \eqref{tuning_condition_exact}, then $a_{k, s}$ will be bounded by a constant such that 
\begin{equation}\label{a_k_constant_final}
	a_{k, s} \leq 2 + 4 e^{9 / 8}.
\end{equation}
   
       
   
Note that this step derives the tuning conditions for the tuning parameters in Theorem \ref{thm.MAB}. 
\end{proof}

\begin{lemma}[Bounding $a_{k, s, 1}$ at \eqref{def_a_k_s_1}]\label{lem_bounding_a_k_s_1}
    Take 
    \[
        s_{a, 1}(T) := \frac{4 \sigma_1^2}{C_1^2 (1 + 2 \lambda)^2 \Delta_k^2} \times\log T.
    \]
    Then for any $s \geq s_{a, 1}(T)$,
    \[
        a_{k, s, 1} \leq T^{-1},
    \]
    when $T \geq 2$.
\end{lemma}

\begin{proof}
    We will bound $a_{k, s, 1}$ by bounding the probability of the event $A_{k, s}^c$. Write
\begin{equation}\label{a_k_1_bound}
    a_{k, s, 1} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}^c)\Big] \leq {\mathbb{E}} \Big[ T  \mathbb{I}(A_{1, s}^c)\Big] = T \mathbb{P} (A_{1, s}^c) 
\end{equation}
Since the summation of independent sub-Gaussian variables is still sub-Gaussian, \eqref{lem_subGtosubE_1} in Lemma \ref{lem_subGtosubE} gives $\overline{R}_{1, s} - \mu_1 \sim \operatorname{subG} (\sigma_1^2 /  s)$. By the concentration inequality of the sub-Gaussian variable,
\[
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \left( \big| \overline{R}_{1, s} - \mu_1 \big| \geq (1 + 2\lambda) C_1 \Delta_k \right) \\
        & \leq 2 \exp \left\{ - \frac{(1 + 2\lambda)^2 C_1^2 \Delta_k^2 s}{2 \sigma_1^2}\right\} \\
        \alignedoverset{\text{take } s \geq s_{a, 1} (T) \geq  \begin{matrix} \frac{2 \sigma_1^2}{C_1^2 (1 + 2 \lambda)^2 \Delta_k^2} \times\log (\sqrt{2} T) \end{matrix}}{\leq} \frac{1}{T^2}.
    \end{aligned}
\]
Hence, we obtain that
\[
    a_{k, s, 1} \leq  T \mathbb{P}(A_{1, s}^c) \leq T^{-1}, \qquad \text{ for any } s \geq s_{a, 1} (T) \asymp \log T.
\]
\end{proof}

\begin{lemma}[Bounding $a_{k, s, 2}$ at \eqref{def_a_k_s_2}]\label{lem_bounding_a_k_s_2}
    Take 
    \[
        \begin{aligned}
            s_{a, 2}(T) := \Bigg[ \big[ \Lambda_1 + \mathfrak{a}_1 + \mathfrak{b}_1 + \Lambda_1\mathfrak{a}_1 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_1}}{2}  + \frac{\sqrt{2 \pi} \Lambda_1 \mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right]  \right\}  + 1\right)\\
            & \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T,
        \end{aligned}
    \]
    where 
    \[
        \mathfrak{a}_1 = \frac{4 \sigma_{\omega}^2 \lambda^2}{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2} , \qquad \mathfrak{b}_1 = \frac{2 \sigma_{\omega}^2 \left[ 2 \lambda^2 \sigma_1^2 + (1 + 2 \lambda)^2 C_2^2 \Delta_k^2 \right]}{3 C_2^2 (1 + 2 \lambda)^2 \Delta_k^2}, \qquad \Lambda_1 = 8 \sqrt{2} \sigma_1^2.
    \]
    Then for any $s \geq s_{a, 2}(T)$,
    \[
        a_{k, s, 2} \leq T^{-1},
    \]
    for any $T \geq 2$.
\end{lemma}

\begin{proof}
    We will bound the probability of $G_{1, s}^c$ conditioning on $\mathcal{H}_{1, s}$ to prove this lemma.
   
Note that 
\[
    \begin{aligned}
    	& {\mathbb{E}}_{\omega, \omega', \omega''} \left[ \sum_{i = 1}^s \big[ (1 + 2\lambda) R_{1, i} - (\overline{R}_{1, s} + \lambda)\big] \omega_i + \sum_{i = 1}^s \lambda (1 + \lambda - \overline{R}_{1, s}) \omega_i' - \sum_{i = 1}^s \lambda (\overline{R}_{1, s} + \lambda) \omega_i'' \right] \\
    	 = & s \big( (1 + 2\lambda) \overline{R}_{1, s} - (\overline{R}_{1, s} + \lambda) + \lambda (1 + \lambda - \overline{R}_{1, s}) - \lambda (\overline{R}_{1, s} + \lambda) \big) = 0,
    \end{aligned}
\]
we can bound the tail probability $\mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s})$ by
\begin{equation}\label{pr_G_bound}
    \begin{aligned}
        & \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s})\\
        = & \mathbb{P} \big( |\overline{Y}_{k', s} - \overline{R}^*_{k', s}| > C_3 \Delta_k \mid \mathcal{H}_{1, s}\big)\\
        = & 2 \mathbb{P}_{\omega, \omega', \omega''}  \left\{ \left| \frac{\sum_{i = 1}^s \big[ (1 + 2\lambda) R_{1, i} - (\overline{R}_{1, s} + \lambda)\big] \omega_i + \sum_{i = 1}^s \lambda (1 + \lambda - \overline{R}_{1, s}) \omega_i' - \sum_{i = 1}^s \lambda (\overline{R}_{1, s} + \lambda) \omega_i''}{(1 + 2 \lambda)(\sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i'')} \right| > C_2 \Delta_k \right\} \\
        \alignedoverset{\text{by Lemma \ref{lem_gau_frac}}}{\leq} 2 \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s \big[ (1 + 2\lambda) R_{1, i} - (\overline{R}_{1, s} + \lambda)\big] \omega_i + \sum_{i = 1}^s \lambda (1 + \lambda - \overline{R}_{1, s}) \omega_i'  \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - \sum_{i = 1}^s \lambda (\overline{R}_{1, s} + \lambda) \omega_i'' - C_2 (1 + 2\lambda) \Delta_k \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] > 0\Bigg\} \\
        & ~~~~~~~~~~  + \mathbb{P}_{\omega, \omega', \omega''} \left\{ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i'' < 0\right\} = 2I + II,
    \end{aligned}
\end{equation}
where 
\[
    \begin{aligned}
        I & = \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s \big[ (1 + 2\lambda) R_{1, i} - (\overline{R}_{1, s} + \lambda)\big] \omega_i + \sum_{i = 1}^s \lambda (1 + \lambda - \overline{R}_{1, s}) \omega_i'  \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - \sum_{i = 1}^s \lambda (\overline{R}_{1, s} + \lambda) \omega_i'' - C_2 (1 + 2\lambda) \Delta_k \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] > 0\Bigg\}.
    \end{aligned}
\]
and 
\[
    II = \mathbb{P}_{\omega, \omega', \omega''} \left\{ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i'' < 0\right\}.
\]
For bounding $\mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s})$, it suffices to bound $I^{(1)}$ and $II^{(2)}$ separately. we first study $II^{(1)}$. Write
\[
    \begin{aligned}
    	II & = \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i'' < 0\right) \\
    	& = \mathbb{P}_{\omega, \omega', \omega''} \left( \frac{\sum_{i = 1}^s (\omega_i - 1) + \lambda \sum_{i = 1}^s (\omega_i' - 1) + \lambda \sum_{i = 1}^s (\omega_i'' - 1)}{\sqrt{s \sigma_{\omega}^2 (1 +2 \lambda^2)}} < - \frac{(1 + 2\lambda) s}{\sqrt{s \sigma_{\omega}^2 (1 +2 \lambda^2)}} \right) \\
    	& = \mathbb{P}_{\omega, \omega', \omega''} \left( Z < - \frac{(1 + 2\lambda) }{\sigma_{\omega} \sqrt{1 + 2 \lambda^2}} \times \sqrt{s} \right) \\
    	\alignedoverset{\text{by Lemma \ref{lemma:gaussian_weight}}}{\leq} \frac{1}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\}.
    \end{aligned}
\]
For studying $I^{(1)}$, we define the following functions of $\{ R_{1,i}\}_{i = 1}^s$:
\[
   f \big(\{ R_{1,i}\}_{i = 1}^s\big) = f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) + f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) - f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big)
\]
with 
\[
    \begin{aligned}
    	f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) & = \sum_{i = 1}^s \big[ (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) - (1 + 2 \lambda ) C_2 \Delta_k  \big] \omega_i,
    \end{aligned}
\]
\[
    f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) = \sum_{i = 1}^s \big[ \lambda (\lambda + 1  - \overline{R}_{1, s}) - (1 + 2 \lambda ) C_2 \Delta_k  \big] \omega_i',
\]
and
\[
    f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big) = \sum_{i = 1}^s \big[ \lambda (\overline{R}_{1, s} + \lambda) + (1 + 2 \lambda ) C_2 \Delta_k  \big] \omega_i''.
\]
   
   
   
Then we can write $I^{(1)} = 2 \mathbb{P}_{\omega, \omega', \omega'} (f_1 + f_2 - f_3 > 0)$. Note that $f_1, f_2$, and $f_3$ are mutually independent conditioning on $\mathcal{H}_{1, s}$. Then 
the expectation is
\[
    {\mathbb{E}} \big[ f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big] = - 3 C_2 (1 + 2 \lambda) \Delta_k s,
\] 
and the variance is
\begin{equation}\label{bound_var_f}
    \begin{aligned}
        & \operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s} \big) \\
        = & \sigma_{\omega}^2 \Bigg [ \sum_{i = 1}^s \big[ (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) - (1 + 2 \lambda ) C_2 \Delta_k  \big]^2 \\
        & \qquad \qquad \qquad + \sum_{i = 1}^s \big[ \lambda (\lambda + 1  - \overline{R}_{1, s}) - (1 + 2 \lambda ) C_2 \Delta_k  \big]^2 + \sum_{i = 1}^s \big[ \lambda (\overline{R}_{1, s} + \lambda) + (1 + 2 \lambda ) C_2 \Delta_k  \big]^2 \Bigg] \\ 
        = & \sigma_{\omega}^2 \Big[ V_1 + V_2 + V_3 \Big],
    \end{aligned}
 \end{equation}

 where 
 \[
     V_1 = \sum_{i = 1}^s \big[ (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) - (1 + 2 \lambda ) C_2 \Delta_k  \big]^2,
 \]
 \[
      V_2 = \sum_{i = 1}^s \big[ \lambda (\lambda + 1  - \overline{R}_{1, s}) - (1 + 2 \lambda ) C_2 \Delta_k  \big]^2,
 \]
 and
 \[
      V_3 = \sum_{i = 1}^s \big[ \lambda (\overline{R}_{1, s} + \lambda) + (1 + 2 \lambda ) C_2 \Delta_k  \big]^2.
 \]
 For bounding the conditional variance above, we calculate its components as follows. 
 \begin{equation}\label{bound_I}
 	\begin{aligned}
 	    V_1 & = \sum_{i = 1}^s  \big[ (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) \big]^2  + s \lambda^2 (\lambda + 1  - \overline{R}_{1, s})^2 + s \lambda^2 (\overline{R}_{1, s} + \lambda)^2 \\
 	    & = \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s})^2 + 2 s \lambda^2 \big( 3 \overline{R}_{1, s}^2 - 3 \overline{R}_{1, s} + \lambda^2 + \lambda + 1 \big) \\
 	    & = \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + (6 \lambda^2 - 1)s (\overline{R}_{1, s} - \mu_1)^2 + 6  s \lambda^2 (2 \mu_1 - 3)  (\overline{R}_{1, s} - \mu_1) + 2 s \lambda^2 \big[\lambda^2 + \lambda + 1 + 3 \mu_1 (\mu_1 - 1) \big] \\
 	    \alignedoverset{\text{Cauchy inequality}}{\leq} 6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2  + 6  s \lambda^2 (2 \mu_1 - 1)  (\overline{R}_{1, s} - \mu_1) + 2 s \lambda^2 \big( \lambda^2 + \lambda + 1 - \mu_1 (1 - \mu_1) \big) \\
 	    & = 6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + 6  s \lambda^2 (2 \mu_1 - 1)  (\overline{R}_{1, s} - \mu_1)  + 2 s \lambda^2 \left( \lambda^2 + \lambda + 1 - \mu_1 (1 - \mu_1)\right),
 	\end{aligned}
 \end{equation}
 \[
     \begin{aligned}
         V_2 & = - 2 (1 + 2\lambda) C_3 \Delta_k \bigg[ \sum_{i = 1}^s  \big( (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) \big) + s \lambda (\lambda + 1  - \overline{R}_{1, s}) -  s \lambda (\overline{R}_{1, s} + \lambda) \bigg] \\
         & = - 2 s (1 + 2\lambda) C_3 \Delta_k \Big[ \big(\overline{R}_{1, s} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) + \lambda (\lambda + 1  - \overline{R}_{1, s}) \big) -  \lambda (\overline{R}_{1, s} + \lambda) \Big] = 0,
     \end{aligned}
 \]
 and
$$V_3= 3 s (1 + 2\lambda)^2 C_3^2 \Delta_k^2. $$ 
Therefore, the conditional variance in \eqref{bound_var_f} is bounded by
 \begin{equation}\label{bound_var_f_new}
 	\begin{aligned}
 		 \sigma_{\omega}^{-2}\operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s} \big) & \leq I^{(2)} + II^{(2)} + III^{(2)} \\
 		 & \leq  \underbrace{6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + 6 s \lambda^2 (2\mu_1 - 1) \left( \overline{R}_{1, s} - \mu_1\right)}_{\text{the random part}} \\
 		 & ~~~~~~~~~~~~~~~~~~~ + \underbrace{2 s \lambda^2 \left( \lambda^2 + \lambda + 1 - 3 \mu_1 (1 - \mu_1)\right) + 3 s (1 + 2\lambda)^2 C_2^2 \Delta_k^2}_{\text{the determined part}} \\
 		 & = \mathfrak{R}_1 + \mathfrak{R}_2 + \mathfrak{D},
 	\end{aligned}
 \end{equation}
where
 \[
     \mathfrak{R}_1 = 6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2, \qquad \mathfrak{R}_2 =  6 s \lambda^2 (2\mu_1 - 1) \left( \overline{R}_{1, s} - \mu_1\right) + 2 s \lambda^2 \left( \lambda^2 + \lambda + 1 - 3 \mu_1 (1 - \mu_1)\right),
 \]
and
\[
    \mathfrak{D} = 3 s (1 + 2\lambda)^2 C_2^2 \Delta_k^2.
\]
Obviously, $\mathfrak{R}_1$ and $\mathfrak{R}_2$ are strictly positive. Furthermore, for $\mathfrak{R}_2$, we can also prove it is non-positive with a high probability.

    
Indeed, note that $0 \leq 3 \mu_1 (1 - \mu_1) \leq \frac{3}{4}$, we have
 \begin{equation}\label{R_2_pr}
     \begin{aligned}
     	\mathbb{P} ( \mathfrak{R}_2 > 0) & = \mathbb{P} \left( 3 (2 \mu_1 - 1)\left( \overline{R}_{1, s} - {\mu_1} \right) > \lambda^2 + \lambda + 1 - 3 \mu_1 (1 - \mu_1)\right) \\
     	& \leq \mathbb{P} \left( \Big| 3 (2 \mu_1 - 1)\left( \overline{R}_{1, s} - {\mu_1} \right)\Big| > \Big| \lambda^2 + \lambda + 1 - 3 \mu_1 (1 - \mu_1) \Big| \right) \mathbb{I}\left( \mu_1 \neq  \frac{1}{2}\right) + \mathbb{P} (\varnothing) \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \\
     	\alignedoverset{0 \leq 3 \mu_1 (1 - \mu_1) \leq \frac{3}{4}}{\leq} \mathbb{P} \left( \Big| \overline{R}_{1, s} - {\mu_1} \Big| > \frac{\lambda^2 + \lambda + 1 / 4}{|3 (2 \mu_1 - 1)|} \right) \mathbb{I}\left( \mu_1 \neq  \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \\
     	\alignedoverset{\text{by sub-Gaussian inequality}}{\leq} \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right).
     \end{aligned}
 \end{equation}


    
       
       
       
       
   
   
   
   
       
       
       
       
       
       
       
       
       
       
       
       
   

   
       
       
       
       
       
       
       
       
       
       
   
   
       
       
       
       
       
       
       
       
       
       
       
       
       
   
   
   
       
       
       
   


   
       
       
       
       
       
   
Then by applying Lemma \ref{lemma:gaussian_weight} again,
\[
    \begin{aligned}
        \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s}) & = 2 I + II\\
        & = 2 \mathbb{P} \left(\frac{f_1 + f_2 - f_3 - {\mathbb{E}} \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{\sqrt{\operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}} > - \frac{{\mathbb{E}} \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{\sqrt{\operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}} \right) \\
        & ~~~~~~~~~~~ + \frac{1}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} \\
        & \leq 2 \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)} \right\} + \frac{1}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\}.
       
    \end{aligned}
\]
Therefore, combining with \eqref{bound_var_f_new}, 
$a_{k, s, 2}$ will be bounded by 
\[
    \begin{aligned}
        a_{k, s, 2} & \leq {\mathbb{E}} \Big[ T  \mathbb{I}(G_{1, s}^c)\Big] =  T {\mathbb{E}} \Big[ \mathbb{P}(G_{1, s}^c \mid \mathcal{H}_{1, s}) \Big] \\
        \alignedoverset{\text{by \eqref{pr_G_bound}}}{\leq} \leq T \big[ 2 I + II \big] \\
        & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)} \right\} + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} \\
        & = T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \operatorname{var} \big(f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)} \right\} \big( \mathbb{I} (\mathfrak{R}_2 \leq 0) + \mathbb{I}(\mathfrak{R}_2 > 0) \big) + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\}\\
        \alignedoverset{\text{by the decomposition \eqref{bound_var_f_new}}}{\leq} T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \sigma_{\omega}^2 \big(\mathfrak{R}_1 + \mathfrak{R}_2 + \mathfrak{D} \big)}\right\} \mathbb{I} (\mathfrak{R}_2 \leq 0) + T \mathbb{P} (\mathfrak{R}_2 > 0) +  \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\}\\
        & \leq T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \sigma_{\omega}^2 \big(\mathfrak{R}_1 + \mathfrak{D} \big)}\right\} + T \mathbb{P} (\mathfrak{R}_2 > 0) + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\}.
    \end{aligned}
\]
Recall $R_{1, i} - \mu_1 \sim \operatorname{subG}(\sigma_1^2)$, and then $(R_{1, i} - \mu_1)^2 - \operatorname{var} (R_{1, i}) := \xi_i \sim \operatorname{subE}(8 \sqrt{2} \sigma_1^2)$.
   
Therefore, for furthermore bounding $\mathfrak{R}_1 + \mathfrak{D}$, we have
\begin{equation}\label{G_function_bound}
	\begin{aligned}
		\mathfrak{R}_1 + \mathfrak{D} & = 6 \lambda^2 \left[ \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 - s \operatorname{var} (R_{1, i}) \right] + \left[ s \operatorname{var} (R_{1, i}) + 3 s(1 + 2\lambda)^2 C_2^2 \Delta_k^2\right] \\
             & = 6 \lambda^2 \sum_{i = 1}^s \xi_i + s \left[ 6 \lambda^2\operatorname{var} (R_{1, i}) + 3 (1 + 2\lambda)^2 C_2^2 \Delta_k^2\right] \\
	
		\alignedoverset{\text{by $\operatorname{var}(R_{1, i}) \leq \sigma_1^2$}}{\leq}  6\lambda^2 \sum_{i = 1}^s \xi_i + 3 s \left[ 2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2\right]  \\
        & = 3 s \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right],
	\end{aligned}
\end{equation}
   
       
       
       
       
       
   
   
       
       
   
where $\overline{\xi} = \frac{1}{s} \sum_{i = 1}^s \xi_i$. Then
\begin{equation}\label{a_2_bound}
	\begin{aligned}
		a_{k, s, 2} & \leq T {\mathbb{E}} \exp \left\{ - \frac{{\mathbb{E}}^2 \big( f_1 + f_2 - f_3 \mid \mathcal{H}_{1, s}\big)}{2 \sigma_{\omega}^2 \big(\mathfrak{R}_1 + \mathfrak{D} \big)}\right\} + T \mathbb{P} (\mathfrak{R}_2 > 0) + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} \\
		\alignedoverset{\text{by \eqref{G_function_bound} and \eqref{R_2_pr}}}{\leq} T {\mathbb{E}}\exp \left\{ - \frac{9 C_2^2 (1 + 2\lambda)^2 \Delta_k^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right] }\right\} \\
		 & ~~~~~~~~~~~~~~~ + T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \right] \\
           & ~~~~~~~~~~~~~~~ + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} \\
		 & = T {\mathbb{E}}\exp \left\{ - \frac{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2 s }{ 2 \sigma_{\omega}^2  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right] }\right\} \\
		 & ~~~~~~~~~~~~~~~ + T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \right] \\
          & ~~~~~~~~~~~~~~~ + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} .
	\end{aligned}
\end{equation}
The next step is to apply Lemma \ref{lem_subE_ineq}. Take 
\[
     \mathfrak{a}_1 = \frac{4 \sigma_{\omega}^2 \lambda^2}{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2} , \qquad \mathfrak{b}_1 = \frac{2 \sigma_{\omega}^2 \left[ 2 \lambda^2 \sigma_1^2 + (1 + 2 \lambda)^2 C_2^2 \Delta_k^2 \right]}{3 C_2^2 (1 + 2 \lambda)^2 \Delta_k^2}, \qquad \lambda_i \equiv 8 \sqrt{2} \sigma_1^2, \qquad i \in [s],
\]
and $\Lambda_1 = (s^{-1}\sum_{i = 1}^s \lambda_i^2)^{1 / 2} = 8 \sqrt{2} \sigma_1^2$, then Lemma \ref{lem_subE_ineq} gives 
\begin{equation}\label{app_sub_E_ineq_1}
    \begin{aligned}
        & {\mathbb{E}}\exp \left\{ - \frac{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2 s }{ 2 \sigma_{\omega}^2  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right] }\right\} \\
        & = {\mathbb{E}} \exp \left\{ - \frac{s}{\frac{4 \sigma_{\omega}^2 \lambda^2}{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2} \overline{\xi}  + \frac{2 \sigma_{\omega}^2 \left[ 2 \lambda^2 \sigma_1^2 + (1 + 2 \lambda)^2 C_2^2 \Delta_k^2 \right]}{3 C_2^2 (1 + 2 \lambda)^2 \Delta_k^2}}\right\} = {\mathbb{E}} \exp \left\{ - \frac{s}{\mathfrak{a}_1 \overline{\xi} + \mathfrak{b}_1} \right\} \\
        & \leq \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_1}}{2} + \frac{\sqrt{2 \pi} \Lambda_1 \mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right]  \exp \bigg\{ - \frac{s}{\sqrt{2 \Lambda_1\mathfrak{a}_1} \vee (\mathfrak{b}_1 + \Lambda_1\mathfrak{a}_1)}\bigg\}  \\
        \alignedoverset{\text{by } \sqrt{2 \Lambda_1\mathfrak{a}_1} \vee (\mathfrak{b}_1 + \Lambda_1\mathfrak{a}_1) \leq \mathfrak{b}_1 + \Lambda_1\mathfrak{a}_1 + \Lambda_1+ \mathfrak{a}_1}{\leq} \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_1}}{2}  + \frac{\sqrt{2 \pi} \Lambda_1\mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right]  s \exp \bigg\{ - \frac{s}{\Lambda_1+ \mathfrak{a}_1 + \mathfrak{b}_1 + \Lambda_1\mathfrak{a}_1}\bigg\}.
       
       
    \end{aligned}
\end{equation}
Hence, by taking 
\[
    \begin{aligned}
        s & \geq \big[ \Lambda_1+ \mathfrak{a}_1 + \mathfrak{b}_1 + \Lambda_1\mathfrak{a}_1 \big] \left(\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_1}}{2}  + \frac{\sqrt{2 \pi} \Lambda_1\mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right]  \right\}  + 3\right) \log T \\
        & \geq \big[ \Lambda_1+ \mathfrak{a}_1 + \mathfrak{b}_1 + \Lambda_1\mathfrak{a}_1 \big] \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_1}}{2}  + \frac{\sqrt{2 \pi} \Lambda_1\mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right] T^3 \right\},
    \end{aligned}
\]
we have
\[
    {\mathbb{E}}\exp \left\{ - \frac{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2 s }{ 2 \sigma_{\omega}^2  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right] }\right\}  \leq \frac{s}{3 T^3} \leq \frac{1}{3T^2}.
\]
   
   
   
Similarly, 
\[
    T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \right] \leq \frac{1}{3T}, 
\]
and
\[
    \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} \leq \frac{1}{3T}
\]
have the solution
\[
    s \geq \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \left[ \log 3 + 2 \log T \right].
\]
and
\[
    s \geq \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \left[ \log (3 / 2) + 2 \log T \right]
\]
respectively.
Therefore, one will have $a_{k, s, 2} \leq \frac{1}{T}$  if we take 
\[
    \begin{aligned}
        s_{a, 2}(T) = \Bigg[ \big[ \Lambda_1+ \mathfrak{a}_1 + \mathfrak{b}_1 + \Lambda_1\mathfrak{a}_1 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_1}}{2}  + \frac{\sqrt{2 \pi} \Lambda_1\mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right]  \right\}  + 1\right)\\
        &  \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T
    \end{aligned}
\]
for any $T \geq 2$.
\end{proof}

\begin{lemma}[Bounding $a_{k, s, 3}$ at \eqref{def_a_k_s_3}]\label{lem_bounding_a_k_s_3}
    Take 
    \[
        \begin{aligned}
             s_{a, 3}(T) = \Bigg[ \big[ \Lambda_1+ \mathfrak{a}_2 + \mathfrak{b}_2 + \Lambda_1\mathfrak{a}_2 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_2}}{2}  + \frac{\sqrt{2 \pi} \Lambda_1\mathfrak{a}_2}{2 \mathfrak{b}_2^2} + \frac{2 \mathfrak{a}_2}{\mathfrak{b}_2} \right]  \right\}  + 1\right) \\
             & \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T
        \end{aligned}
    \]
    where 
    \[
        \mathfrak{a}_2 = \frac{\sigma_{\omega}^2 \lambda^2}{3 (\lambda - 1)^2 C_1^2 \Delta_k^2}, \qquad \mathfrak{b}_2 = \frac{\sigma_{\omega}^2\left[ 2 \lambda^2 \sigma_1^2 + 25 (1 + 2\lambda)^2 C_1^2 \Delta_k^2 \right]}{6 (\lambda - 1)^2 C_1^2 \Delta_k^2}, \qquad \Lambda_1= 8 \sqrt{2} \sigma_1^2.
    \]
    Then for any $s \geq s_{a, 3}(T)$ and $\lambda > 1$,
    \[
        a_{k, s, 3} \leq T^{-1}.
    \]
    for any $T \geq 2$.
\end{lemma}

\begin{proof}
    Different from the proofs for bounding $a_{k, s, 1}$ and $a_{k, s, 2}$, in which we control the probability of the bad events,
$a_{k, s, 3}$ is defined on good event instead. We need a different technique to deal with $a_{k, s, 3}$. Note that 
\begin{equation}\label{a_k_3_sw}
    \begin{aligned}
        \{N_{1, s}(\tau_{k})<T^{-1}\} & \supseteq \{N_{1, s}(\tau_{k}) < (T^{2}-1)^{-1}\} \\
        \alignedoverset{\text{by } (T^2 - 1)^{-1} \leq T^{-1}}{=} \big\{Q_{1, s}\left(\tau_{k}\right)^{-1}<1+\left(T^{2}-1\right)^{-1}\big\} \\
        & = \big\{Q_{1, s}(\tau_{k})^{-1} < (1-T^{-2})^{-1}\big\} = \big\{\left[1-Q_{1, s}(\tau_{k})\right]<T^{-2}\big\},
    \end{aligned}
\end{equation}
and $a_{k, s, 3} \leq {\mathbb{E}} \big[ N_{1, s} (\tau_k) \mathbb{I} (A_{1, s} \cap G_{1, s}) \big]$. 
Thus, similar to 
\citet{wang2020residual}, to bound $a_{k, s, 3} \leq T^{-1}$ for $s \geq s_{a. 3}$, it is sufficient to find $s \geq s_{a,3}$ satisfying $\big\{[1-Q_{1,s}(\tau_{k})]<T^{-2}\big\}$ on the event $\{A_{1, s} \cap G_{1, s}\}$.

We write 
\[
    \big[1-Q_{1, s}(\tau_{k})\big]  \mathbb{I} (A_{1, s})\mathbb{I}  (G_{1,s}) = \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Gamma_1 \, | \, \mathcal{H}_{1, s} \big)  \mathbb{I} (A_{1, s} \, \cap \, G_{1, s})
\]
with $\Gamma_1 := \Gamma_1(\tau_1) = \tau_1 - \overline{R}_{1, s}^*$ measuring the difference between $\overline{Y}_{1, s}$ and $\overline{R}_{1,s}^*$. 
On the event $A_{1, s} \, \cap \, G_{1, s}$, $\Gamma_1$ can be bounded in an interval.
To see this, 
\[
    \begin{aligned}
    	A_{1, s} & = \bigg\{ \overline{R}_{1, s}^* - \frac{\mu_1 + \lambda}{1 + 2\lambda} > - C_1 \Delta_k \bigg\} \, \cap \, \bigg\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2 \lambda}  < C_1 \Delta_k \bigg\} \\
    	& = \bigg\{ \Gamma_1 < C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} + \tau_1 \bigg\} \, \cap \, \bigg\{ \Gamma_1 > - C_2 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} + \tau_1 \bigg\} \\
    	& = \bigg\{ \tau_1 - C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda}  < \Gamma_1 < \tau_1 + C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda}  \bigg\}
    \end{aligned}
\]
and
\[
    \begin{aligned}
        G_{1, s} & = \big\{ - C_2 \Delta_k \leq \overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq C_2 \Delta_k \big\} \\
        & = \big\{ \tau_1 -C_2 \Delta_k \leq \overline{Y}_{1, s} + \Gamma_1 \leq \tau_1 + C_2 \Delta_k \big\} \\
        & = \big\{ \tau_1 - C_2 \Delta_k - \overline{Y}_{1, s} \leq \Gamma_1 \leq \tau_1 + C_2 \Delta_k - \overline{Y}_{1, s}\big\} \\
        \alignedoverset{\text{by } - C_2 \Delta_k + \overline{R}_{1, s}^* \leq \overline{Y}_{1, s}  \leq C_2 \Delta_k + \overline{R}_{1, s}^*}{\subseteq} \big\{ \tau_1 - 2 C_2 \Delta_k - \overline{R}_{1, s}^* \leq \Gamma_1 \leq \tau_1 + 2 C_2 \Delta_k - \overline{R}_{1, s}^* \big\}.
       
       
    \end{aligned}
\]
Then combine the above two results and take 
\begin{equation}\label{C_1_2}
    C_1 = 2 C_2,
\end{equation}
we will have
   
       
       
       
       
   
\begin{equation}\label{Gamma_bound}
    \begin{aligned}
    	& A_{1, s} \, \cap \, G_{1, s}  \\
     & \subseteq \bigg\{ \tau_1 - C_1 \Delta_k - \overline{R}_{1, s}^* \wedge \frac{\mu_1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 \leq \tau_1 + C_1 \Delta_k - \overline{R}_{1, s}^* \vee \frac{\mu_1 + \lambda}{1 + 2 \lambda}  \bigg\} \\
    	\alignedoverset{\text{by} \frac{\mu_1 +  \lambda}{1 + 2 \lambda} - C_1 \Delta_k  < \overline{R}_{1, s}^* < \frac{\mu_1 +  \lambda}{1 + 2 \lambda} + C_1 \Delta_k }{\subseteq} \bigg\{\tau_1 - 2 C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 \leq \tau_1 + 2 C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \bigg\}.
    
    
    
    \end{aligned}
\end{equation}
    
   
   
   
   
   
Now take
\[
    \tau_1 :=  \frac{\mu_1 + \lambda}{1 + 2\lambda} - \frac{6 \lambda C_1 \Delta_k}{1 + 2\lambda}.
\]
then $ \tau_1 \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda}$, and thus this choice satisfying \eqref{requirement_0}.
We furthermore get on $A_{1, s} \, \cap \, G_{1, s}$, 
\[
    \begin{aligned}
        \Gamma_1 & \leq \tau_1 + 2 C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} \\
        & = 2 C_1 \Delta_k - \frac{6 \lambda C_1 \Delta_k}{1 + 2\lambda} = - 2 C_1 \Delta_k \frac{\lambda - 1}{2 \lambda + 1} \\
        \alignedoverset{\text{by } \lambda \geq 1}{\leq} - \frac{2 C_1 (\lambda - 1)  \Delta_k}{2 \lambda + 1} < 0,
    \end{aligned}
\]
and
\[
    \begin{aligned}
        0 > \Gamma_1 & \geq \tau_1 - 2 C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} \\
        & = - 2 C_1 \Delta_k - \frac{6 \lambda C_1 \Delta_k}{1 + 2\lambda} = - 2 C_1 \Delta_k \left[ 1 + \frac{3 \lambda}{1 + 2 \lambda}\right] \\
        \alignedoverset{\text{by } \frac{3 \lambda}{1 + 2\lambda} \leq \frac{3}{2}}{\geq} - 2 C_1 \Delta_k \cdot \frac{5}{2} = - 5 C_1 \Delta_k.
    \end{aligned}
\]
Thus, we get $ - \Gamma_1 \in \left( \frac{2 C_1 (\lambda - 1)  \Delta_k}{2 \lambda + 1}, 5 C_1 \Delta_k \right) \subseteq (0, + \infty)$. Return to the quantity we care about, $ \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s})$, we write it as 
\[
    \begin{aligned}
        & \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s}) \\
        = & \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Gamma_1 \, | \, \mathcal{H}_{1, s} \big)  \mathbb{I} (A_{1, s} \, \cap \, G_{1, s})\\
        = & {\mathbb{E}} \Bigg[ \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) \omega_i + \lambda \sum_{i = 1}^s (2 R_{1, i} \omega_i - \overline{R}_{1, s} \omega_i' - \overline{R}_{1, s} \omega_i'') 
        \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s (\omega_i' - \omega''_i) \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - \Gamma_1 (1 + 2\lambda) \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] < 0\Bigg\} \mathbb{I} (A_{1, s} \, \cap \, G_{1, s}) \Bigg] .
    \end{aligned}
\]
Similar to the technique we use to bound the conditional probability $\mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s})$, we just need to replace $C_2 \Delta_k$ by $-\Gamma_1$ in \eqref{a_2_bound} and get that 
\begin{equation}\label{Gamma_1_bound_next}
    \begin{aligned}
         & \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I} (A_{1, s}) \mathbb{I}(G_{1, s}) \\
         \alignedoverset{\text{apply steps in  \eqref{pr_G_bound}}}{\leq} T {\mathbb{E}} \Bigg[ \exp \left\{ - \frac{9 (- \Gamma_1)^2 (1 + 2 \lambda)^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + (- \Gamma_1)^2 (1 + 2 \lambda)^2 \right] }\right\}  \mathbb{I} (A_{1, s} \, \cap \, G_{1, s}) \Bigg] \\
         & ~~~~~~~~~~~~~~~ + T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \right] \\
          & ~~~~~~~~~~~~~~~ + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} \\
         \alignedoverset{\text{by the bound of } -\Gamma_1}{\leq} T {\mathbb{E}}\exp \left\{ - \frac{36 (\lambda - 1)^2 C_1^2 \Delta_k^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + 25 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2  \right] }\right\}  \\
         & ~~~~~~~~~~~~~~~ + T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}\right\} \times \mathbb{I}\left( \mu_1 \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_1 = \frac{1}{2}\right) \right] \\
          & ~~~~~~~~~~~~~~~ + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} .
        
        
    \end{aligned}
\end{equation}
   
       
       
       
       
       
   
Similarly, we define
\[
    \mathfrak{a}_2 = \frac{\sigma_{\omega}^2 \lambda^2}{3 (\lambda - 1)^2 C_1^2 \Delta_k^2}, \qquad \mathfrak{b}_2 = \frac{\sigma_{\omega}^2\left[ 2 \lambda^2 \sigma_1^2 + 25 (1 + 2\lambda)^2 C_1^2 \Delta_k^2 \right]}{6 (\lambda - 1)^2 C_1^2 \Delta_k^2}.
\]
Take
\[
    s \geq \big[ \Lambda_1+ \mathfrak{a}_2 + \mathfrak{b}_2 + \Lambda_1\mathfrak{a}_2 \big] \left(\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_2}}{2}  + \frac{\sqrt{2 \pi} \Lambda_1\mathfrak{a}_2}{2 \mathfrak{b}_2^2} + \frac{2 \mathfrak{a}_2}{\mathfrak{b}_2} \right]  \right\}  + 3\right) \log T,
\]
then applying Lemma \ref{lem_subE_ineq} and the steps in \eqref{app_sub_E_ineq_1} again, 
\[
    
     \begin{aligned}
         & T {\mathbb{E}}\exp \left\{ - \frac{36 (\lambda - 1)^2 C_1^2 \Delta_k^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + 25 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2  \right] }\right\} \\
         & = T {\mathbb{E}}\exp \left\{ - \frac{6 (\lambda - 1)^2 C_1^2 \Delta_k^2 s}{\sigma_{\omega}^2 \left[ 2 \lambda^2 \overline{\xi} +  2 \lambda^2 \sigma_1^2 + 25 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2  \right] }\right\} \\
         & = T {\mathbb{E}} \exp \left\{- \frac{s}{\frac{\sigma_{\omega}^2 \lambda^2}{3 (\lambda - 1)^2 C_1^2 \Delta_k^2} \overline{\xi} + \frac{\sigma_{\omega}^2\left[ 2 \lambda^2 \sigma_1^2 + 25 (1 + 2\lambda)^2 C_1^2 \Delta_k^2 \right]}{6 (\lambda - 1)^2 C_1^2 \Delta_k^2}} \right\} = T {\mathbb{E}} \exp \left\{ -\frac{s}{\mathfrak{a}_2 \overline{\xi} + \mathfrak{b}_2}\right\} \\
         \alignedoverset{\text{by applying steps in \eqref{app_sub_E_ineq_1}}}{\leq} \frac{1}{3T}
     \end{aligned}
\]
Therefore, to find $s$ such that $[1 - Q_{1, s} (\tau_k)] < T^{-2}$ on $\{ A_{1, s} \, \cap \, G_{1, s}\}$, we let
\[
   \begin{aligned}
       s \geq s_{a, 3}(T) = \Bigg[ \big[ \Lambda_1+ \mathfrak{a}_2 + \mathfrak{b}_2 + \Lambda_1\mathfrak{a}_2 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_2}}{2}  + \frac{\sqrt{2 \pi} \Lambda_1\mathfrak{a}_2}{2 \mathfrak{b}_2^2} + \frac{2 \mathfrak{a}_2}{\mathfrak{b}_2} \right]  \right\}  + 1\right) \\
       & \vee \frac{18 \sigma_{\omega}^2 (2 \mu_1 - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2} \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T,
      
   \end{aligned}
\]
and then we get that $a_{s, 3} \leq T^{-1}$.
\end{proof}

\subsection{Lemmas on bounding $b_k$.}

\begin{lemma}[Bounding $b_{k, s, 1}$ at \eqref{def_b_k_s_1}]\label{lem_bounding_b_k_s_1}
   
    Take 
    \[
        s_{b, 1} (T) = \frac{2 \sigma_k^2}{C_1^2 (1 + 2 \lambda)^2 \Delta_k^2} \times\log T.
    \]
    Then for any $s \geq s_{b, 1}(T)$,
    \[
        b_{k, s, 1} \leq T^{-1},
    \]
    when $T \geq 2$.
\end{lemma}

\begin{proof}
    Similar to Lemma \ref{lem_bounding_b_k_s_1}, by noting 
    \[
        s_{b, 1}(T) \geq \frac{\sigma_k^2}{C_1^2 (1 + 2 \lambda)^2 \Delta_k^2} \times\log (2 T),
    \]
    the Hoeffding inequality gives
    \[
        \begin{aligned}
            b_{k, s, 1} & = {\mathbb{E}}  \big[ \mathbb{I}(Q_{k, s} (\tau_k) > T^{-1} ) \mathbb{I} (A_{k, s}^c)\big] \\
            & \leq \mathbb{P} (A_{k, s}^c) \\
            & = \mathbb{P} \left( |\overline{R}_{k, s} - \mu_k| \geq (1 + 2\lambda) C_1 \Delta_k \right) \overset{\text{by Hoeffding inequality}}{\leq} T^{-1}.
        \end{aligned}
    \]
\end{proof}

\begin{lemma}[Bounding $b_{k, s, 2}$ at \eqref{def_b_k_s_2}]\label{lem_bounding_b_k_s_2}
    There exists $s_{b, 2}(T) \asymp \log T$ such that 
    Take
    \[
        \begin{aligned}
            s_{b, 2}(T) = \Bigg[ \big[ \Lambda_k + \mathfrak{a}_1 + \mathfrak{b}_1 + \Lambda_k \mathfrak{a}_1 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_1}}{2}  + \frac{\sqrt{2 \pi} \Lambda_k \mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right]  \right\}  + 1\right) \\
            & \vee \frac{18 \sigma_{\omega}^2 (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2}  \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T,
        \end{aligned}
    \]
    where 
    \[
        \mathfrak{a}_1 = \frac{4 \sigma_{\omega}^2 \lambda^2}{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2} , \qquad \mathfrak{b}_1 = \frac{2 \sigma_{\omega}^2 \left[ 2 \lambda^2 \sigma_1^2 + (1 + 2 \lambda)^2 C_2^2 \Delta_k^2 \right]}{3 C_2^2 (1 + 2 \lambda)^2 \Delta_k^2}, \qquad \Lambda_k = 8 \sqrt{2} \sigma_k^2.
    \]
    Then for any $s \geq s_{b, 2}(T)$,
    \[
        b_{k, s, 2} \leq T^{-1}
    \]
    when $T \geq 2$.
\end{lemma}

\begin{proof}
    Similar to bounding $a_{k, s, 2}$ in Lemma \ref{lem_bounding_a_k_s_2}, if we take
    \[
        \begin{aligned}
            s_{b, 2}(T) = \Bigg[ \big[ \Lambda_k + \mathfrak{a}_1 + \mathfrak{b}_1 + \Lambda_k \mathfrak{a}_1 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_1}}{2}  + \frac{\sqrt{2 \pi} \Lambda_k \mathfrak{a}_1}{2 \mathfrak{b}_1^2} + \frac{2 \mathfrak{a}_1}{\mathfrak{b}_1} \right]  \right\}  + 1\right) \\
            & \vee \frac{18 \sigma_{\omega}^2 (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2}  \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T,
        \end{aligned}
    \]
    with $\Lambda_k = 8 \sqrt{2} \sigma_k^2$, we have 
    \[
        \begin{aligned}
            b_{k, s, 2} & \leq  {\mathbb{E}} \big[ \mathbb{I}(G_{k, s}^c) \big] \\
            & \leq {\mathbb{E}}\exp \left\{ - \frac{3 C_2^2 (1 + 2\lambda)^2 \Delta_k^2 s }{ 2 \sigma_{\omega}^2  \left[ 2 \lambda^2 \overline{\zeta^{(k)}}  +  2 \lambda^2 \sigma_k^2 + (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \right] }\right\} \\
		& ~~~~~~~~ +  \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_k - 1)^2}\right\} \times \mathbb{I}\left( \mu_k \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_k = \frac{1}{2}\right) \right] + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} \leq T^{-2},
        \end{aligned}
    \]
   
    where $\overline{\zeta^{(k)}} = \frac{1}{s} \sum_{i = 1}^s \zeta^{(k)}_i$ with $\zeta^{(k)}_i$ are iid independent sub-Exponential variables such that $\zeta^{(k)}_i \sim \operatorname{subE}(8 \sqrt{2} \sigma_k^2)$. Then $b_{k, s, 2} \leq T^{-2} \leq T^{-1}$.
\end{proof}

\begin{lemma}[Bounding $b_{k, s, 3}$ at \eqref{def_b_k_s_3}]\label{lem_bounding_b_k_s_3}
    Take
    \[
        \begin{aligned}
            s_{b, 3} (T) = \Bigg[\big[ \Lambda_k + \mathfrak{a}_2 + \mathfrak{b}_2 + \Lambda_k \mathfrak{a}_2 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_2}}{2} + \frac{\sqrt{2 \pi} \Lambda_k\mathfrak{a}_2}{2 \mathfrak{b}_2^2} + \frac{2 \mathfrak{a}_2}{\mathfrak{b}_2} \right]  \right\}  + 1\right) \\
            & \vee \frac{18 \sigma_{\omega}^2 (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2}  \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T,
        \end{aligned}
    \]
    where
    \[
        \mathfrak{a}_2 = \frac{\sigma_{\omega}^2 \lambda^2}{3 (\lambda - 1)^2 C_1^2 \Delta_k^2}, \qquad \mathfrak{b}_2 = \frac{\sigma_{\omega}^2\left[ 2 \lambda^2 \sigma_1^2 + 25 (1 + 2\lambda)^2 C_1^2 \Delta_k^2 \right]}{6 (\lambda - 1)^2 C_1^2 \Delta_k^2}, \qquad \Lambda_k = 8 \sqrt{2} \sigma_k^2.
    \]
    Then for any $s \geq s_{b, 3}(T)$ and $\lambda > 1$,
    \[
        b_{k, s, 3} \leq T^{-1},
    \]
    when $T \geq 2$.
\end{lemma}

\begin{proof}
    The basic idea is the same as bounding $a_{k, s, 3}$. The only difference is we replace $\big[ 1 - Q_{k, s} (\tau_k) \big]$ by $Q_{k, s} (\tau_k)$. Again, we will first bound $\Gamma_k = \overline{Y}_{k, s}  - \overline{R}_{k, s}^*$ on the event $A_{k, s} \, \cap \, G_{k, s}$.
   
   
       
       
       
   
   
       
       
       
   
Exactly the same as before, we let
\[
    \tau_k = \frac{\mu_k + \lambda}{1 + 2 \lambda} + \frac{6 \lambda C_1 \Delta_k}{1 + 2 \lambda}
\]
   
   
   
which satisfies $\tau_k \geq \frac{\mu_k + \lambda}{1 + 2 \lambda} $. To ensure $\tau_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda}$, one just need to take $C_1 = \frac{1}{6 \lambda}$, and then $C_2 = \frac{1}{2} C_1 =  \frac{1}{12 \lambda}$ by \eqref{C_1_2}. Next, we will obtain a bound for $\Gamma_k$ on $A_{k, s} \, \cap \, G_{k, s}$ as
\[
    \begin{aligned}
        \Gamma_k & \geq \tau_k - 2 C_1 \Delta_k - \frac{\mu_k + \lambda}{1 + 2 \lambda} \\
        & = 2 C_1 \Delta_k \frac{\lambda - 1}{2 \lambda + 1} \\
        \alignedoverset{\text{by } \lambda > 1}{>} 0,
    \end{aligned}
\]
and
\[
    \begin{aligned}
        \Gamma_k & \leq \tau_k + 2 C_1 \Delta_k - \frac{\mu_k + \lambda}{1 + 2 \lambda} \\
        & = 2 C_1 \Delta_k \left( 1 + \frac{3 \lambda}{1 + 2 \lambda}\right) \\
        \alignedoverset{\text{by } \frac{3 \lambda}{1 + 2\lambda} \geq \frac{3}{2}}{\leq} 2 C_1 \Delta_k \left( 1 + \frac{3}{2}\right) = 5 C_1 \Delta_k,
    \end{aligned}
\]
i.e., $\Gamma_k \in \left(\frac{2 C_1 (\lambda - 1) \Delta_k}{2 \lambda + 1}, 5 C_1 \Delta_k \right) \subseteq (0, \infty)$. Therefore,
   
\[
    \begin{aligned}
         &  Q_{k,s} (\tau_k) \mathbb{I} (A_{k, s}) \mathbb{I}(G_{k, s}) \\
         & = \mathbb{P} \left( \overline{Y}_{k, s} - \overline{R}_{k, s}^* \geq \Gamma_k \mid \mathcal{H}_{k, s} \right)  \mathbb{I} (A_{k, s} \, \cap \, G_{k, s}) \\
         \alignedoverset{\text{apply steps in  \eqref{pr_G_bound}}}{\leq} T {\mathbb{E}} \Bigg[ \exp \left\{ - \frac{9 \Gamma_k^2 (1 + 2 \lambda)^2 s^2 }{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\zeta^{(k)}} +  2 \lambda^2 \sigma_1^2 + \Gamma_k^2 (1 + 2 \lambda)^2 \right] }\right\}  \mathbb{I} (A_{k, s} \, \cap \, G_{k, s})  \Bigg] \\ & ~~~~~~~~~~~~~~~ + T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_k - 1)^2}\right\} \times \mathbb{I}\left( \mu_k \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_k = \frac{1}{2}\right) \right] \\
          & ~~~~~~~~~~~~~~~ + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} \\
         \alignedoverset{\text{by the bound of } \Gamma_k}{\leq} T {\mathbb{E}}\exp \left\{ - \frac{36 (\lambda - 1)^2 C_1^2 \Delta_k^2 s^2}{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\zeta^{(k)}} +  2 \lambda^2 \sigma_1^2 + 25 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2  \right] }\right\} \\
         & ~~~~~~~~~~~~~~~ + T \left[ \exp \left\{ - \frac{(\lambda^2 + \lambda + 1 / 4)^2 s}{18 \sigma_{\omega}^2 (2 \mu_k - 1)^2}\right\} \times \mathbb{I}\left( \mu_k \neq \frac{1}{2}\right) + 0 \times \mathbb{I}\left( \mu_k = \frac{1}{2}\right) \right] \\
          & ~~~~~~~~~~~~~~~ + \frac{T}{2} \exp \left\{ - \frac{s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}\right\} .
        
        
    \end{aligned}
\]
   
       
       
       
       
       
       
       
       
   
where $\overline{\zeta^{(k)}} = \frac{1}{s} \sum_{i = 1}^s \zeta^{(k)}_i$ with $\zeta^{(k)}_i$ are iid independent sub-Exponential variables such that $\zeta^{(k)}_i \sim \operatorname{subE}(8 \sqrt{2} \sigma_k^2)$.
Take
\[
    s \geq \big[ \Lambda_k + \mathfrak{a}_2 + \mathfrak{b}_2 + \Lambda_k \mathfrak{a}_2 \big] \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_2}}{2} + \frac{\sqrt{2 \pi} \Lambda_k\mathfrak{a}_2}{2 \mathfrak{b}_2^2} + \frac{2 \mathfrak{a}_2}{\mathfrak{b}_2} \right]  \right\}  + 1\right),
\]
by applying Lemma \ref{lem_subE_ineq} again,
\[
     \begin{aligned}
         & T {\mathbb{E}}\exp \left\{ - \frac{36 (\lambda - 1)^2 C_1^2 \Delta_k^2 s^2}{6  \sigma_{\omega}^2 s  \left[ 2 \lambda^2 \overline{\zeta^{(k)}} +  2 \lambda^2 \sigma_1^2 + 25 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2  \right] }\right\} \\
         = & T {\mathbb{E}} \exp \left\{ - \frac{s}{\mathfrak{a}_2  \overline{\zeta^{(k)}} + \mathfrak{b}_2}\right\} \leq \frac{1}{3T}.
     \end{aligned}
\]
Therefore, we have $b_{k, s, 3} \leq T^{-1}$ if we take
\[
    \begin{aligned}
         s \geq s_{b, 3} (T) = \Bigg[\big[ \Lambda_k + \mathfrak{a}_2 + \mathfrak{b}_2 + \Lambda_k \mathfrak{a}_2 \big] & \left(\frac{1}{3}\log^{-1}2 \times \log \left\{ 3 \left[  1 + \frac{\sqrt{\pi \mathfrak{b}_2}}{2} + \frac{\sqrt{2 \pi} \Lambda_k\mathfrak{a}_2}{2 \mathfrak{b}_2^2} + \frac{2 \mathfrak{a}_2}{\mathfrak{b}_2} \right]  \right\}  + 1\right) \\
         & \vee \frac{18 \sigma_{\omega}^2 (2 \mu_k - 1)^2}{(\lambda^2 + \lambda + 1 / 4)^2}  \vee \frac{2 \sigma_{\omega}^2 (1 + 2 \lambda^2)}{(1 + 2 \lambda)^2} \Bigg] \times 3 \log T.
    \end{aligned}
\]
\end{proof}

   
        
        
       
       
       
       
   
   

\section{Technical Lemmas}\label{sec_lem_tech}

\begin{lemma}\label{lemma:gaussian_weight}
	Let $Z$ be a standard Gaussian variable, then the tail probability $\mathbb{P}(Z > x)$ satisfies
	\[
	    \frac{1}{4} \exp (- x^2 ) < \mathbb{P}(Z > x) \leq \frac{1}{2} \exp ( - {x^2} / {2} )
	\]
	for any $x \geq 0$. 
\end{lemma}

\begin{proof}
	Denote $Q(x) := \mathbb{P} (Z > x)$ for $x > 0$ be the tail probability for a standard Gaussian variable $Z$. Let $Z_1$ and $Z_2$ are two independent standard Gaussian random variables, then
	\[
	    \begin{aligned} 
	        \mathbb{P} (Z_1 \leq x, \, Z_2 \leq x) &=\int_{-x}^x \int_{-x}^x \frac{1}{2 \pi} \exp \left[\left(-z_1^2-z_2^2\right) / 2\right] \, \mathrm{d} z_1 \mathrm{d} z_2 \\ 
	        & \leq \int_0^{\sqrt{2} x} \int_0^{2 \pi} \frac{1}{2 \pi} \exp \left(-r^2 / 2\right) r \, \mathrm{d} \theta \, \mathrm{d} r \\ 
	        &=1-\exp (-x^2)
	    \end{aligned}
	\]
	giving $[1-2 Q(x)]^2 \leq 1-\exp (-x^2)$ for $x \geq 0$, or, equivalently, $\exp (-x^2) \leq 4 Q(x)-4 Q^2(x)$. But, since $4 Q^2(x)>0$ for all $x$, we get that
	\[
	    Q(x) > \frac{1}{4} \exp (-x^2)
	\]
    which gives the lower bound in the lemma. For the upper bound, we note that
    \[
        \begin{aligned}
            \exp \left(x^2 / 2\right) Q(x) &=\exp \left(x^2 / 2\right) \int_x^{\infty}(2 \pi)^{-1 / 2} \exp \left(-t^2 / 2\right) \mathrm{d} t \\
            &=\int_x^{\infty}(2 \pi)^{-1 / 2} \exp \left(-\left(t^2-x^2\right) / 2\right) \mathrm{d} t \\
            &<\int_x^{\infty}(2 \pi)^{-1 / 2} \exp \left(-(t-x)^2 / 2\right) \mathrm{d} t =\frac{1}{2},
         \end{aligned}
    \]
    which gives the upper bound.
\end{proof}

\begin{lemma}\label{lem_subGtosubE}
    Suppose $\{X_i\}_{i = 1}^n$ are i.i.d. zero-mean sub-Gaussian with variance proxy $\sigma^2$, and $\overline{X} = \frac{1}{n} \sum_{i = 1}^n X_i$, then
    \[
        {\mathbb{E}} \exp\big( \lambda\overline{X}^2 \big) \leq e^{9 / 8}
       
    \]
    for any $|\lambda| \leq \frac{n}{8 \sigma^2}$.
\end{lemma}

\begin{proof}
    Note that $X - \mu \sim \operatorname{subG} (\sigma^2)$ implies
    \begin{equation}\label{lem_subGtosubE_1}
        \overline{X} - \mu \sim \operatorname{subG} (\sigma^2 / n)
    \end{equation}
    by the fact that
    \[
        \begin{aligned}
        	{\mathbb{E}} \exp \left\{ s (\overline{X} - \mu) \right\} & = \prod_{i = 1}^n \exp \left\{ \frac{s}{n} ({X}_i - \mu) \right\} \\
        	(\text{by } X_i - \mu \text{ is sub-Gaussian}) & \leq \prod_{i = 1}^n \exp \left\{ \frac{s^2}{n^2} \frac{\sigma^2}{2}\right\} = \exp \left\{ \frac{s^2 (\sigma^2 / n)}{2}\right\}.
        \end{aligned}
    \]
    Then by Proposition 4.3 in \citet{zhang2020concentration}, $(\overline{X} - \mu)^2 - n^{-1} \operatorname{var} (X) \sim \operatorname{subE} (8\sqrt{2}\sigma^2 / n, 8\sigma^2 / n)$. Therefore,
    \[
        \begin{aligned}
            {\mathbb{E}} \exp\left\{ \lambda(\overline{X} - \mu)^2 \right\} & = {\mathbb{E}} \exp\left\{ \lambda \big( (\overline{X} - \mu)^2 - n^{-1}\operatorname{var}(X) \big) \right\} {\mathbb{E}} \exp \{ \lambda n^{-1}  \operatorname{var}(X)\} \\
            & \leq \exp \left\{ \frac{(8 \sqrt{2} \sigma^2 / n)^2 \lambda^2}{2}\right\} \cdot \exp \left\{ \lambda n^{-1} \sigma^2 \right\} = \exp \left\{ \frac{64 \lambda^2 \sigma^4}{n^2} + \frac{\lambda \sigma^2}{n}\right\} \leq e^{9 / 8}
        \end{aligned}
    \]
    for any $\lambda \leq \frac{n}{8 \sigma^2}$.
\end{proof}

\begin{lemma}\label{lem_gau_frac}
	Suppose $X$ and $Y$ are Gaussian variables satisfying ${\mathbb{E}} X = 0$ and ${\mathbb{E}} Y > 0$, then
	\[
	    \mathbb{P} \left(\frac{|X|}{|Y|} > c \right) \leq 2 \mathbb{P} (X > c Y) + \mathbb{P} \left( Y < 0 \right)
	\]
	for any $c > 0$.
\end{lemma} 

\begin{proof}

        We have
	\[
	    \begin{aligned}
	    	\mathbb{P} \left(\frac{|X|}{|Y|} > c \right) & = \mathbb{P} \left(|X| > c |Y| \right) \\
	    	& = \mathbb{P} \left( \{ X > c Y \} \, \cap \, \{ X > 0\} \, \cap \, \{ Y > 0 \} \right) + \mathbb{P} \left( \{ - X > c Y \} \, \cap \, \{ X \leq 0\} \, \cap \, \{ Y > 0 \} \right) \\
	    	& ~~~~~~~~~~ + \mathbb{P} \left( \{ X >-  c Y \} \, \cap \, \{ X > 0\} \, \cap \, \{ Y \leq 0 \} \right) + \mathbb{P} \left( \{ - X > - c Y \} \, \cap \, \{ X \leq 0\} \, \cap \, \{ Y \leq 0 \}\right) \\
	    	& = \mathbb{P} \left( \{ X > c Y \} \, \cap \, \{ X > 0\} \, \cap \, \{ Y > 0 \} \right)  + \mathbb{P} \left( \{ - X > c Y \} \, \cap \, \{ - X \geq 0\} \, \cap \, \{ Y > 0 \} \right) + \mathbb{P} (Y \leq 0) \\
	    	\alignedoverset{\text{by using $X$ is symmetric about $0$.}}{\leq} 2 \mathbb{P} (  X > c Y ) + \mathbb{P} (Y \leq 0) 
	    \end{aligned}
	\]
	for any $c > 0$.
\end{proof}

\begin{lemma}[sub-Exponential concentration]\label{lem_subE_ineq}
    Consider zero-mean independent random variables $ X_i \sim \operatorname{subE}(\lambda_i)$, and positive constants $a$ and $b$ satisfy $a \overline{X} + b > 0$ with the mean $\overline{X} := \frac{1}{s} \sum_{i = 1}^s X_i$. Then,
    \[
        {\mathbb{E}} \exp \left\{ - \frac{s}{a \overline{X} + b}\right\} \leq \left[  1 +  \frac{\sqrt{\pi b}}{2}  + \frac{\sqrt{2 \pi} \lambda a}{2 b^2} + \frac{2 a}{b} \right] s \exp \bigg\{ - \frac{s}{\sqrt{2 \lambda a} \vee (b + \lambda a)}\bigg\},
    \]
    for $s \in \mathbb{N}$ such that $2 \lambda a s / b \geq 1$, where $\lambda := \left(\frac{1}{s} \sum_{i=1}^s \lambda_i^2\right)^{1 / 2}$. Specially, we have $\log {\mathbb{E}} \exp \left\{ - \frac{s}{a \overline{X} + b}\right\} \lesssim -s$.
   
   
\end{lemma}

\begin{proof}
   
    Denote $Y := a \overline{X} + b$ as a strictly positive random variable. For any non-negative strictly increasing function $f$, we have
    \[
        {\mathbb{E}} f(Y) = \int_0^{\infty} \mathbb{P} \left( f(Y) > r\right) \, \mathrm{d} r = \int_0^{\infty} \mathbb{P} \left( Y > f^{-1} (r)\right) \, \mathrm{d} r.
    \]
    Then for any fixed $s \in \mathbb{N}$,
    \[
        \begin{aligned}
        	{\mathbb{E}} \exp \left\{ - \frac{s}{a\overline{X} + b}\right\} & = {\mathbb{E}} \exp \left\{ - \frac{s}{Y} \right\} = \int_0^{\infty} \mathbb{P} \left( \exp \left\{ - \frac{s}{Y}\right\} > r \right) \, \mathrm{d} r \\
        	& = \int_0^1 \mathbb{P} \left( Y > \frac{s}{\log r^{-1}} \right) \, \mathrm{d} r \\
        	& = \int_0^1 \mathbb{P} \left( \overline{X} > \frac{1}{a} \left[ \frac{s}{\log r^{-1}} - b \right]\right) \, \mathrm{d} r \\
        	& = \int_0^{\exp \left\{ -\frac{s}{b}\right\}} \mathbb{P} \left( \overline{X} > \underbrace{\frac{1}{a} \left[ \frac{s}{\log r^{-1}} - b \right]}_{\leq 0}\right) \, \mathrm{d} r + \int_{\exp \left\{ -\frac{s}{b}\right\}}^{1} \mathbb{P} \left( \overline{X} > \underbrace{\frac{1}{a} \left[ \frac{s}{\log r^{-1}} - b \right]}_{> 0}\right) \, \mathrm{d} r \\
        	& \leq \exp \left\{ -\frac{s}{b}\right\} \times 1 + \int_{\exp \left\{ -\frac{s}{b}\right\}}^{1} \mathbb{P} \left( \overline{X} > \underbrace{\frac{1}{a} \left[ \frac{s}{\log r^{-1}} - b \right]}_{> 0}\right) \, \mathrm{d} r \\
        	\alignedoverset{\text{by letting $u = \frac{s}{\log r^{-1}}$}}{=} \exp \left\{ -\frac{s}{b}\right\} + \int_b^{\infty} \mathbb{P} \left( \overline{X} > \underbrace{\frac{u - b}{a}}_{>0} \right) \exp\left\{ - \frac{s}{u}\right\} \frac{s}{u^2} \, \mathrm{d} r.
        \end{aligned} 
    \]
    Recall $\overline{X}$ is the average of independent zero-mean sub-exponential variables, denote $\lambda := \left(\frac{1}{s} \sum_{i=1}^s \lambda_i^2\right)^{1 / 2}$. 
    Then, we have 
    \[
        \mathbb{P} \left( \overline{X} > t \right) \leq \exp \left\{ -\frac{1}{2}\left(\frac{s t^2}{{\lambda}^2} \wedge \frac{s t}{\lambda}\right) \right\}
    \]
    for any $t > 0$ by Corollary 4.2.(c) in \citet{zhang2020concentration}. Therefore, we can furthermore bound the expectation as
    \[
        \begin{aligned}
            {\mathbb{E}} \exp \left\{ - \frac{s}{a\overline{X} + b}\right\} \alignedoverset{\text{by sub-Exponential inequality}}{\leq} \exp \left\{ -\frac{s}{b}\right\} +\int_b^{\infty} \frac{s}{u^2} \exp \left\{ - \frac{s}{u}\right\} \exp \left\{- \frac{1}{2} \frac{s (u - b)^2}{\lambda^2 a^2} \wedge \frac{s (u - b)}{\lambda a} \right\} \, \mathrm{d} u \\
        	& \leq \exp \left\{ -\frac{s}{b}\right\} +\frac{s}{b^2} \int_b^{\infty} \exp \left\{- \left[ \frac{s}{u} + \frac{1}{2} \frac{s(u - b)^2}{\lambda^2 a^2} \wedge \frac{s(u - b)}{\lambda a} \right] \right\} \, \mathrm{d} u \\
        	& = \exp \left\{ -\frac{s}{b}\right\} + \left[ \int_b^{b + \lambda  a} + \int_{b + \lambda a }^{\infty} \right] \frac{s}{u^2} \exp \left\{- \left[ \frac{s}{u} + \frac{1}{2} \frac{s(u - b)^2}{\lambda^2 a^2} \wedge \frac{s(u - b)}{\lambda a} \right] \right\} \, \mathrm{d} u \\
        	& =: \exp \left\{ -\frac{s}{b}\right\} + \big[ I + II\big].
        \end{aligned}
    \]
    The next step is bounding both $I$ and $II$. For $I$, we obtain
    \[
        \begin{aligned}
        	I & = \int_b^{b + \lambda  a} \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)^2}{2 \lambda^2 a^2}\right]\right\} \, \mathrm{d} u \\
        	& \leq \int_b^{b + \lambda  a} \frac{s}{b^2} \exp \left\{ - \left[ \frac{s}{b + \lambda  a} + \frac{s(u - b)^2}{2 \lambda^2 a^2}\right]\right\} \, \mathrm{d} u \\
        	& = \frac{s}{b^2} \exp \left\{ - \frac{s}{b + \lambda a}\right\} \int_b^{b + \lambda  a} \exp \left\{ - \frac{ s (u - b)^2}{2 \lambda^2 a^2}\right\} \, \mathrm{d} u \\
        	& \leq \frac{s}{b^2} \exp \left\{ - \frac{s}{b + \lambda a}\right\} \int_b^{\infty} \exp \left\{ - \frac{(u - b)^2}{2 \lambda^2 a^2 / s}\right\} \, \mathrm{d} u = \frac{\sqrt{2 \pi} \lambda a }{2 b^2} \exp \left\{ -\frac{s}{b + \lambda a}\right\}
        \end{aligned}
    \]
    For $II$, we decompose it as
    \[
        \begin{aligned}
            II &= \int_{b + \lambda  a}^{\infty} \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u \\
            & = \left[ \int_{b + \lambda a}^{\frac{2 \lambda a s}{b}} + \int_{\frac{2 \lambda a c}{b}}^{\infty} \right] \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u \\
            & = \int_{b + \lambda a }^{\frac{2 \lambda a s}{b}} \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u + \exp \left\{ -\frac{s}{b}\right\} \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{s}{u^2} \exp \left\{ - \frac{s(u - b) ( u - \frac{2 \lambda a s}{b})}{2 \lambda a u}\right\} \, \mathrm{d} u \\
            & =: II_1 + II_2 .
        \end{aligned}
    \] 
    And for $II_1$, 
    \[
        \begin{aligned}
            II_1 & = \int_{b + \lambda a}^{\frac{2 \lambda a s}{b}} \frac{s}{u^2}  \exp \left\{ - \left[ \frac{s}{u} + \frac{s(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u \\
            & = \int_{b + \lambda a}^{\frac{2 \lambda a s}{b}} \frac{s}{u^2}  \exp \left\{ - s \times \left[ \frac{1}{u} + \frac{(u - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u \\
            \alignedoverset{g(u) = \frac{1}{u} + \frac{(u - b)}{2 \lambda a } \text{ is increasing on } u \geq \sqrt{2 \lambda a}}{\leq} \int_{b + \lambda a}^{\frac{2 \lambda a s}{b}} \frac{s}{u^2}  \exp \left\{ - s \times \left[ \frac{1}{\sqrt{2 \lambda a} \vee (b + \lambda a )} + \frac{(\sqrt{2 \lambda a} \vee (b + \lambda a ) - b)}{2 \lambda a }\right]\right\} \, \mathrm{d} u  \\
            & \leq \frac{s}{(b + \lambda a) } \left[ \frac{2 \lambda a s}{b} - \big( b + \lambda a \big) \right] \exp \left\{ - \frac{s}{\sqrt{2 \lambda a} \vee (b + \lambda a )}\right\}.
        \end{aligned}
    \]
   
   
       
        
        
        
       
   
   
   
       
        
        
        
       
   
   
   
       
        
        
       
   
   
    For $II_2$, if $\frac{2 \lambda a s}{b} \geq 1$, i.e. $\lambda s \geq \frac{b}{2 a}$, we obtain that
    \[
        \begin{aligned}
        	II_2 & = \exp \left\{ -\frac{s}{b}\right\}  \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{s}{u^2} \exp \left\{ - \frac{s(u - b) ( u - \frac{2 \lambda a s}{b})}{2 \lambda a u}\right\} \, \mathrm{d} u \\
        	& \leq \exp \left\{ -\frac{s}{b}\right\}  \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{s}{u^2} \exp \left\{ - \frac{s(u - \frac{2 \lambda a s}{b})^2}{2 \lambda a u}\right\} \, \mathrm{d} u \\
            \alignedoverset{\text{by } \frac{2 \lambda a s}{b} \geq 1}{\leq} s \exp \left\{ -\frac{s}{b}\right\} \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{1}{u^{3 / 2}} \exp \left\{ - \frac{(u - \frac{2 \lambda a s}{b})^2}{2 \lambda a u}\right\} \, \mathrm{d} u \\
            \alignedoverset{\text{by } \frac{2 \lambda a s}{b} \geq 1}{\leq} s \exp \left\{ -\frac{s}{b}\right\} \sqrt{\frac{2 \lambda a s}{b}} \int_{\frac{2 \lambda a s}{b}}^{\infty} \frac{1}{u^{3 / 2}} \exp \left\{ - \frac{(u - \frac{2 \lambda a s}{b})^2}{2 \lambda a u}\right\} \, \mathrm{d} u \\
            & = s \exp \left\{ - \frac{s}{b}\right\}   \sqrt{\frac{2 \lambda a s}{b}} \frac{b}{2s}\sqrt{\frac{\pi}{2 \lambda a}} \left[ 1 - e^{4 s / b} \frac{2}{\sqrt{\pi}} \int_{2 \sqrt{s / b}}^{\infty} e^{-t^2} \,  \mathrm{d} t \right] \\
            \alignedoverset{\text{by Lemma \ref{lem_erfc}}}{\leq} s \exp \left\{ - \frac{s}{b}\right\}   \sqrt{\frac{2 \lambda a s}{b}} \frac{b}{2s}\sqrt{\frac{\pi}{2 \lambda a}} (1 - 0) = \frac{\sqrt{\pi b s}}{2} \exp \left\{ - \frac{s}{b}\right\}.
               
               
        
        
        
           
        \end{aligned}
    \]
    Combining these results, we obtain that
    \[
        \begin{aligned}
        	{\mathbb{E}} \exp \left\{ - \frac{s}{aX + b}\right\} & \leq \exp \left\{ -\frac{s}{b}\right\} + I +  II_1 + II_2 \\
        	& \leq \exp \left\{ -\frac{s}{b}\right\} + \frac{\sqrt{2 \pi} \lambda a s}{2 b^2} \exp \left\{ -\frac{s}{b + \lambda a}\right\} \\
        	& \qquad + \frac{s}{(b + \lambda a) } \left[ \frac{2 \lambda a s}{b} - \big( b + \lambda a \big) \right] \exp \left\{ - \frac{s}{\sqrt{2 \lambda a} \vee (b + \lambda a )}\right\} + \frac{\sqrt{\pi b s}}{2} \exp \left\{ - \frac{s}{b}\right\} \\
                & \leq \left[ 1 +  \frac{\sqrt{\pi b s}}{2}  \right] \exp \left\{ - \frac{s}{b}\right\} + \left[ \frac{\sqrt{2 \pi} \lambda a s}{2 b^2}  + \frac{2 \lambda a s^2}{b(b + \lambda s)}\right] \exp \bigg\{ - \frac{s}{\sqrt{2 \lambda a} \vee (b + \lambda a)}\bigg\} \\
                & \leq \left[ 1 +  \frac{\sqrt{\pi b s}}{2}  \right] \exp \left\{ - \frac{s}{b} \right\} + \left[ \frac{\sqrt{2 \pi} \lambda a}{2 b^2} + \frac{2 a}{b}\right] s \exp \bigg\{ - \frac{s}{\sqrt{2 \lambda a} \vee (b + \lambda a)}\bigg\}  \\
                \alignedoverset{\text{by } b \leq \sqrt{2 \lambda a} \vee (b + \lambda a)}{\leq} \left[  1 +  \frac{\sqrt{\pi b}}{2}  + \frac{\sqrt{2 \pi} \lambda a}{2 b^2} + \frac{2 a}{b} \right] s \exp \bigg\{ - \frac{s}{\sqrt{2 \lambda a} \vee (b + \lambda a)}\bigg\} ,
        \end{aligned}
    \]
    which gives the result we need.
   
\end{proof}

\begin{lemma}\label{lem_erfc}
	For any $x \geq 0$,
	\[
	    \frac{2 e^{x^2}}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} \, \mathrm{d} t \leq 1.
	\]
\end{lemma}

\begin{proof}
	\[
	    \begin{aligned}
	    	\frac{2 e^{x^2}}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} \, \mathrm{d} t & = \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{- (t^2 - x^2)} \, \mathrm{d} t \\
	    	& \leq \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{- (t - x)^2} \, \mathrm{d} t \\
	    	& = \frac{2}{\sqrt{\pi}} \int_0^{\infty} \frac{e^{- t^2}}{2 \times \frac{1}{2}} \, \mathrm{d} t = \frac{2}{\sqrt{\pi}} \frac{\sqrt{2 \pi \times \frac{1}{2}}}{2} = 1.
	    \end{aligned}
	\]
\end{proof}


\section{More Experiment Results and Details}\label{sec:appendix_experiments}
\subsection{Results for linear bandits}\label{sec:results_LB}



\begin{figure*}[!t]
    \includegraphics[width=\linewidth]{Fig/LB_ber.pdf}
   
    \vspace{-.8cm}
    \caption{Performance of \name{MBE} in three linear bandit problems.
}
    \label{fig:LB}
\end{figure*}

   
   
   
   
   

We also consider the linear bandit problem. 
The linear bandit version of \name{MBE} is presented in Appendix \ref{sec:MBTS_LB}.
We experiment with several dimensions $p = 10, 15, 20$. The number of arms is $K = 100$. 
The feature vector $x_k \in \mathbb{R}^p$ of arm $k$ is generated as follows. 
For the last 10 arms, the features are drawn uniformly at random from $(0, 1)$. 
For the first $90$ arms, we consider a practical setup where they are low-rank: we first generate a loading metric $A = (a_{ij}) \in \mathbb{R}^{p \times 5}$ from $\text{Uniform} (0, 1)$, then  sample $b  \in \mathbb{R}^{5}$ from $\text{Uniform} (0, 1)$, and finally constructs  
$x_{k} = A b$. 
The parameter vector $\theta \in \mathbb{R}^p$ is uniformly sampled from $[0, 1]^p$. 
We normalize the feature vectors such that the mean reward $\mu_k = x_k^{\top} \theta $ falls within the interval $[0, 1]$. 
The rewards of arm $k$ are drawn i.i.d. from $\text{Bernoulli}(x_k^{\top} \theta)$. 



   
   
   
   

We still compare \name{MBE} with the method for linear bandit version of  \name{GIRO}, \name{PHE}, and \name{ReBoot} with tuning to their best performance over the hyper-parameter set $\{2^{k-4}\}_{k=0}^6$ and report the best performance of each method. 
For \name{TS}, we use Gaussian for both its reward and prior distribution, and calibrate their parameters using the true model. 
The total rounds are $T = 20000$ and our results are averaged over $50$ randomly chosen problems. 
Most other details are similar to our MAB experiments. 


We present the results in Figure \ref{fig:LB}, where we vary either $\sigma_{\omega}$ or $\lambda$ in the two subplots. 
We can see that \name{naive MB} leads to a linear regret. 
Hence, the pseudo-reward also matters in this problem. 
\name{MBE} achieves comparable performance with strong baselines such as \name{TS}. 
Another finding  is that \name{MBE} is robust to its tuning parameters. 
Finally, \name{Reboot} needs to pull $K$ times to initialize (the linear regret part in the first $K$ rounds) due to the nature of its design. 
In contrast, most other linear bandit algorithms typically only need $p$ rounds of forced exploration. 
This shows the limitation of the \name{ReBoot} framework (See Appendix \ref{sec:ReBoot}). 







\subsection{Additional results}\label{sec:results_robust_lam}

In this section, we study the performance of \name{MBE} with respect to a few other hyper-parameters. 


We first study the robustness to another tuning parameter $\lambda$. 
Recall that $\lambda$ controls the amount of external perturbation. 
Specifically, we repeat the experiment in \ref{sec:exp_MAB} with $\sigma_\omega^2$ fixed as 0.5 and with different values of $\lambda$ $(0.25, 0.5, 0.75)$.  
From Figure \ref{fig:MAB_lam}, it can be seen that a small amount of pseudo-rewards ($\lambda=0.25$) seems sufficient in these settings, and the results are fairly stable. 
We believe this is because the exploration of \name{MBE} is main driven by the internal randomness in the data. 



\begin{figure*}[!t]
    \includegraphics[width=\linewidth]{Fig/MAB_lam.pdf}
    \vspace{-.5cm}
    \caption{Performance of \name{MBE} with different values of $\lambda$ in MAB. }
   
    \label{fig:MAB_lam}
\end{figure*}

In Figure \ref{fig:MAB_K}, we repeat our main experiments with $K$ changed to $25$. 
We can see that our main conclusions still hold. 



\begin{figure*}[!t]
    \includegraphics[width=\linewidth]{Fig/MAB_K.pdf}
    \vspace{-.5cm}
     \caption{Performance of \name{MBE} with $K = 25$. }
   
    \label{fig:MAB_K}
\end{figure*}


Finally, in Figure \ref{fig:MAB_B}, we implement \name{MBE} with different number of replicates $B$. 
As expected, more replicates does help exploration due to a better approximation to the whole bootstrapping distribution. 
Yet, we find that $B=50$ suffices to generate comparable performance with \name{TS} and the performance of \name{MBE} becomes relatively stable for larger values of $B$. 


\begin{figure*}[!t]
    \includegraphics[width=0.9\linewidth]{Fig/MAB_B.pdf}
    \vspace{-.5cm}
     \caption{Performance of \name{MBE} with different number of replicates $B$. }
   
    \label{fig:MAB_B}
\end{figure*}


\subsection{Details of the real data experiments}\label{sec:exp_details}

Our real data experiments closely follow \citet{wan2022towards}. 
For completeness, we provide information of the three problems here, and refer interested readers to \citet{wan2022towards} and references therein. 


In an online learning to rank problem, we aim to select and rank $K$ items from a pool of $L$ ones. 
We itreatively interacts with users to learn about their preferences. 
The cascading model is popular in learning to rank \citep{kveton2015cascading}, which models the user behaviour as glancing from top to bottom (like a cascade) and choose to click an item following a Bernoulli distribution when she looks at that item. 
Therefore, we will binary outcomes for all items that the user has examined, and there are complex dependency between them. 

In a slate optimization (or called assortment optimization) problem, we aim to offer $K$ items from a pool of $L$ ones, especially when there exist substitution effects. 
The Multinomial Logit (MNL) model characterizes the choice behaviour as a multinomial distribution based on the attractiveness of each item. 
Since the offer subset changes over rounds, the joint likelihood is actually complex. 
To get the posterior or confidence bounds, one has to resort to an epoch-type offering schedule \citep{agrawal2017thompson}. 

Online combinatorial optimization also has numerous  applications \citep{wen2015efficient}, including maximum weighted matching, ads allocation, webpage optimization, etc. 
It is common that every chosen item will generate a separate observation, known as the semi-bandit problem. 
We consider a special problem in our experiments, where we need to choose $K$ persons from a pool under constraints. 




The three datasets we used (and related problem setups) are studied in corresponding TS papers in the literature. 
To general random rewards, we need to either generate from a real data-calibrated model or  by directly sampling from the dataset. 
We follow  \citet{wan2022towards} and references therein. 
For cascading or MNL bandits, we split the dataset into a training and a testing set, use the training to estimate the reward model, and compare on the testing set. 
For semi-bandits, we sample rewards from the dataset. 





\section{Bandit Problem and Thompson Sampling}

We consider a general (context-free) form, where at each round $t$, the agent needs to choose an action $A_t \in \mathcal{A}_t$ and then receive the corresponding reward $R_t = f(A_t;\theta) + \epsilon_t$, where $\epsilon_t$ is a mean-zero noise term. 
The goal is to minimize the cumulative regret $\sum_t {\mathbb{E}} \big[ max_{a \in \mathcal{A}_t} f(a;\theta)  - f(A_t;\theta) \big]$. 
When $\mathcal{A}_t$ is a fixed finite set and $f(A_t;\theta)$ equals to some $r_{A_t}$, we are considering MAB. 
When $A_t$ is a vector and $f(A_t;\theta) = A_t^T \theta$, we are considering linear bandits. 
We can use these two as our main examples, with more complicated ones in mind. 


A greedy approach is to train a supervised learning model $f(;\hat{\theta}_t)$ with all available data and then choose $A_t = \argmax_{a \in \mathcal{A}_t} f(a;\hat{\theta}_t)$. 
This is a common practice in the company and upon which we would like to add exploration via quantifying uncertainty so that one can escape from the local optimum. 

There are two popular algorithmic frameworks, Upper Confidence Bound (UCB) and Thompson Sampling (TS). 
UCB adopts a frequentist viewpoint and construct an upper confidence bound for $f(a;\hat{\theta}$ as $\hat{U}_t(a)$, for every $a$. 
UCB then chooses $A_t = \argmax_a \hat{U}_t(a)$. 
 
TS adopts a Bayesian viewpoint, maintain a posterior over the model, and sample action from it. 
Specifically, we can maintain a posterior over $\theta$ as $Q_t(\theta)$, sample $\Tilde{\theta}_t \sim Q_t(\theta)$, and then 
choose $A_t = \argmax_a f(a; \Tilde{\theta}_t)$. 
This is equivalent to directly maintaining a posterior over $A^*_t = \argmax_{a \in \mathcal{A}_t} f(a;\theta)$ and then sample $A_t$. 

However, UCB and TS are typically only analytically/computationally feasible when $f$ is simple. 
For a complex or even black-box (pipeline-like) $f$, some perturbation/bootstrapping-based UCB/TS-type exploration would be a natural option. 
Besides, such an approach is more data-driven (as Bootstrapping directly utilizes the uncertainty in data) and requires less assumptions on the DGP. 

However, the theoretical understanding is still largely missing. 
Moreover, a naive adaption of Bootstrapping is probably not sufficient, as the type of guarantees we have is typically asymptotic or non-asymptotic but for i.i.d. data, while bandits typically require non-asymptotic results for dependent data. 
Certain additional perturbation or variance inflation might be needed to ensure enough exploration. 



\section{Multiplier Bootstrap-based Thompson Sampling}

There are many types of bootstrapping. 
Non-parametric bootstrapping is resampling of training examples with replacement. 
Yet in bandits, the data size grows and it could be computationally challenging to implement in an online fashion. 
This can be approximated computationally efficiently by i.i.d. Poisson weights. 
We outline our mental algorithm first. 
Modifications (such as appropriate variance inflation) might be needed to yield desired regret. 

\textbf{Mental Algorithm.}
We assume the existence of an efficient cost-sensitive least-squares oracle $\mathcal{O}_f$, i.e., give a parameterized model $f(;\theta)$ and the history $\{(R_t, A_t, W_t)\}$ where $W_t$ is some weight associated with the $t$-th data point, 
we can obtain the weighted least square solution for $\theta$ as $\mathcal{O}_f \big( \{(R_t, A_t, W_t)\} \big)$. 
We maintain $M$ models. 
In round $t$, one model is chosen uniformly at random and the best action under that model is taken. 
After we get the observation $(R_t, A_t)$, 
for each model $m$, 
we sample a weight $W_t$ from certain distribution with mean $1$ (e.g., $Poisson(1)$), and then update the model with the weighted least square. 
We note that maintaining $M$ models with fixed multiplier weights is only a computationally tractable way to approximate the full version, which re-assigns the weights for all data points every time. 
In our theory, we can focus on the latter one. 

\section{Method}
Our MAB with mean reward between $[0,1]$, based on my (toy) simulation study, 
I suspect that two pseudo rewards (perturbed also necessary) might be enough? 
Guassian multiplier seems also enough and much faster (due to the ease of sampling). 

{\color{blue}{
Q: would be symmetric be a concern?
}}

\textbf{DGP:}
\begin{itemize}
    \item MAB: Gaussian, Bernoulli, Beta
    \item Contextual: linear, logisitc
    \item Structured Bandits
\end{itemize}

\textbf{Baseline:}
\begin{itemize}
\item Brano: GIRO (computationally slow; non-parametric + pseudo reward), PHE-MAB, PHE-linear. 
\item Guang Cheng: residual Bootstrap-based UCB for MAB (-> linear bandits), residual Bootstrap-based TS with pseudo rewards (only MAB)
\item TS, UCB
\end{itemize}



Note that MAB can be regarded as a special case of linear bandits where we have a fixed set of $K$ arms, each one is parameterized as an indicator vector. 
We conjecture that, 

become perturbed again? what is the main difference?

\input{main_alg_MAB}



Intuitively, the above two procedures add some randomness to each dimension of $\boldsymbol{\theta}$, which works similarly with the prior. 
For more complicated models with a potentially infinite dimension, 
we can consider sample some features and attach some random rewards to boost our exploration.

\clearpage
\newpage

\section{Theory}

\subsection{Proof Scheme for MAB}

Several things we should note during our derivations, which possibly will affect our algorithm design as well: 
\begin{itemize}
    
    {\color{blue}{
    \item Maintaining $B$ models is just a way to approximate the true bootstrapping sample. In analysis, we may want to analyze the full version (i.e., re-sample a totally new sample at every time point)
    }}
    \item It may/should not be a good idea to add a fixed amount of additional noise at the beginning. All existing literature adds a noise with variable $ta$, where $t$ is the current time point. Though one conjecture is that $ta$ might be too much (not adaptive to $t$ at least), a fixed amount of randomness may not be sufficient (though enough in  small-scale experiments)
\end{itemize}

We can see that Algorithm 1 and 2 use different injecting artificial randomness to deal with MAB, Algorithm 3 deals with linear bandits, and Algorithm 4 tackles kernel bandits. The three types algorithms require different proof techniques. We can start with MAB where can use the similar techniques in \cite{wang2020residual}.

\subsection{Theory for Algorithm 1 (details will add later)}

\subsubsection{Schedule of Proof}

The aim is to show the cumulative regret $\operatorname{Reg}_T$ is no more than the order $\log T$. So our main task is to show that $\operatorname{Reg}_T \lesssim \log T$.

\noindent The proof of $\operatorname{Reg}_T \lesssim \log T$ can be divided into three parts:

\begin{itemize}
    \item[1.] The first step is to conclude (or state) the result in \cite{kveton2019garbage}, that is
    \begin{equation*}
        \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
    \end{equation*}
    with
    \begin{equation*}
        a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] = \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1
    \end{equation*} 
    where $Q_{k, s} (\tau) = \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_s  \big)$ and $N_{k, s} (\tau) = 1 / Q_{k, s} (\tau) - 1$ for some $\tau > 0$. So our target switch from $\operatorname{Reg}_T$ to $a_k$ and $b_k$. And the next two main steps will be bounding $a_k$ and $b_k$ separately.
    
    \item[2.] The second step is bounding $a_k = \sum_{s = 0}^{T - 1} a_{k, s}$. Note that
    \begin{equation*}
        a_{k, s} \leq {\mathbb{E}} Q_{1, s}(\tau_k)^{-1} \leq {\mathbb{E}} \big[ \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s})^{-1} ) \big],
    \end{equation*}
    we need to first find the lower bound of the tail probability $\mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s})$ and then find the upper bound of the expectation.
    \begin{itemize}
        \item[2.1.] The lower bound of tail probability $\mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s})$.
        Note that 
        \begin{equation*}
            \overline{Y}_{1, s} = \left(0 + {\color{red}{\omega_0 \cdot \lambda_1}} + \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \omega_{\ell} R_{1, \ell} \right) / \left( {\color{red}{\lambda_2}} + \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \omega_{\ell} \right)
        \end{equation*}
    \end{itemize}
    will be only random on the bootstrap weights $\omega_{\ell}$ when conditions on $\mathcal{H}_{1, s}$ and the lower bound of the tail probability must require the exact distribution of $\omega_{\ell}$ or the harsh restriction for $\omega_{\ell}$ will be involved (see the discussion of \cite{zhang2020non} for details), where $\lambda_1$ and $\lambda_2$ is the tuning parameter.
    
    For simplicity, we assume $\omega_{\ell} \sim \operatorname{Exp}(1)$ and define $R_{1, 0} := \lambda_1 + \mu_1$ then
    \begin{equation*}
    \begin{aligned}
        \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s}) & = \mathbb{P}_{\omega} \left(  \sum_{\ell = 0}^{|\mathcal{H}_{1, s}|} (R_{1, \ell} - \mu_1) \cdot \omega_{\ell} \geq \lambda_2 \cdot \mu_1 \right) \\
        & \geq \mathbb{P}_{\omega} \left( \bigcap_{\ell = 0}^{|\mathcal{H}_{1, s}|} \big\{ (R_{1, \ell} - \mu_1) \omega_{\ell} \geq |\mathcal{H}_{1, s}|^{-1} \lambda_2 \mu_1 \big\} \right) \\
        {\color{red}{& = 2 \prod_{\ell = 0}^{|\mathcal{H}_{1, s}|} \mathbb{P}_{\omega} \left( (R_{1, \ell} - \mu_1) \omega_{\ell} \geq |\mathcal{H}_{1, s}|^{-1} \lambda_2 \mu_1 \right) \\
        & = \prod_{\ell = 0}^{|\mathcal{H}_{1, s}|} \exp \left\{ - \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| (R_{1, \ell} - \mu_1)} \right\}}} \\
        & = \exp \left\{ - \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| \lambda_1}\right\} \prod_{\ell = 1}^{|\mathcal{H}_{1, s}|} \exp \left\{ - \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| (R_{1, \ell} - \mu_1)} \right\}.
    \end{aligned}
\end{equation*}

    {\color{red}{
        For simplify this question, we first assume
        some specific distributions. 
        
        \begin{itemize}
            \item the Gaussian distribution $R_{1, \ell } \sim N(\mu_1, \sigma^2)$ (used in \cite{wang2020residual}) requires $R_{1, \ell } - \mu_1 > 0$ guaranteeing an upper bound $\lesssim \frac{1}{\sigma \sqrt{\pi}} G^{3, 0}_{0, 0}\left( \left. - \frac{1}{2} \ 0 \ 0\right| \frac{1}{8 \sigma^2}\right)$ with Meijer G-function. However, if we do not require $R_{1, \ell } - \mu_1  > 0$, then ${\mathbb{E}} \exp \left\{ 1 / (R_{1, \ell } - \mu_1) \right\} = \infty$;
            
            \item  another common assumption is $R_{1, \ell} \sim \operatorname{ber} (\mu_1, 1)$. In this case,
            \begin{equation*}
                {\mathbb{E}} \exp \left\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| (R_{1, \ell} - \mu_1)} \right\} = \exp \left\{ - \frac{\lambda_2}{|\mathcal{H}_{1, s}|}\right\} (1 - \mu_1) + \exp \left\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}|(1 - \mu_1)}\right\} \mu_1.
            \end{equation*}
            Note that For any positive number $\mu, \lambda > 0$ and $n \in \mathbb{N}$, we have
                \begin{equation*}
                    \begin{aligned}
                        & \Bigg[ \exp \bigg\{ - \frac{\lambda}{n}\bigg\} (1 - \mu) + \exp \bigg\{ \frac{\lambda \mu}{n (1 - \mu)}\bigg\} \mu \Bigg]^n \\
                        = & e^{-\lambda} (1 - \mu)^n \Bigg[ 1 + \frac{1}{n} \frac{n \mu}{1 - \mu} \exp \bigg\{ \frac{n}{\lambda} + \frac{\lambda \mu}{n(1 - \mu)}\bigg\} \Bigg]^n \\
                        \leq & e^{- \lambda} (1 - \mu)^n \exp \Bigg\{ \frac{n \mu}{1 - \mu} \exp \bigg\{ \frac{n}{\lambda} + \frac{\lambda \mu}{n(1 - \mu)}\bigg\}\Bigg\} \\
                        = & \exp \Bigg\{ n 
                         \log (1 - \mu) + \frac{n \mu}{1 - \mu} \exp \bigg\{ \frac{n}{\lambda} + \frac{\lambda \mu}{n(1 - \mu)}\bigg\} - \lambda \Bigg\}
                    \end{aligned}
                \end{equation*}
                by using the fact $(1 + x / n)^n \leq e^x$. Let the above inequality less than $1$, it sufficiently has $\mu > 1 / 2$, i.e. the optimal arm's mean reward large than $1/2$, this requirement also can be seen in \cite{wang2020residual}.
        \end{itemize}
        
        Up to now, we can conclude that for any $s \in [T]$, the $a_{k, s}$ has the upper bound
        \begin{equation*}
            \begin{aligned}
                a_{k, s} & \leq \exp \left\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| \lambda_1}\right\} \prod_{\ell = 1}^{|\mathcal{H}_{1, s}|} {\mathbb{E}} \exp \left\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| (R_{1, \ell} - \mu_1)} \right\} \\
                & = \exp \Bigg\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| \lambda_1} + |\mathcal{H}_{1, s}| \log (1 - \mu_1) + \frac{|\mathcal{H}_{1, s}| \mu_1}{1 - \mu_1} \exp \bigg\{ \frac{|\mathcal{H}_{1, s}|}{\lambda_1} + \frac{\lambda_1 \mu_1}{|\mathcal{H}_{1, s}|(1 - \mu_1)}\bigg\} - \lambda_1 \Bigg\}.
            \end{aligned}
        \end{equation*}
        with $\lambda_1$ and $\lambda_2$ still need to verify later. So we have partly finished section 2.1.
        
        \begin{itemize}
            \item[2.2] We denote $\overline{Y}_{1, s}^* := \frac{1}{|\mathcal{H}_{1, s}|} \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \omega_{\ell} R_{1, \ell}$ is the true weighted target function, and then
            \begin{equation*}
                \overline{Y}_{1, s} = \frac{\omega_0 \cdot \lambda_1 + |\mathcal{H}_{1, s}| \overline{Y}_{1, s}^*}{\lambda_2 + \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \omega_{\ell}}.
            \end{equation*}
            
            Specify some good events such that $A_{k, s} := \{ \overline{Y}_{k, s} - \mu_k > C_1 \Delta_k\}$ and $A_{k, s}^* := \{ \overline{Y}_{k, s}^* - \mu_k > C_1 \Delta_k\}$ and define
            \begin{equation*}
                a_{k, s, 1} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) \Big], \qquad a_{k, s, 2} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}^c) \Big],
            \end{equation*}
            with $s_{a, j} (T) := \min \{ s : a_{k, s, j} \leq T^{-1}\}$ (it is well-defined by the fact $\mu_1 > 0.5$). In this step, we will show that $\max_{j = 1, 2} s_{a, j} (T) \lesssim \log T$.
        \end{itemize}
        
        \begin{itemize}
            \item[] \begin{itemize}
                \item[2.2.0] On the event $A_{1, s}^*$ and  $A_{1, s}$ 
            \end{itemize}
        \end{itemize}
        \begin{equation*}
            \begin{aligned}
                \mathbb{P} \big(A_{1, s}^* \, | \, \mathcal{H}_{1, s} \big) & = \mathbb{P} \bigg\{ \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \frac{R_{1, \ell}}{|\mathcal{H}_{1, s}|} (\omega_{\ell} - 1) \geq C_1 \Delta_k + \mu_1 - \overline{R}_{1, s}\bigg\} \\
                & \leq \exp \bigg\{ - \frac{1}{2} \frac{|\mathcal{H}_{1, s}|(C_1 \Delta_k + \mu_1 - \overline{R}_{1, s})^2}{\sqrt{\sum_{\ell = 1}^{|\mathcal{H}_{1,s}|} R_{1, \ell}^2}} \wedge \frac{|\mathcal{H}_{1, s} | (C_1 \Delta_k + \mu_1 - \overline{R}_{1, s})}{\max_{\ell\in [|\mathcal{H}_{1, s}|]} R_{1, \ell}}\bigg\} \\
                & \lesssim \exp \Big\{  - \frac{1}{2}(C_1 \Delta_k + \mu_1 - \overline{R}_{1, s})^2 \Big\},
            \end{aligned}
        \end{equation*}
        and
        \begin{equation*}
            \begin{aligned}
                & \mathbb{P} \big(A_{1, s} \, | \, \mathcal{H}_{1, s} \big) \\
                = & \, \mathbb{P} \bigg\{ \lambda_1 \cdot (\omega_0 - 1) + \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} (R_{1, \ell} - \mu_1 - C_1 \Delta_k) \cdot (\omega_{\ell} - 1) \\
                & \qquad \qquad \qquad \qquad \qquad \qquad \geq \lambda_2 \cdot (\mu_1 + C_1 \Delta_k) - \lambda_1 + |\mathcal{H}_{1, s}| (\mu_1 + C_1 \Delta_k - \overline{R}_{1, s})\bigg\} \\
                \leq & \exp \Bigg\{ - \frac{1}{2} \frac{\big[ (\lambda_2 + |\mathcal{H}_{1, s}|) \mu_1 + C_1 (\lambda_2 + |\mathcal{H}_{1, s}|) \Delta_k - \lambda_1 - |\mathcal{H}_{1, s}| \overline{R}_{1, s}\big]^2}{\lambda_1^2 + \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} R_{1, \ell}^2} \\
                & \qquad \qquad \qquad \qquad \qquad \qquad \wedge \frac{(\lambda_2 + |\mathcal{H}_{1, s}|) \mu_1 + C_1 (\lambda_2 + |\mathcal{H}_{1, s}|) \Delta_k - \lambda_1 - |\mathcal{H}_{1, s}| \overline{R}_{1, s}}{\lambda_1 \,  \vee \, 1} \Bigg\} \\
                \overset{\lambda_1 = O(|\mathcal{H}_{1, s}|)}{\leq} &  \exp \Bigg\{ - \frac{1}{2}\frac{\big[ (\lambda_2 + |\mathcal{H}_{1, s}|) \mu_1 + C_1 (\lambda_2 + |\mathcal{H}_{1, s}|) \Delta_k - \lambda_1 - |\mathcal{H}_{1, s}| \overline{R}_{1, s}\big]^2}{\lambda_1^2 + s} \\
                &  \qquad \qquad \qquad \qquad \qquad \qquad \wedge \frac{(\lambda_2 + |\mathcal{H}_{1, s}|) \mu_1 + C_1 (\lambda_2 + |\mathcal{H}_{1, s}|) \Delta_k - \lambda_1 - |\mathcal{H}_{1, s}| \overline{R}_{1, s}}{\lambda_1}\Bigg\}
            \end{aligned}
        \end{equation*}
        by concentration inequality and the fact that $\lambda_1 \geq 1$ (or it will be meaningless). From this, a possible choose is {\color{blue}{$\lambda_1 = C_1 |\mathcal{H}_{1, s}|\Delta_k$ (?????)}} and $\lambda_2 = 0$, then
        \begin{equation*}
            \begin{aligned}
                \mathbb{P} (A_{1, s}) & = {\mathbb{E}} \big[ \mathbb{P} \big(A_{1, s} \, | \, \mathcal{H}_{1, s} \big) \big] \\
                & \leq {\mathbb{E}} \exp \left\{ - \frac{(\overline{R}_{1, s} - \mu_1)^2}{C_1 \Delta_k^2 + s}\right\} = 
            \end{aligned}
        \end{equation*}
        
        \begin{itemize}
            \item[] \begin{itemize}
                \item[2.2.1] Bound $s_{a, 1}(T)$: only need to note that
            \end{itemize}
        \end{itemize}
        
        \begin{equation*}
            \{ N_{1, s} (\tau_k) < T^{-1} \} \supseteq \left\{\left[1-Q_{1, s}(\tau_{k})\right]<T^{-2}\right\}
        \end{equation*}
        with
        \begin{equation*}
            \begin{aligned}
                \big[ 1-Q_{1, s}(\tau_{k}) \big] 1(G_{1, s}) & = \mathbb{P}_{\omega} \left(  \left\{ \sum_{\ell = 0}^{|\mathcal{H}_{1, s}|} (R_{1, \ell} - \mu_1) \cdot \omega_{\ell} \geq \lambda_2 \cdot \mu_1 \right\} \cap G_{1, s} \right) \\
                & = 
            \end{aligned}
        \end{equation*}
        the construction of $G_{1, s}$ is intuitive from the above event.
    }}
\end{itemize}
~~~~\\


\iffalse

{\color{blue}{
{\color{red}{First, we assume $R_{k, \ell}$ is bernoulli such that $R_{k, \ell} \, \overset{\text{i.i.d.}}{\sim} \, \operatorname{ber}(\mu_k, 1)$.}} in Algorithm 1, we have $\overline{Y}_{k, s} =  \sum_{\ell = 1}^{|\mathcal{H}_s|} w_{\ell} \cdot R_{k, \ell}$ for $k \in [K]$ and $s \in [T]$ with $\omega_{\ell} \overset{\text{i.i.d}}{\sim } \rho(\omega)$, it is enough to deal with the algorithm 1 with $\overline{Y}_{k}$ replaced by $\overline{Y}_{k, s} $. 

Denote $Q_{k, s} (\tau) = \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_s  \big)$ and $N_{k, s} (\tau) = 1 / Q_{k, s} (\tau) - 1$. Then from Theorem 1 in \cite{kveton2019garbage}, we have 
\begin{equation*}
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
\end{equation*}
with
\begin{equation*}
    a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] = \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1
\end{equation*} 
for any $\tau_k \in \mathbb{R}$. So it is enough to bound $a_k$ and $b_k$.

\noindent $\circ$ \textbf{Bound $a_k$:}

\noindent Define the good events:
\begin{equation*}
    A_{1, s} := \big\{ \overline{Y}_{1, s} - \mu_1 > - \Delta_k / 4 \big\}, \qquad G_{1, s} := \{ \}.
\end{equation*}
For $\tau_k \leq \mu_1$, we have $a_{k, s} \leq {\mathbb{E}} Q_{1, s}(\tau_k)^{-1} \leq {\mathbb{E}} \big[ \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s})^{-1} \big]$. We need to deal with tail probability of $\rho(\omega)$, which first be assumed as $\operatorname{Exp}(1)$,
\begin{equation*}
    \begin{aligned}
        \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s}) & = \mathbb{P}_{\omega} \left( |\mathcal{H}_s|^{-1} \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \omega_{\ell} R_{1, \ell} > \mu_1 \right) \\
        & = \mathbb{P}_{\omega} \left( \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|}  R_{1, \ell} \cdot ( \omega_{\ell} - 1) > - \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|}  (R_{1, \ell} - \mu_1)\right) \\
        & \geq \mathbb{P}_{\omega} \left( \bigcap_{\ell = 1}^{|\mathcal{H}_{1, s}|} \Big\{ R_{1, \ell} \cdot ( \omega_{\ell} - 1) > -   (R_{1, \ell} - \mu_1) \Big\} \right)\\
        & = \prod_{\ell = 1}^{|\mathcal{H}_{1, s}|} \exp \left\{ - \left( \frac{\mu_1 - R_{1, \ell}}{R_{1, \ell}} + 1\right) \right\} = \exp \left\{ - \mu_1 \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \frac{1}{R_{1, \ell}}\right\},
    \end{aligned}
\end{equation*}

\textbf{(first method:)}

which implies
\begin{equation*}
    \begin{aligned}
        a_{k, s} & \leq {\mathbb{E}} \exp \left\{  \mu_1 \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \frac{1}{R_{1, \ell}}\right\} = \left( {\mathbb{E}} \exp \{ \mu_1 R_{1, 1}^{-1} \}\right)^{|\mathcal{H}_{1, s}|}  \\
        (R_{1, 1} \neq 0) & \leq \Big[ e \mathbb{P} (R_{1, 1} > \mu_1) + ( 1 - \epsilon \mu_1 )^{|\mathcal{H}_{1, s}| + 1 / 2} \Big] ^{|\mathcal{H}_{1, s}|}
    \end{aligned}
\end{equation*}
where $\epsilon$ is some lower bound for the reward of the optimal arm by the fact that for there exists some $\epsilon > 0$,
\begin{equation*}
    \begin{aligned}
        {\mathbb{E}} \exp \{ \mu_1 R_{1, 1}^{-1} \} & \leq e \mathbb{P} (R_{1, 1} > \mu_1 ) + \int_{x \leq \mu_1} \exp (x^{-1} \mu_1) \, f_1(x) \, d x \\
        & \leq  e \mathbb{P} (R_{1, 1} > \mu_1 ) + \int_{\frac{1}{\epsilon |\mathcal{H}_{1, s}|} < x \leq \mu_1} \left( 1 - \frac{\mu_1}{|\mathcal{H}_{1, s}| x}\right)^{|\mathcal{H}_{1, s}| + \frac{x}{2 \mu_1}} f_1(x) \, d x \\
        & \leq e \mathbb{P} (R_{1, 1} > \mu_1) + \int_{\frac{1}{\epsilon |\mathcal{H}_{1, s}|} < x \leq \mu_1} \left( 1 - \epsilon \mu_1 \right)^{|\mathcal{H}_{1, s}| + 1 / 2} \, f_1(x) \, dx \\
        & \leq e \mathbb{P} (R_{1, 1} > \mu_1) + ( 1 - \epsilon \mu_1 )^{|\mathcal{H}_{1, s}| + 1 / 2}.
    \end{aligned}
\end{equation*}
This implies we have bounded $a_{k, s}$ given $\mathbb{P} \big(R_{1, 1} - {\mathbb{E}} R_{1, 1} > 0\big) < 1 / e$. 

\textbf{(second method:)} 

This first method requires $\mathbb{P} \big(R_{1, 1} - {\mathbb{E}} R_{1, 1} > 0\big) < 1 / e$, which is anti-intuitive for many symmetric distributions such that $\mathbb{P} \big(R_{1, 1} - {\mathbb{E}} R_{1, 1} > 0\big) =  \mathbb{P} \big(R_{1, 1} - {\mathbb{E}} R_{1, 1} \leq 0\big) = 1 / 2$. Here, we can give another method to bound $a_{k, s}$ by using the Theorem in \cite{mond2000inequalities}, which gives
\begin{equation*}
    1 \leq \frac{e^{-x} - x e^{-1 / x}}{1 - x} \leq \frac{3}{e}, \qquad x \in (0, 1),
\end{equation*}
By applying this inequality, 
\begin{equation*}
    \begin{aligned}
        & {\mathbb{E}} \exp \{ \mu^{-1} R_{1, 1}\} \\
        \leq & {\mathbb{E}} \frac{\mu_1^{-1}R_{1, 1}}{e^{- \mu_1^{-1} R_{1, 1}} - \frac{3}{e} (1 - \mu_1^{-1} R_{1, 1})} \mathbb{I} (R_{1, 1} \leq \mu_1) + {\mathbb{E}} \frac{\mu_1^{-1}R_{1, 1}}{(\mu_1^{-1}R_{1, 1} - 1) + e^{-\mu_1^{-1}R_{1, 1}}} \mathbb{I} (R_{1, 1} > \mu_1) \\
        =:& I + II
    \end{aligned}
\end{equation*}
with
\begin{equation*}
    \begin{aligned}
        I & \leq {\mathbb{E}} \frac{\mu_1^{-1}R_{1, 1}}{\frac{3}{e} (1 - \mu_1^{-1} R_{1, 1})} \mathbb{I} (R_{1, 1} \leq \mu_1)  - {\mathbb{E}} \frac{\mu_1^{-1}R_{1, 1}}{e^{- \mu_1^{-1} R_{1, 1}}} \mathbb{I} (R_{1, 1} \leq \mu_1)  \\
        & \leq \frac{e}{3} {\mathbb{E}} \frac{R_{1, 1} \mathbb{I} (R_{1, 1} \leq \mu_1)}{\mu_1 - R_{1, 1}} - \frac{1}{\mu_1} {\mathbb{E}} R_{1, 1} e^{\mu_1^{-1}R_{1, 1}}\mathbb{I} (R_{1, 1} \leq \mu_1) := I_1 - I_2
    \end{aligned}
\end{equation*}
satisfying
\begin{equation*}
    \begin{aligned}
        I_1 \leq \frac{e}{3} \sum_{k = 1}^{\infty} {\mathbb{E}} (\mu_1^{-1}R_{1, 1})^k \mathbb{I} (R_{1, 1} \leq \mu_1)  \leq \frac{e}{3} {\mathbb{E}} \exp (\mu_1^{-1} R_{1, 1})\mathbb{I} (R_{1, 1} \leq \mu_1) \leq \frac{e^2}{6} \cdot \exp \left\{ \frac{1}{2 \mu_1^2 \sigma_1^2}\right\}
    \end{aligned}
\end{equation*}
and
\begin{equation*}
    \begin{aligned}
        I_2 \geq \frac{1}{\mu_1} {\mathbb{E}} R_{1, 1} = 1
    \end{aligned}
\end{equation*}
impling $I \leq e^{2 + 1/(2 \mu_1^2 \sigma_1^2)} / 6 - 1$. And similarly, we have $II \leq e^{1 + 1/(2 \mu_1^2 \sigma_1^2)} / 2 - 1$, which gives the bound 
\begin{equation*}
    a_{k, s} \leq \big( e^{2 + 1/(2 \mu_1^2 \sigma_1^2)} / 6 + e^{1 + 1/(2 \mu_1^2 \sigma_1^2)} / 2 - 3 \big)^{|\mathcal{H}_{1, s}|}, \qquad k \in [K], \qquad s = 0, \ldots, T - 1
\end{equation*}
as long as $R_{1, 1}$ is psuedo-symmetric sub-Gaussian variable with 
\begin{equation*}
    {\mathbb{E}} e^{(R_{1, 1} - \mu)t} \mathbb{I}(R_{1, 1} \leq \mu_1) \vee {\mathbb{E}} e^{(R_{1, 1} - \mu)t} \mathbb{I}(R_{1, 1} > \mu_1) \leq \frac{1}{2} e^{\frac{t^2\sigma_1^2}{2}},
\end{equation*}
and $y := 1 / (2 \mu_1^2 \sigma_1^2)$ gives $e^{2 + y}/ 6 + e^{1 + y} / 2 - 3 \in (0, 1)$, i.e. $\mu_1 \sigma_1 > \sqrt{\frac{e (e + 3)}{48}} \approx 0.5690618$, which implies the mean reward of the optimal arm cannot be too small.

\noindent Then we decompose $a_{k,s}$ as $a_{k, s} = a_{k, s, 1} + a_{k, s, 1} + a_{k, s, 1}$
\begin{gather*}
    a_{k, s, 1} := {\mathbb{E}} \big[ (N_{1, s} (\tau_k) \wedge T) \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s})\big] \\
    a_{k, s, 2} := {\mathbb{E}} \big[ (N_{1, s} (\tau_k) \wedge T) \mathbb{I} (A_{1, s}^c) \mathbb{I} (G_{1, s})\big] \\
    a_{k, s, 3} := {\mathbb{E}} \big[ (N_{1, s} (\tau_k) \wedge T) \mathbb{I} (G_{1, s}^c)\big],
\end{gather*}
the next step we need to do is bounding $s_{a, j} := \{s : a_{k, s, j} \leq T^{-1} \}$ for $j = 1, 2, 3$.
\vspace{2ex}

\noindent $\circ$ $\circ$ \textbf{Bound $s_{a, 1}$:} 

\noindent First note that $\{ N_{1, s}(\tau_k) < T^{-1} \} \supseteq \{ N_{1, s}(\tau_k) < (T^2 - 1)^{-1} \} = \{ [1 - Q_{1, s}(\tau_k)] < T^{-2}\}$ with
\begin{equation*}
    \begin{aligned}
        1 - Q_{1, s}(\tau_k) = \mathbb{P} (\overline{Y}_{1, s} \leq \mu_1 \, | \, \mathcal{H}_{1, s}) \leq \mathbb{P} ()
    \end{aligned}
\end{equation*}
}}


\fi


\iffalse
{\color{red}{
As we can see, Algorithm 2 and 3 as a practical supplementary for Algorithm 1. Hence, it is enough to deal with Algorithm 1.
Note that in Algorithm 1, we have $\overline{Y}_{k, s} =  \sum_{\ell = 1}^{|\mathcal{H}_s|} w_{\ell} \cdot R_{k, \ell}$ for $k \in [K]$ and $s \in [T]$ with $\omega_{\ell} \overset{\text{i.i.d}}{\sim } \rho(\omega)$, it is enough to deal with the algorithm 1 with $\overline{Y}_{k}$ replaced by $\overline{Y}_{k, s} $. 

Denote $Q_{k, s} (\tau) = \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_s  \big)$ and $N_{k, s} (\tau) = 1 / Q_{k, s} (\tau) - 1$. Then from Theorem 1 in \cite{kveton2019garbage}, we have 
\begin{equation*}
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
\end{equation*}
with
\begin{equation*}
    a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] = \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1
\end{equation*} 
for any $\tau_k \in \mathbb{R}$. So it is enough to bound $a_k$ and $b_k$.

\noindent $\circ$ \textbf{Bound $a_k$:}

\noindent Define the good events:
\begin{equation*}
    A_{1, s} := \big\{ \overline{Y}_{1, s} - \mu_1 > - \Delta_k / 4 \big\}, \qquad G_{1, s} := \{ \}.
\end{equation*}
For $\tau_k \leq \mu_1$, we have $a_{k, s} \leq {\mathbb{E}} Q_{1, s}(\tau_k)^{-1} \leq {\mathbb{E}} \big[ \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s})^{-1} \big]$. To deal with tail probability, we need to use lower bound of tail probability of Gaussian variables, such that
\begin{equation*}
    \begin{aligned}
        \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s}) & = \mathbb{P}_{\omega} \left( \sum_{\ell = 1}^{|\mathcal{H}_s|} \omega_{\ell} R_{1, \ell} > \mu_1 \right) \\
        (\text{assume } \omega_s \sim N(0, \sigma_{\omega}^2)) & \geq \left\{ \begin{array}{ll}
            \exp \left\{ - \frac{\mu_1^2}{\sigma_{\omega}^2 \sum_{\ell = 1}^{|\mathcal{H}_s|} R_{1, \ell}^2} \right\},  &  \mu_1 \geq  \sigma_{\omega} \| R_{1, s}\|_2\\
            1 - \Phi(\sqrt{2 \pi}), & \mu_1 < \sigma_{\omega} \| R_{1, s}\|_2
        \end{array}\right.
    \end{aligned}
\end{equation*}
where $\| R_{1, s}\|_2^2 = {\sum_{\ell = 1}^{\mathcal{H}_s} R_{1, \ell}}^2 $. Hence
\begin{equation*}
    \begin{aligned}
        a_{k, s} & \leq {\mathbb{E}} \big[ \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s})^{-1} \big] \\
        & \leq {\mathbb{E}} \exp \left\{ \frac{\mu_1^2}{\sigma_{\omega}^2 \| R_{1, s}\|_2^2} \right\} \mathbb{I} \big(\| R_{1, s}\|_2 \leq \sigma_{\omega}^{-1} \mu_1 \big) + \big[ 1 - \Phi(\sqrt{2 \pi}) \big]^{-1} \mathbb{P} \big( \| R_{1, s}\|_2 >  \sigma_{\omega}^{-1} \mu_1  \big)
    \end{aligned}
\end{equation*}

\begin{equation*}
    \begin{aligned}
        \mathbb{P} (\overline{Y}_{1, s}^{b_s} > \mu_1 \, | \, \mathcal{H}_{1, s}) & = \mathbb{P} \big(\overline{Y}_{1, s}^{B} > \mu_1 \, | \, \mathcal{H}_{1, s} \big ) \\
        & \leq \mathbb{P} \left( \overline{Y}_{1, s}^{B - 1} > \frac{(c_{1, s} + 1)\mu_1 - R_s \cdot \omega_B}{c_{1, s}} \, \Big| \, \mathcal{H}_{1, s }\right) \\
        & \leq \mathbb{P} \left( \overline{Y}_{1, s}^{0} > \Big( \frac{c_{1, s} + 1}{c_{1, s}}\Big)^B \cdot \mu_1 - \frac{R_s}{c_{1, s}} \sum_{b = 1}^B \omega_b \, \Big| \, \mathcal{H}_{1, s} \right) \\
        & \leq  \mathbb{P} \left( \sum_{b = 1}^B w_b > \Big\{ \Big( \frac{c_{1, s} + 1}{c_{1, s}}\Big)^B \mu_1 - \overline{Y}_{1, s}^{0} \Big\} \cdot \frac{c_{1, s}}{R_s} \, \Big| \, \mathcal{H}_{1, s} \right) \\
        & \leq \exp \left\{ - \frac{2}{n} \left(\Big\{ \Big( \frac{c_{1, s} + 1}{c_{1, s}}\Big)^B \mu_1 - \overline{Y}_{1, s}^{0} \Big\} \cdot \frac{c_{1, s}}{R_s} - \frac{B}{2} \right)^2 \right\} 
    \end{aligned}
\end{equation*}
Note the fact that $\overline{Y}_{1, s}^0 = \frac{1}{c_{1, s} - 2} \sum_{\ell = 1}^{c_{1, s} - 2} Y_{1,\ell}$, hence
\begin{equation*}
    \begin{aligned}
        a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{2}{n} \left(\Big\{ \Big( \frac{c_{1, s} + 1}{c_{1, s}}\Big)^B \mu_1 - \overline{Y}_{1, s}^{0} \Big\} \cdot \frac{c_{1, s}}{R_s} - \frac{B}{2} \right)^2 \right\} \\
        & \leq e^ {\frac{B^2}{2 n}} \cdot {\mathbb{E}} \Bigg\{ {\mathbb{E}} \exp \bigg[\frac{2 c_{1, s}^2}{n R_s^2 } \big( \overline{Y}_{1, s} - \mu_1\big)^2 \, \Big| \, R_s  \bigg] \cdot \exp \bigg[ \frac{2 c_{1, s}^2 \mu_1^2}{n R_s^2 } \Big( 1 - \frac{(1 + c_{1, s})^B}{c_{1, s}^B}\Big)^2 \bigg] \Bigg\} \\
        & \leq e^ {\frac{B^2}{2 n}} {\mathbb{E}} \exp \bigg[ \frac{2 c_{1, s}^4}{n^2 R_s^4 } \frac{16\sigma_1^4}{(c_{1, s} - 2)^2} + \frac{2 c_{1, s}^2 \mu_1^2}{n R_s^2 } \Big( 1 - \frac{(1 + c_{1, s})^B}{c_{1, s}^B}\Big)^2 \bigg] \\
        & = 
    \end{aligned}
\end{equation*}
by the fact
\begin{equation*}
    \overline{Y}_{1, s}^0 - \mu_1 \, \sim \, \operatorname{subG} \left( \frac{\sigma_1^2}{c_{1, s} - 2}\right) \quad \text{ and } \quad \big(\overline{Y}_{1, s}^0 - \mu_1 \big)^2 \, \sim \, \operatorname{subE} \left( \frac{16\sigma_1^2}{c_{1, s} - 2}\right).
\end{equation*}
}}
\fi


1. nonrandom or random weights in each step?











\input{Related_Work}

\clearpage

\bibliographystyle{aaai2021.bst}





\section{Introduction}

We start our discussion on MBTS with the multi-armed bandit (MAB) problem. For any positive integer $M$, let $[M] = \{1, \dots, M\}$. The agent sequentially choose arm $A_t \in [K]$, and then receive a stochastic reward $R_t$. 



At each round $t$, the agent needs to choose an action $I_t \in \mathcal{A}_t := [K]$. The goal is to minimize the cumulative regret 
\begin{equation}\label{eq:REG}
    \operatorname{Reg}_{T} := {\mathbb{E}} \operatorname{Reg}_{T}(Y,\mathcal{A}) = {\mathbb{E}} \sum_{t=1}^{T} [\mu_1-\mu_{A_{t}}]=\sum_{k=2}^{K} \Delta_{k} {\mathbb{E}} \Big[\sum_{t=1}^{T} 1 \left\{A_{t}=k\right\}\Big],
\end{equation}
with assuming the first arm is the optimal one, i.e. $\mu_1 = \max_{k \in [K]} \mu_k$. From \cite{kveton2019garbage}, we know that the optimal rate for $\operatorname{Reg}_{T} $ is exactly $\log T$. Hence, researchers' aim is finding an algorithm making the cumulative regrets achieve the optimal rate. 
\vspace{2ex}

Generally speaking, there are two popular algorithmic frameworks, Upper Confidence Bound (UCB) and Thompson Sampling (TS):

\begin{itemize}
    \item UCB adopts a frequentist viewpoint and construct an upper confidence bound for $\mu_k$ as $\widehat{U}_k$, for every $k \in [K]$, UCB then chooses $I_t = \arg \max_{k \in [K]} \widehat{U}_{k}$;
    
    \item TS adopts a Bayesian viewpoint, maintain a posterior over the model, and sample action from it. Specifically, we can maintain a posterior over $k \in [K]$ as $P_{\omega_k} \in \mathcal{P}$, sample $\omega_k \sim P_{\omega_k}$, and then choose $I_t = \arg \max_{k \in [K]} Y(k, \omega_k)$ with some target function $Y(\cdot, \omega_k): [K] \rightarrow \mathbb{R}$ similar to $\widehat{U}_k$ in UCB. This is equivalent to directly maintaining a posterior over $I^*_t = \arg \max_{k \in [K]} Y_k$ and then sample $I_t$.
\end{itemize}

Here have been several articles touch this problem, and as we said before, they either base on UCB or TS. It is worthy to note that for a specific problem, i.e. MAB with some particular type of random rewards, the UCB method is partly restricted. This is by the fact that once we got the Bootstrap upper bound with weights $\mathbf{w}_n = (w_1, \ldots, w_n)^{\top} \in \mathbb{R}^n$ such that
\begin{equation*}
    \mathbb{P}_{\mathbf{R}_{k, n}} \left[ \frac{1}{n} \sum_{i = 1}^n R_{k, i} - \mu_k > q_{\alpha} (\mathbf{R}_{k, n}, \mathbf{w}_n)\right] \leq \alpha,
\end{equation*}
then that the algorithm will be exactly become $I_t = \arg \max_{k \in [K]}  \left[ \frac{1}{n} \sum_{i = 1}^n R_{k, i} + q_{\alpha} (\mathbf{R}_{k, n}, \mathbf{w}_n) \right]$ with some pre-specific $\alpha \asymp o(T^{-1})$. In the contrast, TS method can vary widely since there are many choices of the Bayesian posterior $P_{\omega_k}$, we conclude the existing literature is listed in chronological order as follows
\begin{itemize}
    
    \item \cite{lu2017ensemble} is the first one to that put TS and try to theoretically analyze the regret in the Bandit problem. However, their results are neither nonasymptotic and problem specific.
    
    \item \cite{vaswani2018new} throw an insight for how to theoretically analyze the cumulative regret under nonasymptotic sense at Bootstrap framework. Here the nonasymptotic means we can get the relationship of the regret and the round $T$ exactly. 
    
    \item \cite{kveton2019garbage} is the pioneer, which exactly use the TS to achieve the optimal rate. It adds $a$ pseudo rewards from $(0, 1)$ with tuning parameter $a$ on each arm's history, and prove this method guarantees an $\log T$ under Bernoulli rewards on $\{ 0, 1\}$;
    
    
    \item \cite{kveton2019perturbed} again uses another Bayesian posterior to deal with the same problem with \cite{kveton2019perturbed}: it adds $a$ Bernoulli perturbation at each reward step, the $a$ is again the tuning parameter. It is worthy to note that this method is quite similar to \cite{kveton2019garbage} sharing with the same condition (under some transformation);
    
    \item \cite{wang2020residual} construct another Bayesian posterior based on the residuals $R_{k, i} - \overline{R}_{k}$, extended the existent theory and the optimal rate $\log T$ to a specific unbound reward, Gaussian setting. Their theory is exactly based on the exact tail of Gaussian distribution.
\end{itemize}

We have to say that the existent literature, which achieves the optimal regret rate, are ALL computationally heavy. As we can see, \cite{kveton2019garbage} and \cite{kveton2019perturbed} need to add $a$ pseudo random quantile at each round, their storage cost and computation cost are $\mathcal{O}(aT), \mathcal{O} (aT^2)$ and $\mathcal{O}(aT), \mathcal{O} (aKT)$ representatively. While \cite{wang2020residual} achieves the rate $\mathcal{O}(T), \mathcal{O} (KT)$, they need EXTRA steps to compute the residuals.
\vspace{2ex}

Compared with the existent methods, our main contributions are
\begin{itemize}
    \item[1.] Our method is directly base on TS method, no other transformation is required. And our storage cost and computation cost are exactly $\mathcal{O}(T), \mathcal{O} (KT)$, no any other constants will be included;
    
    \item[2.] Our theoretical analysis is only dependent on the tail probability of the Bootstrap weights $\boldsymbol\omega$. As long as the rewards are bounded, our method can be applied. (instinct uncertainty or \cite{kveton2019perturbed} will use the pseudo rewards based on the known reward property, feasibility and empiricality)
    
    \item[3.] Our method can be easily extended to linear bandit and more complex setting.
    
\end{itemize}

\section{Algorithm and Theoretical Analysis}

\begin{algorithm}[!h]
\SetAlgoLined
\KwData{
number of arms $K$, 
multiplier weight distribution $\rho(\omega)$
}

Set $\mathcal{H}_{k, s}$ is the history of the arm $k$ with exact $s$ rewards. and $\overline{Y}_{k, 0} = -\infty, \forall k \in [K]$

\For{t = 1, \dots, T}{

    Offer $I_t = \arg \max_{k \in [K]} \overline{Y}_{k, s}$ (break tie randomly)

    Observe reward $R_t$

    \For{k = 1, \dots, K}{
    Sample the multiplier weights $\{\omega_l\}_{l=0}^{s} \sim \rho(\omega)$. {\color{blue}{Note that ${\mathbb{E}} \omega_{\ell}$ should be equal to one to ensure the consistency.}}
    
    Update the mean reward $\overline{Y}_{k, s} = \left(\omega _0\lambda s + \sum_{\ell = 1}^{s} \omega_{\ell} R_{k, \ell} \right) / \left( \sum_{\ell = 1}^{s} \omega_{\ell} \right)$, where $R_{k, l}$ is the $l$-th non-zero element of $\mathcal{H}_{k, t}$ with some tranformation $L$ and $s = |\mathcal{H}_{k, t}|$.
    }
    
} 
\caption{TOY Algorithm: Multiplier Bootstrap-based Thompson Sampling for MAB with $[0, 1]$ rewards}\label{alg:MBTS-MAB}
\end{algorithm}

\begin{theorem}
    Consider a $K$-armed bandit where the reward distribution of arm $k$ is bounded. Then, the $T$ round regret of our method satisfies:
    \begin{equation*}
        \operatorname{Reg}_T \lesssim \log T,
    \end{equation*}
    where the constant $C$ only depends on the bounds of rewards and $\lambda$.
\end{theorem}






\section{Proof of Theorem \ref{thm.MAB}}



The proof Theorem \ref{thm.MAB} can be divided into three main parts. The first part is recapping a decomposition of the cumulative regret as our proof relies on the following decomposition from \cite{kveton2019garbage}. 
We recap its proof in Appendix \ref{sec:proof_lemma} for completeness. 


\begin{lemma}\label{lem.a.b}
Recall that we select arms according to the rule 
$A_t = \arg \max_{k \in [K]} \overline{Y}_{k, t}$. 
For any $\{ \tau_k \}_{k = 2}^K \subseteq \mathbb{R}^{K - 1}$, 
we have the bound
\begin{equation*}
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
\end{equation*}
with
\begin{equation*}
    a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] =: \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \sum_{s = 0}^{T - 1} \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1 =: \sum_{s = 0}^{T - 1} b_{k, s} + 1
\end{equation*} 
where $Q_{k, s} (\tau) = \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_{k, T} \big)$ and $N_{k, s} (\tau) = 1 / Q_{k, s} (\tau) - 1$.
\end{lemma}

From the definition of $a_{k, s}$ and $b_{k, s}$, $a_k$ controls that the tail probability of the optimal arm is not too small, while $b_k$ controls that the tail probabilities of rewards of all arms are not too large in general. 
It is worthy to note that for here we only need to consider the lower bound of the tail of the optimal arm. The intuition is that we only need to the probability of taking a relatively large value in the optimal arm is not too small. And we do not care about how small the probability of a large value occurred in the other arms. And this Lemma is valid for any algorithm in MAB problem as long as the action step is $A_{t} = \arg \max_{k \in [K]} \overline{Y}_{k, t}$.


Based on Lemma \ref{lem.a.b}, it is enough to bound $a_k$ and $b_k$ separately. We will do this as the following framework.


\todorw{$\cdot$ is also typically not recommended. Suggest to either remove, or replace with $\times$ when necessary.}

\begin{tikzpicture}[node distance=2cm]
    \node (step1) [startstop] {Decomposition with Lemma \ref{lem.a.b}}; 
    
    \node (step2) [process, below of=step1, xshift=-5cm] {Step I : Bound $a_k = \sum_{s = 0}^{T - 1} a_{k, s}$ as order $\log T$}; 
    
    \node (step2.0) [process, below of=step2, xshift=0cm] {Consider $s_a := \max \{{s : a_{k, s}} \geq T^{-1} \}$, then $$a_k \leq \max_{s} a_{k, s} \cdot s_a + T^{-1} \cdot (T - s_a).$$};
    
    \node (step2.1) [startstop, below of=step2.0, xshift=-2.5cm] {Bound $a_{k, s}$ with a constant};
    
    \node (step2.1.1) [startstop, below of=step2.0, xshift=-2.5cm] {Bound $a_{k, s}$ with a constant};
    
    \node (step2.1.2) [startstop, below of=step2.1.1, xshift=-1cm] {Use anti-concentraction};
    
    \node (step2.2.1) [startstop, below of=step2.0, xshift=2.5cm] {Bound $s_a$ as $\log T$};
     
    \node (step2.1.2) [startstop, below of=step2.2.1, xshift=-0.5cm] {May need to decompose $a_{k, s} = \sum_{j = 1}^3 a_{k, s, j}$};
     
    \node (step3) [process, below of=step1, xshift=4cm] {Step II : Bound $b_k = \sum_{s = 0}^{T - 1} b_{k, s} + 1$ as order $\log T$}; 
    
    \node (step3.0) [process, below of=step3, xshift=0cm] {Consider $s_b := \max \{{s : b_{k, s}} \geq T^{-1} \}$, then $$b_k \leq s_b + T^{-1} \cdot (T - s_b).$$};
    
    \node (step3.1) [startstop, below of=step3.0, xshift=0cm] {Bound $s_b$ as $\log T$};
    
    \node (step3.2) [startstop, below of=step3.1, xshift=0cm] {Also need to decompose $b_{k, s} = \sum_{j = 1}^3 b_{k, s, j}$};
    
    \draw [arrow] (step1) -- (step2);

    \draw [arrow] (step1) -- (step3);
    
\end{tikzpicture}




\subsection{First Step: Bounding $a_k$}

As the diagram above, for bounding $a_k$ as order $\log T$, what we need to do first is up-bounding $a_{k, s}$ with a constant (free of $T$), and then bound $s_a$ as order $\log T$

\subsubsection{Bound $a_{k,s}$ as a constant}

For doing this, note that if
\begin{equation}\label{condition.1}
    \tau_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda}
\end{equation}
we will have
\begin{equation*}
    a_{k, s} \leq {\mathbb{E}} Q_{1, s}(\tau_k)^{-1} \leq  {\mathbb{E}} \left[ Q_{1, s}^{-1}\left(  \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right) \right] = {\mathbb{E}} \Bigg[ \mathbb{P} \bigg(\overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda} \, \Big| \, \mathcal{H}_{1, T} \bigg)^{-1} \Bigg],
\end{equation*}
we need to first find the lower bound of the tail probability $\mathbb{P} (\overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda} \, | \, \mathcal{H}_{1, T})$ and then bound its expectation over the optimal arm's history $\mathcal{H}_{1, T}$, where $\mathcal{H}_{k, t}$ represents the rewards of $k$-th arm up to $t$-th round.




Recall in our algorithm, the target function is for some fixed arm $k$. 
For a fixed arm $k$, we can denote that $s := |\mathcal{H}_{k, T}|$ as the total number of pulling the $k$-th arm. 
Under this simplicity, we have 
\begin{equation*}
    \overline{Y}_{k, s} = \frac{\sum_{i = 1}^s (\omega_{i} R_{k, i} + \lambda  \omega_{i}') }{\sum_{i = 1}^s (\omega_{i} + \lambda\omega_{i}' + \lambda \omega_{i}'')}. 
\end{equation*}
We use $\mathbb{P}_{\omega} (A)$ to emphasize that the probability is only taken over the random variable $\omega$ alone. We have 
\todorw{The reason that I added $| H$ in line 3 before, is because the current notation ($\mathbb{P}_{\omega} (A)$ ) is not rigorously defined.}
\begin{equation*}
        \begin{aligned}
            & \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda} \, \Big| \, \mathcal{H}_{1, s} \right) \\
            = \, & \mathbb{P} \bigg( \sum_{i = 1}^s \omega_i (R_{1, i} - \mu_1) + \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s  (\omega_i' - \omega_i'') + \lambda \sum_{i = 1}^s (2 \omega_i R_{1, i} - \omega_i' \mu_1 - \omega_i'' \mu_1) > 0 \mid \ \mathcal{H}_{1, s} \bigg) \\
            = \, &  \mathbb{P}_{\omega, \omega', \omega''} \bigg\{ \sum_{i = 1}^s \big( (2 \lambda + 1) R_{1, i} - (\mu_1 + \lambda)\big) \omega_i > \lambda (\mu_1 - 1 - \lambda) \sum_{i = 1}^s \omega_i' + \lambda (\lambda + \mu_1) \sum_{i = 1}^s \omega_i''  \bigg\} \\
            \geq \, & \exp \bigg\{ - \frac{3s(2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left[s \lambda^2 (\lambda + 1 - \mu_1)^2 + s\lambda^2 (\lambda + \mu_1)^2\right]}\bigg\} \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  \\
            & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~ + \big[ 1 - \Phi(\sqrt{2 \pi})\big] \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \\
            \geq \, & \exp \bigg\{ - \frac{3(2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2\lambda^2  \left[ (\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2\right]}\bigg\} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}.
        \end{aligned}
\end{equation*} 
where the last inequality is based on the property of Gaussian random variables, which we summarize in the following lemma. 

\begin{lemma}\label{lemma:gaussian_weight}
    Suppose $x_i, y_i, z_i \in \mathbb{R}$ and 
   
   
   
    $\{\omega_i, \omega_i', \omega_i''\}$ are independent $N(1, \sigma_{\omega}^2)$ random variables, then
    \begin{equation*}
        \begin{aligned}
            & \mathbb{P} \bigg( \sum_{i = 1}^n x_i \omega_i > \sum_{i = 1}^n y_i \omega_i' + \sum_{i = 1}^n z_i \omega_i'' \bigg) \\
            \geq & \exp \bigg\{ - \frac{3(S_x - S_y - S_z)^2}{2 \sigma_{\omega}^2(T_x + T_y + T_z)}\bigg\} \cdot 1 \big(S_x \geq S_y + S_z + \sqrt{2\pi} \big) + \big( 1 - \Phi(\sqrt{2\pi} )\big) \cdot 1 \big(S_x \leq S_y + S_z + \sqrt{2\pi}\big), 
        \end{aligned}
    \end{equation*}
    for any $n \in \mathbb{N}$ where we denote $S_x = \sum_{i = 1}^n x_i$ and $T_x = \sum_{i = 1}^n x_i^2$, and similar notation for $S_y, S_z, T_y, T_z$.
\end{lemma}


\iffalse
\begin{remark}
    If we choose to use same weight version, Algorithm 3, to analyze, we need to compute 
    \begin{equation*}
    \begin{aligned}
        \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu + \lambda}{1 + 2 \lambda}\right) = \mathbb{P} \bigg( (2\lambda + 1) \sum_{i = 1}^s \omega_i (R_{1, i} - \mu_1) > 0 \bigg) = \mathbb{P} \bigg( \sum_{i = 1}^s \omega_i (R_{1, i} - \mu_1) > 0 \bigg).
    \end{aligned}
    \end{equation*}
    It can be seen that this degenerate to the analysis in Algorithm 1.
\end{remark}
\fi



Hence, we immediately have that $a_{k, s}$ is bounded by
\begin{equation*}
    \begin{aligned}
        a_{k, s} & \leq {\mathbb{E}} \exp \bigg\{\frac{3(2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \lambda^2 ((\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2 )}\bigg\} + \big[ 1 - \Phi(\sqrt{2\pi} )\big]^{-1}  \\
        & \leq \exp \bigg\{\frac{3(2 \lambda + 1)^2 \cdot 1}{2 \sigma_{\omega}^2 \lambda^2 ((\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2 )}\bigg\} + \big[ 1 - \Phi(\sqrt{2\pi} )\big]^{-1} \\
        & \leq \exp \bigg\{\frac{3(2 \lambda + 1)^2}{2 \sigma_{\omega}^2 \lambda^2 \big[(\lambda + 1 )^2 + \lambda^2 \big]}\bigg\} + \big[ 1 - \Phi(\sqrt{2\pi} )\big]^{-1} 
    \end{aligned}
\end{equation*}
where we utilize the \red{boundedness} of the rewards. Hence, we get an upper bound for $a_{k, s}$.

\red{
    In this step, we can see that to bound $a_{k, s}$, one only need to have the MGF of $(\overline{R}_{1, s} - \mu_1)^2$ is bounded, hence the bounded assumption is not essential here. 
   
}


\todorw{$(\overline{R}_{1, s} - \mu_1)^2$ is sub-exponential, the MGF of which is only bounded in an interval around $0$. We can hence derive the required condition on $\lambda$ and $\sigma_{\omega}$.}
\todohw{Yes, $(\overline{R}_{1, s} - \mu_1)^2$ is sub-exponential when $R_{k, i}$ is sub-Gaussian. Indeed, the restriction of tuning parameter $\lambda$ in other literature is always derived from this step.}
\todorw{So, if we have time, I suggest to derive it (after finishing the other parts)}



\subsubsection{Bound $s_a$ with a order $\log T$}

Let $a_T \lesssim b_T$ represent $a_T \leq c  b_T$ for some constant $c$.
After bounding $a_{k, s}$, then next step is to find $s_a := \max \{s: a_{k, s} \geq  T^{-1}\}$ and prove that $s_a \lesssim \log T$. However, this may be difficult by the instinct construction of $a_{k, s}$. Fortunately, we can decompose $a_{k, s}$ as $a_{k, s} = \sum_{j = 1}^m a_{k, s, j}$ with some simple $a_{k, s, j}$, and then construct $s_{a, j} := \max \{ s: a_{k, s, j} \geq T^{-1} \}$ and show $s_{a, j} \lesssim \log T$. Recall in the proof sketch diagram we have the decomposition that
\begin{equation*}
    a_k \leq \max_{s} a_{k, s} \cdot \max_j s_{a, j} + m T^{-1} \cdot (T - \max_j s_{a, j}).
\end{equation*}
and we have bounded $a_{k, s}$ as a constant, what remains is bounding $s_{a, j}$ for $j \in [m]$ as $\log T$.

For establish simpler $a_{k, s, j}$, a common method is decomposing $a_{k, s}$ on some good events with their complement sets. Recall the essential ingredient in the algorithm is based on the mean rewards of each arm. We denote the shifted version of average reward as
\begin{equation*}
    \overline{R}_{k, s}^* := \frac{\sum_{i = 1}^s R_{k, i} + \lambda s}{s (1 + 2 \lambda)} = \frac{\lambda}{1 + 2 \lambda} + \frac{1}{1 + 2\lambda } \overline{R}_{k, s} + \frac{\lambda}{1 + 2 \lambda} \cdot 0,
\end{equation*}



where $\overline{R}_{k, s} = \frac{1}{s} \sum_{i = 1}^s R_{k, i}$ and then define good events\footnote{In the original version: we set that $A_{k, s} := \{ \overline{R}_{k, s}^* - \mu_k > - C_1 \Delta_k\}$ and $A_{k, s}^{\prime} := \{ \overline{R}_{k, s}^* - \mu_k < C_2 \Delta_k\}$} with some $C_1, C_2 \in (0, 1)$,
\begin{equation*}
    A_{k, s} := \left\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2\lambda} > - C_1 \Delta_k\right\}, \qquad A'_{k, s} := \left\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2\lambda} < C_2 \Delta_k\right\}
\end{equation*}
\iffalse
\todohw{The major difference between now and previous verison is that the original verison $C_1, C_2$ do not relate to tuning parameter $\lambda$. Indeed, I do not know whether is correct that $C_1$ and $C_2$ are functions of $\lambda$ (which will furthermore ask that $\tau_k$ depends on $\lambda$)...}
\todohw{There is no way to avoid this problem as $\lambda$ appears on the denominator. The only possible solution is to drop the denominator...}
\todohw{Although we can take any $\{ \tau_k\}$, but I am not sure whether they can depend on $\lambda$.}
\todohw{However, I find there is no way to use $A_{k, s}$ in the previous verison, as it will lead to 
\begin{equation*}
    - C_1 \Delta_k (1 + 2\lambda) + 2 \lambda (2 \mu_1 - 1) \leq 0,
\end{equation*}
i.e. $C_1 \Delta_k (1 + 2 \lambda) > \lambda (2 \mu_1 - 1)$, which asks $\mu_1 \leq \frac{1}{2}$. And then the iternation by the algorithm will make $\mu_1 \equiv 0$.
}
\todohw{I find that if we change the $\overline{Y}_{k, s}$ to 
\begin{equation*}
    \overline{Y}_{k, s} = \frac{\omega R + \lambda \omega'}{\omega + \omega'}
\end{equation*}
all proof can be give with the original $A_{k, s}$. But the problem is how to explain.
}
\fi
Define:
\begin{equation*}
    a_{k, s, 1} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) \Big], \qquad a_{k, s, 2} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}^c) \Big],
\end{equation*}
with $s_{a, j} := \max \{ s: a_{k, s, j} \geq T^{-1} \}$. What we need to do is bounding $s_{a, j}$ as order $\log T$.



\iffalse
In this step, our target is to show that 
\begin{align*}
\max_{j = 1, 2} s_{a, j} \lesssim \log T, 
\end{align*}
which implies
\begin{equation*}
    \begin{aligned}
        a_k & = \sum_{s = 0}^{T - 1} a_{k, s} \leq \max_{s} a_{k, s} \cdot \max_{j = 1, 2} s_{a, j}  + (T -  \max_{j = 1, 2} s_{a, j}) \cdot (T^{-1} + T^{-1})\\
        & \leq \left[ \exp \bigg\{\frac{3(2 \lambda + 1)^2 ((1 - \mu_1)^2 + \mu_1^2 )}{4 \sigma_{\omega}^2 \lambda^2 ((\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2 )}\bigg\} + \big[ 1 - \Phi(\sqrt{2\pi} )\big]^{-1} \right] \cdot & \max_{j = 1, 2} s_{a, j} (T)  + (T - \max_{j = 1, 2} s_{a, j} (T)) \cdot 2 T^{-1} \\
        & \lesssim 2 + \log T.
    \end{aligned}
\end{equation*}
\fi


\paragraph{Tackling $s_{a, 2}$}
The construction of $\overline{R}_{k, s}^*$ indicates that the behaviour of $\overline{R}_{k, s}^*$ should be close to the true quantity $\overline{Y}_{k, s}$. 
Therefore, we first focus on studying the probability of the bad event $A_{1, s}^c$ defined previously. 
By the Hoeffding's inequality, we have\footnote{In the original verison we have :$\mathbb{P} (A_{1, s}^c) = \mathbb{P} \bigg( \mu_1 - \overline{R}_{1, s} \geq C_1 \Delta_k + \frac{\lambda}{1 + 2 \lambda}\bigg) \leq \exp \bigg\{ - 2s \Big(  C_1 \Delta_k + \frac{ \lambda}{1 + 2 \lambda} \Big) \bigg\} \leq T^{-2}, $}
\begin{equation*}
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \bigg( \frac{\overline{R}_{1, s} - \mu_1}{1 + 2 \lambda} \leq - C_1 \Delta_k \bigg) = \mathbb{P} \big( \mu_1 - \overline{R}_{1, s} \geq (1 + 2 \lambda) C_1 \Delta_k \big) \\
        & \leq \exp  \bigg\{ - \frac{2(1 + 2 \lambda)^2 C_1^2 \Delta_k^2 s^2 }{s \cdot (1 - 0)^2}\bigg\} = \exp \big\{ - 2 s C_1^2 (1 + 2\lambda)^2 \Delta_k^2  \big\}
    \end{aligned}
\end{equation*}
as $\{ R_{k, i} \}_{i = 1}^s$ are all bounded in $[0, 1]$.
If we take 
\begin{equation*}
    s \geq s_{a, 2} : = \frac{\log T}{C_1^2 (1 + 2\lambda)^2 \Delta_k^2}
\end{equation*}
and then we will have
\begin{equation*}
    a_{k, s, 2} \leq T {\mathbb{E}} 1\big( A_{1, s}^c \big) \leq T^{-1}.
\end{equation*}
i.e the above $s_{a, 2}$ is what we need.


\begin{remark}
    In this step, we use the \red{boundedness} of the rewards $R_{k, s}$ so that we can apply the Hoeffding's inequality. However, the boundedness assumption is not essential to derive the final bound above. 
    In fact, as long as we have 
    \begin{equation*}
        \log \mathbb{P} \big( \mu_1 - \overline{R}_{1, s} \geq (1 + 2 \lambda) C_1 \Delta_k \big) \lesssim -s, 
    \end{equation*}
    we can still find a proper $s_{a,2}$. It can be easy to hold when the rewards are sub-Gaussian variables.
\end{remark}






\paragraph{Dealing with $s_{a, 1}$}

To study $a_{k, s, 1}$, recall we have $a_{k, s, 1} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) \Big] \approx {\mathbb{E}} N_{1, s} (\tau_k)$ as we have large possibility on the good event $A_{1, s}$: we have $\mathbb{P} (A_{1, s}) \geq 1 - T^{-2}$ when we have $s \geq s_{a, 2}$.
However, as $N_{k, s}(\tau_k)$ depends on $Y_{k, s}$ rather than $R_{k, s}$. It is worthy to study the gap between them, and define another good event based on the gap. 

The gap between $\overline{Y}_{1, s}$ and $\overline{R}_{1, s}^*$ is
\begin{equation*}
    \begin{aligned}
        \overline{Y}_{1, s} - \overline{R}_{1, s}^* \, | \, \mathcal{H}_{1, T} = \frac{D_1 + D_2 + D_3 }{(1 + 2 \lambda)\sum_{i = 1}^s  (\omega_i + \lambda \omega_i' + \lambda \omega_i'')}
    \end{aligned}
\end{equation*}
with
   
   
   
\begin{gather*}
    D_1 : = \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) \omega_i, \\
    D_2 := \lambda \sum_{i = 1}^s (2 R_{1, i} \omega_i - \overline{R}_{1, s} \omega_i' - \overline{R}_{1, s} \omega_i''), \\
    D_3 := \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s (\omega_i' - \omega_i''). 
\end{gather*}
Notice that we have $D_j$ concentrate around zero. Conditioning on the history, $D_1 + D_2 + D_3$ follows a Gaussian distribution as
\begin{equation*}
    D_1 + D_2 + D_3 \, | \, \mathcal{H}_{1, T} \, \sim \, N \left( 0, \, \sigma_{\omega}^2 \sum_{i = 1}^s \bigg[ \big[ (1 + 2 \lambda) R_{1, i} - \lambda - \overline{R}_{1, s}\big]^2 + \big[ \lambda (1 + 2 \lambda) - \lambda^2 - \overline{R}_{1, s} \lambda \big]^2 \Big] + s (\lambda^2 + \lambda \overline{R}_{1, s})^2  \right).
\end{equation*}
That is the numerator of the difference between $\overline{Y}_{1, s}$ and $\overline{R}_{1, s}^*$ is a zero mean normal distribution and its variance is $O(s)$. Note that the denominator is also order $s$, $\overline{Y}_{1, s} - \overline{R}_{1, s}^*$ will behave like a zero mean distribution with $O(s^{-1})$ variance, which will converge to zero in probability. 

Therefore, in non-asymptotic sense, there is also a high probability that $\overline{Y}_{1, s} - \overline{R}_{1, s}^* \, | \, \mathcal{H}_{1, T}$ is close to zero. We define a series of good events 
\begin{equation*}
    G_{k', s} := \bigg\{- C_3 \Delta_k \leq \overline{Y}_{k', s} - \overline{R}_{k', s}^* \leq C_3 \Delta_k \bigg\},
\end{equation*}
for some $C_3 > 0$. 
\todorw{Same suggestion that we should have shorter sentence with clearer logic. For example, the sentence below spans at least 13 lines. }
The probability of $G^c_{1, s}$ conditioned on the history $\mathcal{H}_{1, T}$ is 
\begin{equation*}
    \begin{aligned}
        \mathbb{P} \big(G_{1, s}^{c}\, | \, \mathcal{H}_{1, T}\big) & = 2 \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ (D_1 + D_2 + D_3) - C_3 (1 + 2 \lambda) \Delta_k \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] > 0 \Bigg\} \\
        & \leq 2 \exp \bigg\{ -\frac{C_3^2 (1 + 2 \lambda)^4 s^{2}}{2 \sigma_{\omega}^2 \left[2 s (\lambda^2 + 1) + s \lambda^2 + s \lambda^2 (\lambda + 1)^2 + s ( C_3^2 \Delta_k^2 + 2 C_3 \Delta_k ) (1 + 2 \lambda)^2 (1 + 2 \lambda^2) \right]} \bigg\} \\
        & \leq 2 \exp \bigg\{ - s \cdot \frac{C_3^2(1 + 2 \lambda)^4}{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + ( C_3^2 \Delta_k^2 + 2 C_3 \Delta_k ) (1 + 2 \lambda)^2 (1 + 2 \lambda^2)\right]}\bigg\}.
    \end{aligned}
\end{equation*}
Here the first inequality we use the fact that 
\begin{equation*}
    \begin{aligned}
        & (D_1 + D_2 + D_3) - C_3 (1 + 2 \lambda) \Delta_k \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] \, \big| \, \mathcal{H}_{1, T}\\
        & =  \sum_{i = 1}^s \big[ (1 + 2 \lambda) R_{1, i} - \lambda - \overline{R}_{1, s} - C_3 (1 + 2 \lambda) \Delta_k \big] \omega_i + \sum_{i = 1}^s \big[ \lambda (1 + 2 \lambda) - \lambda^2 - \overline{R}_{1, s} \lambda - \lambda C_3 (1 + 2 \lambda) \Delta_k \big] \omega_i'  \\
        & ~~~~~~~~~~~~~~~~~~ + \sum_{i = 1}^s \big[ - (\lambda^2 + \lambda \overline{R}_{1, s}) - \lambda C_3 (1 + 2\lambda) \Delta_k \big] \omega_i'' \\
        & \sim N \Bigg( - C_3 (1 + 2 \lambda)^2 s,  \, \sigma_{\omega}^2 \sum_{i = 1}^s \bigg[ \big[ (1 + 2 \lambda) R_{1, i} - \lambda - \overline{R}_{1, s}\big]^2 + \big[ \lambda (1 + 2 \lambda) - \lambda^2 - \overline{R}_{1, s} \lambda \big]^2 \Big] \\
        & \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad + s \sigma_{\omega}^2 (\lambda^2 + \lambda \overline{R}_{1, s})^2 + s  \sigma_{\omega}^2 C_3^2 \Delta_k^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2)\\
        & \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad \qquad + 2 s \lambda (\lambda - 1) (1 + 2 \lambda) C_3 \Delta_k \big[ 2 \overline{R}_{1, s} - 1\big]  \Bigg) \\
        & =: N \big( - C_3 (1 + 2 \lambda)^2 s, \, \sigma_{\omega}^2 \Sigma_{G_{1, s}} \big)
    \end{aligned}
\end{equation*}
\todorw{I do not recommend using $:=$ to define stuffs.}
where we use the fact that the cross term is
\begin{equation*}
    \begin{aligned}
        & - \sum_{i = 1}^s \big[ (1 + 2 \lambda) R_{1, i} - \lambda - \overline{R}_{1, s}\big] C_3 (1 + 2 \lambda) \Delta_k - \sum_{i = 1}^s\big[ \lambda (1 + 2 \lambda) - \lambda^2 - \overline{R}_{1, s} \lambda \big] \lambda C_3 (1 + 2 \lambda) \Delta_k \\
        & ~~~~~~~~~~~~ + \sum_{i = 1}^s (\lambda^2 + \lambda \overline{R}_{1, s}) \lambda C_3 (1 + 2\lambda) \Delta_k \\
        & = \lambda C_3 (1 + 2 \lambda) \Delta_k \sum_{i = 1}^s \Big( \big[ \lambda^2 + \lambda \overline{R}_{1, s}\big] - \big[ \lambda (1 + \lambda) - \lambda \overline{R}_{1, s}\big] - \big[ 2 R_{1, i} - 1 \big]\Big) \\
        & = s \lambda (\lambda - 1) (1 + 2 \lambda) C_3 \Delta_k \big[ 2 \overline{R}_{1, s} - 1\big].
    \end{aligned}
\end{equation*}
Indeed, $\Sigma_{G_{1, s}} := \Sigma_{G_{1, s}, 1} + \Sigma_{G_{1, s}, 2} + \Sigma_{G_{1, s}, 3} + \Sigma_{G_{1, s}, 4} + \Sigma_{G_{1, s}, 5}$ can be furthermore bounded by four positive terms
\begin{equation*}
    \Sigma_{G_{1, s}, 1} := \sum_{i = 1}^s  \big[ (1 + 2 \lambda) R_{1, i} - \lambda - \overline{R}_{1, s}\big]^2 \leq 2 \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s})^2 + 2 \lambda^2 \sum_{i = 1}^s (2 R_{1, i} - 1)^2 \leq 2 s (\lambda^2 + 1),
\end{equation*}
\begin{equation*}
    \Sigma_{G_{1, s}, 2} := \sum_{i = 1}^s  \big[ \lambda (1 + 2 \lambda) - \lambda^2 - \overline{R}_{1, s} \lambda \big]^2 = \sum_{i = 1}^s \lambda^2 (1 - \overline{R}_{1, s})^2 = s\lambda^2 (1 - \overline{R}_{1, s})^2 \leq s \lambda^2,
\end{equation*}
\begin{equation*}
    \Sigma_{G_{1, s}, 3} := s (\lambda^2 + \lambda \overline{R}_{1, s})^2 \leq s \lambda^2 (\lambda + 1)^2, \qquad \Sigma_{G_{1, s}, 4} := s C_3^2 \Delta_k^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2)
\end{equation*}
and a potential negative term
\begin{equation*}
    \Sigma_{G_{1, s}, 5} := 2 s \lambda (\lambda - 1) (1 + 2 \lambda) C_3 \Delta_k \big[ 2 \overline{R}_{1, s} - 1\big] \in \big[ - 2 s |\lambda^2 - \lambda| (1 + 2 \lambda) C_3 \Delta_k, \, 2s |\lambda^2 - \lambda| (1 + 2 \lambda) C_3 \Delta_k \big].
\end{equation*}
\todohw{Adding the cross-term will case the extra term $\Delta_k$ to the denominator.}

by the fact $R_{k, i} \in [0, 1]$, which implies
\begin{equation*}
    \begin{aligned}
        0 < \Sigma_{G_{1, s}} & \leq 2 s (\lambda^2 + 1) + s \lambda^2 + s \lambda^2 (\lambda + 1)^2 + s C_3^2 \Delta_k^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2) +  2s |\lambda^2 - \lambda| (1 + 2 \lambda) C_3 \Delta_k \\
        & \leq 2 s (\lambda^2 + 1) + s \lambda^2 + s \lambda^2 (\lambda + 1)^2 + s  (1 + 2 \lambda)^2 (1 + 2 \lambda^2) +  s (1 + 2 \lambda)^2 (1 + 2 \lambda^2) \big( C_3^2 \Delta_k^2 + 2 C_3 \Delta_k \big).
    \end{aligned}
\end{equation*}
as $|\lambda^2 - \lambda|  \leq 1 + 2 \lambda^2$.

\todorw{Another general comment. I find a lot of `that` are missing. We can omit it in limited cases. Even in those cases, as your sentences are typically long, omitting  the conjunctions will make the sentence hard to read. 
I suggest to keep all of them. }
Thus, if we take 
\begin{equation*}
    \begin{aligned}
        s = s_{a, 1, 2} & := \bigg[ \frac{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + ( C_3^2 \Delta_k^2 + 2 C_3 \Delta_k ) (1 + 2 \lambda)^2 (1 + 2 \lambda^2)\right]}{C_3^2(1 + 2 \lambda)^4} \bigg] \cdot \big[ \log 2 + 2 \log T\big] \\
        & \leq \frac{6 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + ( C_3^2 \Delta_k^2 + 2 C_3 \Delta_k ) (1 + 2 \lambda)^2 (1 + 2 \lambda^2)\right]}{C_3^2(1 + 2 \lambda)^4} \log T \\
        & =: H_1 (\lambda, \sigma_{\omega}^2, C_3, \Delta_k) \log T
    \end{aligned}
\end{equation*}
for any $T \geq 2$, then 
\begin{equation*}
    a_{k, 1, 2} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) 1(G_{1, s}^c)\Big] \leq T \mathbb{P} (G_{1, s}^c) \leq T^{-1}.
\end{equation*}

   

\todorw{Same comment as for the last section. Always make sure the logic is complete. The goal is to bound $s_{a,1,2}$, the conclude with it by stating the result clearly. Do not leave the step to readers. }



\todorw{We need to understand why we only need the difference to be bounded. }

\begin{remark}
    The \red{boundedness} assumption is again not essential here, since we only require
    \todorw{State, explicitly, where above we need this inequality. (It is very difficult to find why we `only require` since it is hard to find the corresponding part from derivations above.)}
    \begin{equation*}
        \begin{aligned}
            & \log {\mathbb{E}} \exp \left\{ - \frac{C_3 (1 + 2\lambda)^2 s}{\sigma_{\omega}^2 \Sigma_{G_{1, s}}}\right\} \\
            = & \log {\mathbb{E}} \exp \left\{ - \frac{\operatorname{const} \cdot s}{\operatorname{const} + \sum_{i = 1}^s (R_{k, i} - \overline{R}_{k, s})^2 + \sum_{i = 1}^s (\text{upper bound of } \mu_k - R_{k, i})^2 + \sum_{i = 1}^s (R_{k, i} - \text{lower bound of } \mu_k )^2}\right\} \lesssim - s
        \end{aligned}
    \end{equation*}
    to ensure that take $s_{a, 1, 2}$ as order $\log T$ to achieve $a_{k, 1, 2} \leq T^{-1}$. A normal distribution $R_{k, s}$ will lead $\sum_{i = 1}^s (R_{k, i} - \overline{R}_{k, s})^2$ and $\sum_{i = 1}^s R_{k, i}^2$ are (non-centerized) chi-square distributed. They can achieve $\mathbb{P} (G_{1, s}) \leq T^{-1}$ with $s$ at most $\log T$.
\end{remark}




\paragraph{Dealing with $s_{a, 1, 1}$}

Now, what remains to do is to bound $s_{a, 1, 1} = \max \{ s: a_{k, 1, 1} \geq T^{-1}\}$ with order $
\log T$ where
\begin{equation*}
    a_{k, 1, 1} = a_{k, 1} - a_{k, 1, 2} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) 1(G_{1, s})\Big].
\end{equation*}

It maybe a little bit involved to tackle with $a_{k, 1, 1}$ directly as we said previously, $a_{k, 1, 1}$ defined on the good events, it will behave largely like ${\mathbb{E}} \big[ N_{1, s} (\tau_k) \wedge T \big]$. 
\todorw{The above is not a correct sentence. Same for the above. Again, if it is complex, one suggestion is to always make the sentence shorter, and then make sure the logic between sentences is clear. }
Hence, we cannot use something like $a_{k, 1, 1} \leq T \mathbb{P} (\text{Event}_s) \leq T \cdot T^{-2} = T^{-2} $, other technique need to introduced: transform the expectation to probability that connected with the good events $A_{1, s}$ and $G_{1, s}$.

Note that, we have
\begin{equation*}
    \begin{aligned}
        \{N_{1, s}(\tau_{k})<T^{-1}\} & \supseteq \{N_{1, s}(\tau_{k}) < (T^{2}-1)^{-1}\} = \big\{Q_{1, s}\left(\tau_{k}\right)^{-1}<1+\left(T^{2}-1\right)^{-1}\big\} \\
        &=\big\{Q_{1, s}(\tau_{k})^{-1} < (1-T^{-2})^{-1}\big\} = \big\{\left[1-Q_{1, s}(\tau_{k})\right]<T^{-2}\big\},
    \end{aligned}
\end{equation*}
and $a_{k, s} \leq {\mathbb{E}} \big[ N_{1, s} (\tau_k) 1(A_{1, s} \cap G_{1, s}) \big]$. As a result, it is sufficient to  find $s \geq s_{a,1,1}$ such that $\big\{[1-Q_{1,s}(\tau_{k})]<T^{-2}\big\}$ on the event $\{A_{1, s} \cap G_{1, s}\}$. 


\todorw{Everything below belongs to ONE paragraph. It makes the paper very hard to read and affect the logic. }

Note that
\begin{equation*}
    \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}) = \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Gamma_1 \, | \, \mathcal{H}_{1, T} \big)  1(A_{1, s}) 1(G_{1, s})
\end{equation*}
with $\Gamma_1 := \Gamma_1(\tau_1) = \tau_1 - \overline{R}_{1, s}^*$ serving as a measurement to balance the difference between $\overline{Y}_{1, s}$ and $\overline{R}_{1,s}^*$. It is not hard to see $\Gamma$ should be negative with a moderate value. Fortunately, on $G_{1, s}$, we have the gap between $\overline{Y}_{1, s}$ and $\overline{R}_{1, s}^*$ not to large. For any fixed $\Gamma_1$ has been chosen,
\begin{equation*}
    \begin{aligned}
        & D_1 + D_2 + D_3 - \Gamma_1 (1 + 2 \lambda) \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg]  \\
        & \qquad \sim \, N \Bigg( - \Gamma_1 (1 + 2 \lambda)^2 s, \, \sigma_{\omega}^2 \sum_{i = 1}^s \bigg[ \big[ (1 + 2 \lambda) R_{1, i} - \lambda - \overline{R}_{1, s}\big]^2 + \big[ \lambda (1 + 2 \lambda) - \lambda^2 - \overline{R}_{1, s} \lambda \big]^2 \Big] \\
        &  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + s \sigma_{\omega}^2 (\lambda^2 + \lambda \overline{R}_{1, s})^2 + s \sigma_{\omega}^2 \Gamma_1^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2)\Bigg),
    \end{aligned}
\end{equation*}
will give the inequality
\begin{equation*}
    \begin{aligned}
        & \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}) \\
        = & \, \mathbb{P}_{\omega, \omega', \omega''} \bigg( D_1 + D_2 + D_3 - \Gamma_1 (1 + 2 \lambda) \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg]  \leq 0\bigg) 1 (A_{1, s} \cap G_{1, s}) \\
        \leq & \exp \left\{ - \frac{(- \Gamma_1)^2 (1 + 2 \lambda)^4 s^2}{2 \Big[ \sigma_{\omega}^2 \big( 2 s (\lambda^2 + 1)  + s \lambda^2 \big) + s \lambda^2 (\lambda + 1)^2 + s \sigma_{\omega}^2 ( \Gamma_1^2 + 2 \Gamma_1) (1 + 2 \lambda)^2 (1 + 2 \lambda^2) \Big]}\right\} 1 (A_{1, s} \cap G_{1, s}) \\
        = & \exp \left\{ -\frac{(- \Gamma_1)^2 (1 + 2 \lambda)^4 s}{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + ( \Gamma_1^2 + 2 \Gamma_1) (1 + 2 \lambda)^2 (1 + 2 \lambda^2)  \right]}\right\} 1 (A_{1, s} \cap G_{1, s}).
    \end{aligned}
\end{equation*}
where we use the same technique as in handling $s_{a, 2, 2}$ to deal with the variance of $D_1 + D_2 + D_3 - \Gamma_1 (1 + 2 \lambda) \left[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\right]$.
\todorw{Another general suggestion. When you have a system of equations, please let the readers know how you jump from one to the next, unless it is trivial. Equations should ALWAYS be part of a readable paragraph. }

Recast that, on $A_{1, s} \, \cap \, G_{1, s}$, we have
\begin{equation*}
    A_{1, s} = \bigg\{ \overline{R}_{1, s}^* - \frac{\mu_1 + \lambda}{1 + 2\lambda} > - C_1 \Delta_k \bigg\} = \bigg\{ \Gamma_1 < C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} + \tau_1 \bigg\} 
\end{equation*}
and
\begin{equation*}
    \begin{aligned}
        G_{1, s} = \big\{ -C_3 \Delta_k \leq \overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq C_3 \Delta_k \big\} & = \big\{ \tau_1 -C_3 \Delta_k \leq \overline{Y}_{1, s} + \Gamma_1 \leq \tau_1 + C_3 \Delta_k \big\} \\
        & = \big\{ \tau_1 - C_3 \Delta_k - \overline{Y}_{1, s} \leq \Gamma_1 \leq \tau_1 + C_3 \Delta_k - \overline{Y}_{1, s}\big\} \\
        & \subseteq \big\{ \tau_1 - 2 C_3 \Delta_k - \overline{R}_{1, s}^* \leq \Gamma_1 \leq \tau_1 + 2 C_3 \Delta_k - \overline{R}_{1, s}^* \big\} \\
        & \subseteq \bigg\{ \tau_1 -  2 C_3 \Delta_k - \frac{1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 \leq \tau_1 + 2 C_3 \Delta_k - \frac{\lambda}{1 + 2 \lambda}\bigg\}
    \end{aligned}
\end{equation*}
then
\begin{equation*}
    \begin{aligned}
        A_{1, s} \, \cap \, G_{1, s} & = \bigg\{ \tau_1 -  2 C_3 \Delta_k - \frac{1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 < \tau_1 + C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} \bigg\} \\
        & \subseteq \bigg\{ \tau_1 -  2 C_3 \Delta_k - 1 \leq \Gamma_1 < C_1 \Delta_k + \tau_1 - \frac{\mu_1}{2}\bigg\} \\
        (\text{if } C_1 \geq 2 C_3 )& \subseteq \bigg\{ \tau_1 -  C_1 \Delta_k - 1 \leq \Gamma_1 < C_1 \Delta_k + \tau_1 - \frac{\mu_1}{2}\bigg\}
    \end{aligned}
\end{equation*}
and hence we derive the requirement that $C_1 \Delta_k - \mu_1 / 2 + \tau_1 < 0$, i.e. $\tau_1 < \mu_1  / 2 - C_1 \Delta_k$, which will give
\begin{equation*}
    \begin{aligned}
        & \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}) \\
        \leq & \exp \left\{ -\frac{(- \Gamma_1)^2 (1 + 2 \lambda)^4 s}{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + (- \Gamma_1)^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2)  \right]}\right\} 1 (A_{1, s} \cap G_{1, s}) \\
        (\text{by taking } \tau_k = \mu_1 / 2 - 2 C_1 \Delta_k)\leq & \exp \left\{ - \frac{C_1^2 \Delta_k^2 (1 + 2 \lambda)^4 s}{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + C_1^2 \Delta_k^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2)  \right]}\right\} 1 (A_{1, s} \cap G_{1, s})
    \end{aligned}
\end{equation*}
by 
\begin{equation*}
    C_1 \Delta_k = \frac{\mu_1 }{2} - \tau_1 - C_1 \Delta_k <  (- \Gamma_1) \leq 1 + C_1 \Delta_k  - \tau_k = 1 - \frac{\mu_1}{2} + 3 C_1 \Delta_k < 4
\end{equation*}
and the fact that the function
\begin{equation*}
    \begin{aligned}
        f_1(x) & := \frac{ x^2 (1 + 2 \lambda)^4}{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + (x^2 + 2x) (1 + 2 \lambda)^2 (1 + 2 \lambda^2)  \right]} \\
        & \geq  \frac{ x^2 (1 + 2 \lambda)^4}{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + 3 x^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2) / 2 \right]} =: f_2(x), \qquad 0 \leq x \leq 4,
    \end{aligned}
\end{equation*}
with $f_2(x)$ increasing on $[0, + \infty)$. By letting 
\begin{equation*}
    \begin{aligned}
        s = s_{a, 1, 1} :=  \frac{4 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + 3 C_1^2 \Delta_k^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2) / 2  \right]}{C_1^2 \Delta_k^2 (1 + 2 \lambda)^4} \log T =: H_2 (\lambda, \sigma_{\omega}^2, C_1, \Delta_k) \cdot \log T
    \end{aligned}
\end{equation*}
then
\begin{equation*}
    a_{k, 1, 1} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) 1(G_{1, s})\Big] \leq T^{-1}.
\end{equation*}
Combining these results, we know if we take
\begin{equation*}
    \begin{aligned}
        s \geq \max \{ s_{a, 2}, s_{a, 1, 1}, s_{a, 1, 2}\} & = \max \left\{ \frac{1}{C_1^2 (1 + 2\lambda)^2 \Delta_k^2},\,  H_1 (\lambda, \sigma_{\omega}^2, C_3, \Delta_k), \, H_2 (\lambda, \sigma_{\omega}^2, C_1, \Delta_k)  \right\} \log T \\
        & =: H' (\lambda, \sigma_{\omega}^2, C_1, C_3, \Delta_k) \cdot \log T
    \end{aligned}
\end{equation*}
then we have orcale that
\begin{equation*}
    \begin{aligned}
        a_k & \leq 3 + \left[\exp \bigg\{\frac{6(2 \lambda + 1)^2}{2\sigma_{\omega}^2 \lambda^2 ((\lambda + 1 )^2 + \lambda^2 )}\bigg\} + \big[ 1 - \Phi(\sqrt{2\pi} )\big]^{-1} \right]  H'(\lambda, \sigma_{\omega}^2, C_1, C_3, \Delta_k)  \log T \\
        & =: 3 + H(\lambda, \sigma_{\omega}^2, C_1, C_3, \Delta_k)  \log T.
    \end{aligned}
\end{equation*}
It is worthy to recall that the above inequality is obtained by taking
\begin{equation}\label{condition.2}
    \tau_k = \mu_1 / 2 - 2 C_1 \Delta_k
\end{equation}
with 
\begin{equation}\label{condition.3}
    C_1 \geq 2 C_3.
\end{equation}

   

   
   
\iffalse
{\color{blue}
If we note that $(- \Gamma_1) = \overline{R}_{1, s}^* - \tau_1$ lies in the interval $\left[ - \tau_1, \frac{1 + \lambda}{1 + 2\lambda} - \tau_1 \right]$, which will be bounded by $\frac{1 + \lambda}{1 + 2\lambda}$ if $\tau_1 \geq 0$, and in this case, put 
\begin{equation*}
     \frac{2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2)}{(1 + 2 \lambda)^2 (1 + 2 \lambda^2) } \leq \frac{1 + \lambda}{1 + 2\lambda}
\end{equation*}
we get that $\lambda^4 - 8 \lambda^2 + 3 \lambda - 1 \geq 0$. We get a possible range for $\lambda$ is $\lambda \geq 2 \sqrt{2}$. Compared with the existent results:
\begin{itemize}
    \item In \cite{kveton2019garbage}, the pseudo rewards $a \in (0, 1)$ should larger than $1 / \sqrt{2}$ due to they need to control the anti concentration of Bernoulli distribution, i.e. control $ {\mathbb{E}} \big[ \mathbb{P}^{-1} (U_{1, s} \geq (\mu_1 + s) \, | \, V_{1, s})\big]$;
    
    \item Similarly in \cite{kveton2019perturbed}, the condition $a > 2$ for tuning parameter is same as that in \cite{kveton2019garbage} to ensure the anti concentration having a valid upper bound;
    
    \item The tuning parameter of the pseudo residual $\sigma_a > 3 / 2$ in \cite{wang2020residual} is also essential by again controlling  the anti concentration;
\end{itemize}
Hence, the condition for the tuning parameter is not equivalent with the existent work (we use the condition to bound $a_{k, 1, 1}$ instead of $a_{k, s}$.). 
}
\fi




\subsection{Second Step: Bounding $b_k$}

The second step is bounding $b_k = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1}\big) + 1$. Recall in the diagram, what we only to do is bounding $s_b = \max \{ s: b_{k, s} \geq T^{-1}\}$ as order $\log T$ (this is by $b_{k, s}$ is naturally bounded by one).


We also need to decompose $b_k$ as $b_k = b_{k, 1, 1} + b_{k, 1, 2} + b_{k, 2} + 1$ with
\begin{gather*}
    b_{k, 1, 1} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-1}) 1(A'_{k, s}) 1(G_{k, s})\big], \\
    b_{k, 1, 2} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-1}) 1(A'_{k, s}) 1(G_{k, s}^c)\big], \\
    b_{k, 2} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-1}) 1(A_{k, s}^{\prime c} )\big].
\end{gather*}
with $A'_{k, s} = A_{k, s}$ using the second definition, then $\mathbb{P} (A^{\prime c}_{k, s}) $ when $s \geq \frac{1}{2}s_{a, 2}'$ with $C_1$ replaced by $C_2$. Then we must have $b_{k, 1, 2} \vee b_{k, 2} \leq T^{-2} < T^{-1}$ as long as $s \geq s'_{a, 2}  / 2 \vee s_{a,1, 2}$.
   
       
   
   
It remains to bound $b_{k, 1, 1}$. Indeed, we note that on $A’_{k, s}$, we have
\begin{equation*}
    A'_{k, s} = \bigg\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2\lambda} < C_2 \Delta_k \bigg\} = \bigg\{ \tau_k - \frac{\mu_k + \lambda}{1 + 2\lambda} - C_2 \Delta_k  < \Gamma_k \bigg\}.
\end{equation*}
Again, on the event 
\begin{equation*}
    \begin{aligned}
        G_{k, s} = \big\{ -C_3 \Delta_k \leq \overline{Y}_{k, s} - \overline{R}_{k, s}^* \leq C_3 \Delta_k \big\} & = \big\{ \tau_k -C_3 \Delta_k \leq \overline{Y}_{k, s} + \Gamma_k \leq \tau_k + C_3 \Delta_k \big\} \\
        & = \big\{ \tau_k - 2 C_3 \Delta_k - \overline{R}_{k, s}^* \leq \Gamma_k \leq \tau_k + 2 C_3 \Delta_k - \overline{R}_{k, s}^* \big\} \\
        & \subseteq \bigg\{ \tau_k - 2 C_3 \Delta_k - \frac{1 + \lambda}{1 + 2\lambda} \leq \Gamma_k \leq \tau_k + 2 C_3 \Delta_k - \frac{\lambda}{1 + 2\lambda}\bigg\}
    \end{aligned}
\end{equation*}
we can conclude that 
\begin{equation*}
    \begin{aligned}
        A'_{k, s} \, \cap \, G_{k, s} & \subseteq  \bigg\{ \tau_k - \frac{\mu_k + \lambda}{1 + 2\lambda} - C_2 \Delta_k  < \Gamma_k \leq\tau_k + C_2 \Delta_k - \frac{\lambda}{1 + 2\lambda} \bigg\} \\
        & \subseteq \bigg\{ \tau_k  - C_2 \Delta_k - \frac{\mu_k}{2}  < \Gamma_k < \tau_k + C_2 \Delta_k \bigg\}
    \end{aligned}
\end{equation*}
if
\begin{equation}\label{condition.4}
    2 C_3 \geq C_2
\end{equation}
Under this establishment, we need $\tau_k - C_2 \Delta_k - \frac{\mu_k }{2} \leq 0$, then we can take
\begin{equation}\label{condition.5}
    \tau_k = 2 C_2 \Delta_k + \mu_k / 2
\end{equation}
and we will obtain that
\begin{equation*}
    C_2 \Delta_k = \tau_k  - C_2 \Delta_k - \frac{\mu_k}{2} < \Gamma_k < \tau_k + C_2 \Delta_k = 3 C_2 \Delta_k + \frac{\mu_k}{2} \leq \frac{7}{2} < 4,
\end{equation*}
then we will have
\begin{equation*}
    \begin{aligned}
         Q_{k, s} (\tau_k) 1(A'_{k, s}) 1(G_{k, s}) & = \mathbb{P} \big(\overline{Y}_{k, s} - \overline{R}_{k, s}^* > \Gamma_k \, | \, \mathcal{H}_{k, T} \big) 1(A'_{k, s}) 1(G_{k, s}) \\
         & = \mathbb{P}_{\omega, \omega^{\prime}, \omega^{\prime \prime}} \Bigg\{ (D_1 + D_2 + D_3) - \Gamma_k (1 + 2 \lambda) \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i^{\prime} + \lambda  \sum_{i = 1}^s \omega_i^{\prime \prime} \bigg] > 0 \Bigg\} 1(A'_{k, s}\cap G_{k, s}) \\
         & \leq 2 \exp \bigg\{ - \frac{\Gamma_k^{2} (1 + 2 \lambda)^4s }{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + (\Gamma_k^2 + 2 \Gamma_k) (1 + 2 \lambda)^2 (1 + 2 \lambda^2)\right]}\bigg\} 1(A'_{k, s}\cap G_{k, s}) \\
         & \leq 2 \exp \bigg\{ - \frac{\Gamma_k^{2} (1 + 2 \lambda)^4s }{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + 3\Gamma_k^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2) / 2\right]}\bigg\} 1(A'_{k, s}\cap G_{k, s}) \\
         & \leq 2 \exp \bigg\{ - \frac{C_2^2 \Delta_k^2 (1 + 2 \lambda)^4s }{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + 3 C_2^2 \Delta_k^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2) /  2\right]}\bigg\} \leq T^{-1}
    \end{aligned}
\end{equation*}
by letting 
\begin{equation*}
    s = s_{a, 1, 1}^{\prime} = \frac{2 \sigma_{\omega}^2 \left[ 2 + 3 \lambda^2 (4 + 2 \lambda + \lambda^2) + 3 C_2^2 \Delta_k^2 (1 + 2 \lambda)^2 (1 + 2 \lambda^2) / 2 \right]}{C_2^2 \Delta_k^2 (1 + 2 \lambda)^4} \log T =: G_1 (\lambda, \sigma_{\omega}^2, C_2, \Delta_k) \cdot \log T.
\end{equation*}

As a consequence, as long as we have
\begin{equation*}
    \begin{aligned}
        s \geq \max\{ s_{a, 2}^{\prime} / 2, s_{a, 1, 2},  s_{a, 1, 1}^{\prime}\} & = \max \left\{ \frac{1}{2 C_2^2 (1 + 2\lambda)^2 \Delta_k^2},\,  H_1 (\lambda, \sigma_{\omega}^2, C_3, \Delta_k), \, G_1 (\lambda, \sigma_{\omega}^2, C_2, \Delta_k)  \right\} \log T \\
        & =: G(\lambda, \sigma_{\omega}^2, C_2, C_3, \Delta_k) \cdot \log T,
    \end{aligned}
\end{equation*}
then the inequality holds for 
\begin{equation*}
    b_k \leq 4 + G(\lambda, \sigma_{\omega}^2, C_2, C_3, \Delta_k) \cdot \log T.
\end{equation*}

Finally, we combine the restrictions (\ref{condition.1}), (\ref{condition.2}), (\ref{condition.3}), (\ref{condition.4}), and (\ref{condition.5}), then we get that $C_1, C_2, C_3 > 0$ should satisfy the following system of equations
\begin{equation*}
    \left\{ \begin{array}{l}
         \frac{\mu_1}{2} - 2 C_1 \Delta_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \\
         2 C_2 \Delta_k + \frac{\mu_k}{2} \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \\
        \tau_k - \frac{\mu_1}{2} - C_1 \Delta_k  \leq \tau_k + C_2\Delta_k - \frac{\mu_k }{2} \\
        C_1 \geq 2C_3 \\
        2 C_3 \geq C_2 \\
        \frac{\mu_1}{2} - 2 C_1 \Delta_k = 2 C_2 \Delta_k + \frac{\mu_k}{2}
    \end{array}\right.
\end{equation*}
thus, we only have
\begin{equation*}
    C_1 = C_2 = \frac{1}{8}, \qquad C_3 = \frac{1}{16},
\end{equation*}
is the unique solution for the above equations.
\todorw{The following is incomplete. Only the one who writes the proof knows why it is done. Either remind the readers of the decomposition, or, combining all result so far until we really reach the final conclusion. Do not skip. }
\todohw{I have added some details, maybe more details can be added if others still cannot understand.}
up to now, we get Theorem \ref{thm.MAB}.






\newpage

\section{Discussions}

\subsection{Some discussions on the proof}

\subsubsection{Discussion on Hyper-parameters:}
\begin{itemize}
    \item In \cite{kveton2019garbage}, the pseudo rewards $a \in (0, 1)$ should larger than $1 / \sqrt{2}$ due to they need to control the anti concentration of Bernoulli distribution, i.e. control $ {\mathbb{E}} \big[ \mathbb{P}^{-1} (U_{1, s} \geq (\mu_1 + s) \, | \, V_{1, s})\big]$;
    
    \item Similarly in \cite{kveton2019perturbed}, the condition $a > 2$ for tuning parameter is same as that in \cite{kveton2019garbage} to ensure the anti concentration having a valid upper bound;
    
    \item The tuning parameter of the pseudo residual $\sigma_a > 3 / 2$ in \cite{wang2020residual} is also essential by again controlling  the anti concentration;
\end{itemize}

which is the exact the place we ask our parameter $\lambda > 0$.

\subsubsection{Why we still use $G_{k, s}$}

As we can see in the proof, only combine $G_{k, s}$ with $A_{k, s}$ could we give $\Gamma_k$ bounded in some closed finite interval. It is not enough to only use $A_{k, s}$. The only reason we can drop $G_{k, s}$ is that when $R_{k, s}$ is bounded, $f(x)$ we mentioned before is increasing.

\section{Proof of Corollary \ref{cor.MAB.main}}

The proof of transforming problem-dependent to problem-independent problem is just based on Cauchy inequality. Indeed, we note that
\begin{equation*}
    H(\lambda, \sigma_{\omega}^2, \Delta_k) \lesssim \frac{1}{\Delta_k^2} + \Delta_k^2 + \Delta_k 
\end{equation*}
and similarly,
\begin{equation*}
    G(\lambda, \sigma_{\omega}^2, \Delta_k) \lesssim \frac{1}{\Delta_k^2} + \Delta_k^2 + \Delta_k 
\end{equation*}
for any $\lambda$ and $\sigma_{\omega}^2$. Hence, consider any $\Delta>0$,
\begin{equation*}
    \begin{aligned}
        \operatorname{Reg}_T & \leq \sum_{\Delta_k : \Delta_k < \Delta} \Delta_k {\mathbb{E}} J_{k, T} + \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k {\mathbb{E}} J_{k, T} \\
        & \lesssim T \Delta + \sum_{k = 2}^K \Delta_k + \sum_{k = 2}^K \Delta_k \bigg[ \frac{1}{\Delta_k^2} + \Delta_k^2 + \Delta_k \bigg] \cdot \log T \\
        & \lesssim T \Delta + \sum_{k = 2}^K \Delta_k +  \sum_{k = 2}^K \frac{\log T}{\Delta_k} + K \log T \\
        & \lesssim  T \Delta  +  K \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k  +  \sum_{\Delta_k : \Delta_k \geq \Delta} \frac{\log T}{\Delta_k} + K \log T \\
        & \lesssim T \Delta + \frac{ K \log T}{\Delta} +  K \mu_1 + K \log T \lesssim \sqrt{K T \log T} + K \mu_1 + K \log T 
    \end{aligned}
\end{equation*}
by Cauchy inequality.


\section{Proof of Corollary \ref{cor.MAB}}

For the first result only need to note that
\begin{equation*}
    \operatorname{Reg}_T \leq 7 \sum_{k = 2}^K \Delta_k + \Big( 164.084 \times 774 + 1152 \Big) \sum_{k = 2}^K \frac{\log T}{\Delta_k} =  7 \sum_{k = 2}^K \Delta_k + 128153 \sum_{k = 2}^K \frac{\log T}{\Delta_k}
\end{equation*}
as $\lambda \rightarrow \infty$. For the second result, similarly as proof in the previously section, consider any $\Delta > 0$
\begin{equation*}
    \begin{aligned}
        \operatorname{Reg}_T & = \sum_{\Delta_k : \Delta_k < \Delta} \Delta_k {\mathbb{E}} J_{k, T} + \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k {\mathbb{E}} J_{k, T} \\
        & \leq T \Delta + 7 \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k  +  128153 \sum_{\Delta_k : \Delta_k \geq \Delta} \frac{\log T}{\Delta_k} \\
        & \leq T \Delta + \frac{ 128153 K \log T}{\Delta} + 7 K \mu_1 \leq 948 \sqrt{K T \log T} + 7 K \mu_1.
    \end{aligned}
\end{equation*}
by Cauchy inequality.


\section{Proof of Theorem \ref{thm.MAB}}\label{proof_thm}


Before starting our proof, we define the sub-exponential variable that mimics the tail of the exponential random variable. To {be} specific, we call a \textit{zero-mean} variable $X$ sub-exponential with parameter $\lambda$ and $\alpha$ if
\[
    {\mathbb{E}} \exp \{ t X \} \leq \exp \{ t^2 \lambda^2 / 2\}, \qquad |t| \leq \frac{1}{\alpha} \red{,}
\]
and we denote $X \sim \operatorname{subE}(\lambda, \alpha)$. 
Similar to the sub-Gaussian variables, we have the following concentration {property}
\[
    \mathbb{P} (X \geq t) \leq \exp \left\{ - \frac{1}{2} \left( \frac{t^2}{\lambda^2} \wedge \frac{t}{\alpha}\right) \right\}, \qquad  {\forall} \, t \geq 0.
\]

   

Recall the quantity of what we care about is the cumulative regret 
\[
    \operatorname{Reg}_{T} := {\mathbb{E}} \operatorname{Reg}_{T}(Y,\mathcal{A}) = {\mathbb{E}} \sum_{t=1}^{T} [R_1 - R_{A_{t}}]=\sum_{k=2}^{K} \Delta_{k} {\mathbb{E}} \Big[\sum_{t=1}^{T} 1 \left\{A_{t}=k\right\}\Big],
\]
where we assume $\mu_1 = \max_{k \in [K]} \mu_k$. In other words, the first arm
is the best arm with the {highest} expected reward. We want to show $\operatorname{Reg}_T$ can be bounded by $C \cdot \log T$ for any $T \in \mathbb{N}$ under our algorithm, {specially} under Algorithm 3. Here $C$ is free of the round $T$. 



\subsection{Basic Idea}

Our proof relies on the following decomposition of the cumulative regret from \cite{kveton2019garbage}. 
We recap its proof in Appendix \ref{sec:proof_lemma} for completeness.


\begin{lemma}\label{lem.a.b}
Recall that we select arms according to the rule 
$A_t = \arg \max_{k \in [K]} \overline{Y}_{k, t}$. 
For any $\{ \tau_k \}_{k = 2}^K \subseteq \mathbb{R}^{K - 1}$, 
we have the bound
\[
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
\]
with
\[
    a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] =: \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \sum_{s = 0}^{T - 1} \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1 =: \sum_{s = 0}^{T - 1} b_{k, s} + 1
\]
where $Q_{k, s} (\tau) = \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_{k, s} \big)$ and $N_{k, s} (\tau) = 1 / Q_{k, s} (\tau) - 1$ with $\mathcal{H}_{k, s}$ is the history $s$ rewards of $k$-th arm.
\end{lemma}
Here the summation index $s$ is the time we pull the $k$-th arm in the total $T$ round{s}. 
From the definition of $a_{k, s}$ and $b_{k, s}$, $a_k$ represents that the tail probability of pulling the optimal arm is not too small, while $b_k$ expresses that the tail probabilities of rewards of all arms are not too large in general. 
It is worthy to note that for here we only need to consider the lower bound of the tail of the distribution of the rewards from optimal arm. 
The intuition is that: (i) we only need the probability of the rewards of the optimal arm taking a relatively large value is not too small; (ii) we do not care about how small the probability of a large value occurred in the other arms. 


Hence, in the following steps, we will bound $a_k$ and $b_k$ separately up to order $\log T$. Recall that we denote two functions $a(s, T) \lesssim b(s, T)$ if $a(s, T) \leq c \times b(s, T)$ for $c$ free of $s$ and $T$. 
And we write $a(s, T) \asymp b(s, T)$ if $a(s, T) \lesssim b(s, T)$ and $a(s, T) \gtrsim b(s, T)$. 
The proof of all lemmas can be found in Appendix \ref{sec:proof_lemma}. 



\todorw{Can we have one paragraph (for each) to summarize how we plan to bound a and b below? 
If we can summarize it concisely using plain English, it means we really understand the logic. 
}

\subsection{Bounding $a_k$}
Recall that our target is bounding $a_k$ and $b_k$ up to order $
\log T$, it is enough to show that {this can hold}
for 
{any large enough} $T$. For doing this, we first prove a weaker statement: $a_{k, s}$ as a function of $T$ can be bounded by some constants free of $T$, 
which will give the weaker bound $a_{k} \lesssim T$. 
Note that if we take
\begin{equation}\label{requirement_0}
    \tau_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda},
\end{equation}
then we will have
\begin{equation}\label{constant_bound}
    a_{k, s} = {\mathbb{E}} \left[ \left( \frac{1}{Q_{1, s}(\tau_k)}  - 1\right) \wedge T \right] \leq {\mathbb{E}} \frac{1}{Q_{1, s}(\tau_k)} \leq {\mathbb{E}} Q_{1, s}^{-1} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right),
\end{equation}
where in the second inequality we use the fact that $Q_{1, s}(\cdot)$ is a decreasing function. Hence, we need to find a lower bound
the tail probability $Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right) = \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right)$. The tail probability can be written as the weighted summation of Gaussian bootstrap weights and hence can use the Gaussian inequality as follows
\[
    \begin{aligned}
        & \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right) \\
        = & \mathbb{P} \left( {\red{\frac{\sum_{i = 1}^s \omega_i R_{1, i} + s \lambda}{\sum_{i = 1}^s \omega_i + 2 s \lambda}}} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s} \right) \\
        = & \mathbb{P}_{\omega} \left( \sum_{i = 1}^s \big[  (1 + 2 \lambda) R_i - (\mu_1 + \lambda) \big] \omega_i > s \lambda (2 \mu_1 - 1) \right) \\
        \geq & \exp \left\{ - \frac{3 s^2 (1 + 2 \lambda)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \frac{1}{s} s^2 \left[  (1 + 2 \lambda) \overline{R}_{1, s} - (\mu_1 + \lambda) \right]^2 }\right\} \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  + \big[ 1 - \Phi(\sqrt{2 \pi})\big] \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \\
        \geq & \exp \left\{ - \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left[  \overline{R}_{1, s} - (\mu_1 + \lambda) / (1 + 2 \lambda) \right]^2 }\right\} \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  + \big[ 1 - \Phi(\sqrt{2 \pi})\big] \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \\
        \geq & \exp \left\{ - \frac{3 s (1 + 2 \lambda)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  + \big[ 1 - \Phi(\sqrt{2 \pi})\big] \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} 
    \end{aligned}
\]
where the $\geq$ is by the following lemma, which states the lower bound of the tail probability of Gaussian variables.

\begin{lemma}\label{lem:gau_low}
    Suppose $\{x_i\}_{i = 1}^n \cup y \in \mathbb{R}$ and $\{ \omega_i\}_{i = 1}^n$ are independent $N(1, \sigma_{\omega}^2)$ random variables, then $\mathbb{P} \left( \sum_{i = 1}^n x_i \omega_i > y \right) \geq \exp \left\{ -\frac{3 \left(\sum_{i = 1}^n x_i - y \right)^2}{2 \sigma_{\omega}^2 \frac{1}{n} \left(\sum_{i = 1}^n x_i \right)^2 } \right\} 1 \left(\sum_{i = 1}^n x_i - y > \sqrt{2 \pi} \right) + \big[ 1 - \Phi(\sqrt{2 \pi})\big]  1 \left(0 < \sum_{i = 1}^n x_i - y \leq \sqrt{2 \pi} \right)$.
\end{lemma}
   
   
       
           
           
      
   
And as $a_{k,s}$ is bounded by the reciprocal of the lower bound of the tail probability in \eqref{constant_bound}, we obtain that 
\[
    \begin{aligned}
        a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (1 + 2 \lambda)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}   + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} 
       
    \end{aligned}
\]
Notice that ${R}_{1, i} - \mu_1$ for $i \in [s]$ is sub-Gaussian. 
For dealing with the expectation, we need the following lemma.
\begin{lemma}\label{lem_subGtosubE}
    Suppose $\{X_i\}_{i = 1}^n$ are i.i.d.  sub-Gaussian with mean $\mu$, then 
    \[
        {\mathbb{E}} \exp\left\{ \lambda(\overline{X} - \mu)^2 \right\} \leq \exp \left\{ \frac{64 \lambda^2 \sigma^4}{n^2} + \frac{\lambda \sigma^2}{n}\right\} \leq e^{9 / 8}
    \]
    for any $|\lambda| \leq \frac{n}{8 \sigma^2}$.
\end{lemma}
Hence, if we take
\begin{equation}
\label{cond.1}
    \frac{3 s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2} \leq \frac{s}{8 \sigma_1^2},
\end{equation}
we will obtain $a_{k, s}$ is bounded by 
\begin{equation}\label{a_k_constant}
    a_{k, s} \leq e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}.
\end{equation}
\begin{remark}
   Lemma \ref{lem_subGtosubE} actually is a special case for the fact that the mean of iid sub-Weibull sample with sample size $n$ will have the local sub-Gaussian property in the interval up to $\sqrt{n}$.
\end{remark}
Take consideration of \eqref{cond.1}, for bound $a_{k, s}$ with a constant, we write the inequality with regarding of 
\begin{equation}\label{tuning_condition}
    \big[ (2 \mu_1 - 1) \sigma_{\omega} - 4 \sigma_1 \big] \lambda \geq 2 \sigma_1
\end{equation}
One can easily see that a immediately necessary condition for \eqref{tuning_condition} hold is $\mu_1 \geq \frac{1}{2}$. And solve it, and we can get regularity condition for tuning parameter $(\lambda, \sigma_{\omega}^2)$ is  
\begin{equation}\label{tuning_condition_exact}
    \lambda \geq \frac{2 \sigma_1}{\big[ (2 \mu_1 - 1) \sigma_{\omega} - 4 \sigma_1 \big]} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sigma_1}{2 \mu_1 - 1}.
\end{equation}
This step derives the whole restriction of tuning parameter in the Theorem. The exactly same situation appears in other literature: the condition of tuning parameter is derived in the step of bounding $a_{k, s}$ with a constant free of $T$, see \cite{kveton2019garbage} and \cite{wu2022residual} for instance.






  

\begin{remark}
   It can be shown that for bounded rewards, we can use any positive $\lambda$ and $\sigma_{\omega}^2$ to ensure that $a_{k, s}$ can be bounded by a constant free of $T$. 
\end{remark}

Next, we decompose $a_{k, s}$ as positive several simple expressions $a_{k, s, 1}$, $a_{k, s, 2}$, and $a_{k, s, 3}$ i.e. $a_{k, s} = a_{k, s, 1} + a_{k, s, 2} + a_{k, s, 3}$, and we can focus on these simpler components. Following the idea in \cite{kveton2019garbage}, we will try to bound 
\[
    s_{a, j} = \max \{ s: a_{k, s, j} \geq T^{-1}\}, \qquad j = 1, 2, 3
\]
with an upper bound having order $\log T$. Indeed, if we successfully do so, then 
\[
    a_{k, s, j} \leq a_{k, s} \times s_{a, j} + T^{-1} \times (T - s_{a, j}) \lesssim \log T
\]
giving an order $\log T$. 

As we can see, decomposition will play an important role in this proof. For decomposition, a common approach is using indicators on good events. We define the following good events for $k$-th arm
\[
    A_{k, s} = \left\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2 \lambda}  > - C_1 \Delta_k \right\}, \qquad A_{k, s}^{\prime} = \left\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2 \lambda}  < C_2 \Delta_k \right\}, 
\]
and
\[
    G_{k', s} := \left\{ - C_3 \Delta_k \leq \overline{Y}_{k', s} - \overline{R}_{k', s}^* \leq C_3 \Delta_k \right\}
\]
with the shifted version mean defined as 
\[
    \overline{R}_{k, s}^* = \frac{\sum_{i = 1}^s R_{k, i} + \lambda s}{s(1 + 2 \lambda)} = \frac{\lambda}{1 + 2 \lambda} + \frac{1}{1 + 2 \lambda} \overline{R}_{k, s}
\]
It is worthy to note that $k'$ in $G_{k', s}$ is different from the fixed index $k$ in $\Delta_k$. And it is also easy to see that under two good events $A_{k', s} \cap G_{k', s}$, our target function $\overline{Y}_{k', s}$ is close to the shifted mean $\frac{\mu_{k'} + \lambda}{1 + 2\lambda}$, which means $\overline{Y}_{k', s}$ does remain the order of expected mean of each arm. 
Therefore, we decompose $a_{k, s}$ as the following three parts
\[
    a_{k, s, 1} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1(A_{1, s}^c)\Big],
\]
\[
    a_{k, s, 2} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1(A_{1, s}) 1 (G_{1, s}^c)\Big], \qquad a_{k, s, 3} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1(A_{1, s}) 1 (G_{1, s})\Big].
\]
As we can see, $a_{k, s, 1}$ and $a_{k, s, 2}$ defined on "bad" events and hence we will bound the tail probability $\mathbb{P} (A_{1, s}^c)$ and $\mathbb{P} (G_{1, s}^c)$, while $a_{k, s, 3}$ defined on good events will need extra steps to bound (we will specify it later). 

\subsubsection{Tackling $a_{k, s, 1}$ with $s_{a, 1}$}

Since the summation of independent sub-Gaussian variable is still sub-Gaussian (see Lemma \ref{lem_subGtosubE}), we can find that $\overline{R}_{1, s} - \mu_1 \sim \operatorname{subG} (\sigma^2 /  s)$. And by the inequality of sub-Gaussian variable,
\[
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \left( \mu_1 - \overline{R}_{1, s} \geq (1 + 2\lambda) C_1 \Delta_k \right) \leq \exp \left\{ - \frac{(1 + 2\lambda)^2 C_1^2 \Delta_k^2 s}{2 \sigma_1^2}\right\} \leq \frac{1}{T^2}
    \end{aligned}
\]
when we take
\[
    s \geq s_{a, 1} := \frac{\sigma_1^2}{C_1^2 (1 + 2 \lambda)^2 \Delta_k^2} \cdot \log T.
\]
Hence, under this setting, we obtain that
\[
    a_{k, s, 1} \leq {\mathbb{E}} \Big[ T  1(A_{1, s}^c)\Big] =  T \mathbb{P}(A_{1, s}^c) \leq T^{-1}
\]
when $s \geq s_{a, 1} \asymp \log T$.

\begin{remark}
   For bounded rewards, there we can just use Hoeffding inequality and still have $s_{a, 1} \asymp \log T$.
\end{remark}

\subsubsection{Tackling $a_{k, s, 2}$ with $s_{a, 2}$}

As $G_{k', s} = \{ |\overline{Y}_{k', s} - \overline{R}^*_{k', s}| \leq C_3 \Delta_k\}$ is a symmetric event, we have 
\[
    \begin{aligned}
        & \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s}) \\
        = & 2 \mathbb{P}_{\omega} \Bigg\{ \sum_{i = 1}^s \big[ (1 + 2\lambda) R_i - (\overline{R}_{1, s} + \lambda) - C_3 \Delta_k (1 + 2 \lambda)\big] \omega_i \geq s \lambda \big[ 2 C_3 \Delta_k (1 + 2 \lambda) + 2 \overline{R}_{1, s} - 1 \big] \Bigg\} \\
        = &  2 \mathbb{P}_{\omega} \Bigg\{  \frac{\sum_{i = 1}^s \big[ (1 + 2\lambda) R_i - (\overline{R}_{1, s} + \lambda) - C_3 \Delta_k (1 + 2 \lambda)\big] \omega_i - s \big[ \lambda(2 \overline{R}_{1, s} - 1) - C_3 \Delta_k (1 + 2 \lambda)\big] }{\sigma_{\omega}^2 \sum_{i = 1}^s \big[ (1 + 2\lambda) R_i - (\overline{R}_{1, s} + \lambda) - C_3 \Delta_k (1 + 2 \lambda)\big]^2} \\
        & \qquad \qquad \geq \frac{s \lambda \big[ 2 C_3 \Delta_k (1 + 2 \lambda) + 2 \overline{R}_{1, s} - 1 \big] - s \big[ \lambda(2 \overline{R}_{1, s} - 1) - C_3 \Delta_k (1 + 2 \lambda)\big] }{\sigma_{\omega}^2 \sum_{i = 1}^s \big[ (1 + 2\lambda) R_i - (\overline{R}_{1, s} + \lambda) - C_3 \Delta_k (1 + 2 \lambda)\big]^2}\Bigg\} \\
        \leq &  2 \mathbb{P} \left( Z \geq \frac{ C_3 \Delta_k (1 + 2 \lambda)^2 s}{\sigma_{\omega}^2 \big[ \lambda(2 \overline{R}_{1, s} - 1) - C_3 \Delta_k (1 + 2 \lambda)\big]^2} \right)
    \end{aligned}
\]
where $Z$ is the standard Gaussian variable. By applying the concentration inequality for standard Gaussian variable such that $\mathbb{P} (Z \geq x) \leq \exp \left\{ - x^2 / 2\right\}$, we get that 
\begin{equation}\label{a_k_s_2}
    \begin{aligned}
        a_{k, s, 2} & \leq {\mathbb{E}} \Big[ T  1(G_{1, s}^c)\Big] =  T {\mathbb{E}} \Big[ \mathbb{P}(G_{1, s}^c \mid \mathcal{H}_{1, s}) \Big] \\
        & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{ C_3^2 \Delta_k^2 (1 + 2 \lambda)^4 s^2 }{2 \sigma_{\omega}^2 \big[ \lambda(2 \overline{R}_{1, s} - 1) - C_3 \Delta_k (1 + 2 \lambda)\big]^4} \right\} \\
        & =  2 T {\mathbb{E}} \exp \left\{ - \frac{ C_3^2 \Delta_k^2 (1 + 2 \lambda)^4 s^4}{2 \sigma_{\omega}^2 \Big[ 2 \lambda \sqrt{s}(\overline{R}_{1, s} - \mu_1) - \sqrt{s} \big(\lambda (1 - 2 \mu_1) + C_3 \Delta_k (1 + 2 \lambda) \big) \Big]^4} \right\}.
    \end{aligned}
\end{equation}
For tackling the expectation above, we first note that $\sqrt{s} (\overline{R}_{1, s} - \mu_1) \sim \operatorname{subG} (\sigma_1^2)$, then $ 2\lambda \sqrt{s} (\overline{R}_{1, s} - \mu_1) \sim \operatorname{subG} \big(4\lambda^2 \sigma_1^2 \big)$. Therefore, by applying Corollary 4 in \cite{zhang2020concentration}, $2 \sigma_{\omega}^2 \Big[ 2 \lambda \sqrt{s}(\overline{R}_{1, s} - \mu_1) - \sqrt{s} \big(\lambda (1 - 2 \mu_1) + C_3 \Delta_k (1 + 2 \lambda) \big) \Big]^4$ can be written as $A(s) X + B(s)$ with some zero-mean random variable $X \sim \operatorname{subW}(1/2)$ and polynomial $A(\sqrt{s})$ and $B(\sqrt{s})$ with the degree at most $4$. Denote $a = A(\sqrt{s}) / s^2$ and $b = B(\sqrt{s}) / s^2$ are free of $s$, then we can apply the following lemma.
\begin{lemma}[sub-Weibull concentration]\label{lem_subE_ineq}
    Suppose zero-mean random variable $X \sim \operatorname{subW}(\theta)$, and positive constant $a, b, c$ satisfying $a X + b > 0$, then
    \[
        {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\}  \leq e^{- c / b} \big[1 + c h_1(a, b, \| X\|_{\psi_{\theta}}) + c h_2(a, b, \| X\|_{\psi_{\theta}}) \big],
    \]
    where $h_1$ and $h_2$ are finite positive functions free of $c$.
\end{lemma}
Hence, the right side of \eqref{a_k_s_2} can be furthermore bounded by 
\[
    a_{k, s, 2} \leq 2 T {\mathbb{E}} \left\{ - \frac{ C_3^2 \Delta_k^2 (1 + 2 \lambda)^4 s^2}{a X + b} \right\}.
\]
Now let us take
\[
    \begin{aligned}
        s \geq s_{a, 2} := & \frac{12 b}{C_3^2 (1 + 2\lambda)^4 \Delta_k^2} \times \log \Big[  2 C_3^2 \lambda^2 (1 + 2\lambda)^4 \Delta_k^2 \big[ 1 + C_3^2 (1 + 2\lambda)^4 \Delta_k^2 (h_1 + h_2) \big] \Big] \log T
    \end{aligned}
\]
we will have $a_{k, s, 2} \leq T^{-1}$.

\subsubsection{Tackling $a_{k, s, 3}$ with $s_{a, 3}$}

As state previously, $a_{k, s, 3}$ defined on good events, we need a special technique to deal with. Write 
\[
    \begin{aligned}
        \{N_{1, s}(\tau_{k})<T^{-1}\} & \supseteq \{N_{1, s}(\tau_{k}) < (T^{2}-1)^{-1}\} = \big\{Q_{1, s}\left(\tau_{k}\right)^{-1}<1+\left(T^{2}-1\right)^{-1}\big\} \\
        &=\big\{Q_{1, s}(\tau_{k})^{-1} < (1-T^{-2})^{-1}\big\} = \big\{\left[1-Q_{1, s}(\tau_{k})\right]<T^{-2}\big\},
    \end{aligned}
\]
and $a_{k, s, 3} \leq {\mathbb{E}} \big[ N_{1, s} (\tau_k) 1(A_{1, s} \cap G_{1, s}) \big]$. Thus, it is sufficient to find $s \geq s_{a,3}$ such that $\big\{[1-Q_{1,s}(\tau_{k})]<T^{-2}\big\}$ on the event $\{A_{1, s} \cap G_{1, s}\}$. 
Note that
\[
    \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}) = \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Gamma_1 \, | \, \mathcal{H}_{1, s} \big)  1(A_{1, s}) 1(G_{1, s})
\]
with $\Gamma_1 := \Gamma_1(\tau_1) = \tau_1 - \overline{R}_{1, s}^*$ measuring the difference between $\overline{Y}_{1, s}$ and $\overline{R}_{1,s}^*$. It is not hard to see $\Gamma_1$ should be negative with a moderate value. And $\Gamma_1$ does have an appropriate on the event $A_{1, s} \, \cap \, G_{1, s}$, this can be true. To see this, we write 
\[
    A_{1, s} = \bigg\{ \overline{R}_{1, s}^* - \frac{\mu_1 + \lambda}{1 + 2\lambda} > - C_1 \Delta_k \bigg\} = \bigg\{ \Gamma_1 < C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} + \tau_1 \bigg\} 
\]
and
\[
    \begin{aligned}
        G_{1, s} = \big\{ -C_3 \Delta_k \leq \overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq C_3 \Delta_k \big\} & = \big\{ \tau_1 -C_3 \Delta_k \leq \overline{Y}_{1, s} + \Gamma_1 \leq \tau_1 + C_3 \Delta_k \big\} \\
        & = \big\{ \tau_1 - C_3 \Delta_k - \overline{Y}_{1, s} \leq \Gamma_1 \leq \tau_1 + C_3 \Delta_k - \overline{Y}_{1, s}\big\} \\
        & \subseteq \big\{ \tau_1 - 2 C_3 \Delta_k - \overline{R}_{1, s}^* \leq \Gamma_1 \leq \tau_1 + 2 C_3 \Delta_k - \overline{R}_{1, s}^* \big\} \\
        & \subseteq \bigg\{ \tau_1 -  2 C_3 \Delta_k - \frac{1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 \leq \tau_1 + 2 C_3 \Delta_k - \frac{\lambda}{1 + 2 \lambda}\bigg\},
    \end{aligned}
\]
then
\[
    \begin{aligned}
        A_{1, s} \, \cap \, G_{1, s} & = \bigg\{ \tau_1 -  2 C_3 \Delta_k - \frac{1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 < \tau_1 + C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} \bigg\} \\
        & \subseteq \bigg\{ \tau_1 -  2 C_3 \Delta_k - 1 \leq \Gamma_1 < C_1 \Delta_k + \tau_1 - \frac{\mu_1}{2}\bigg\} \\
        (\text{if } C_1 \geq 2 C_3 )& \subseteq \bigg\{ \tau_1 -  C_1 \Delta_k - 1 \leq \Gamma_1 < C_1 \Delta_k + \tau_1 - \frac{\mu_1}{2}\bigg\}.
    \end{aligned}
\]
It is easy to see that in the above derivation, we add the requirement that
\begin{equation}\label{requirement_1}
     \Delta_k + \tau_1 - \frac{\mu_1}{2} = 0 \leq 0 \qquad \text{ and }  \qquad C_1 \geq 2C_3.
\end{equation}
Therefore, we obtain the range of the positive quantity $-\Gamma_1$ is
\[
    C_1 \Delta_k = \frac{\mu_1 }{2} - \tau_1 - C_1 \Delta_k <  (- \Gamma_1) \leq 1 + C_1 \Delta_k  - \tau_k = 1 - \frac{\mu_1}{2} + 3 C_1 \Delta_k < 4
\]
i.e. $ (- \Gamma_1) \in (C_1 \Delta_k, 4)$ when we take
\begin{equation}\label{requirement_2}
    \tau_1 = \frac{\mu_1}{2} - \Delta_k.
\end{equation}
Besides, as the tail probability we care is 
\[
    \begin{aligned}
        & \big[ 1 - Q_{1,s} (\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \\
        = & 2 \mathbb{P}_{\omega} \Bigg\{ \sum_{i = 1}^s \big[ (1 + 2\lambda) R_i - (\overline{R}_{1, s} + \lambda) - \Gamma_1 (1 + 2 \lambda)\big] \omega_i \geq s \lambda \big[ 2 \Gamma_1 (1 + 2 \lambda) + 2 \overline{R}_{1, s} - 1 \big] \Bigg\},
    \end{aligned}
\]
what we need to deal with is by replacing  $C_3 \Delta_k$ by $(-\Gamma_1)$ in the above section. By using the same technique, one can get that
\[
    \begin{aligned}
        \big[ 1 - Q_{1,s} (\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \leq T^{-2}
    \end{aligned}
\]
when
\[
   \begin{aligned}
       s \geq s_{a, 3} := & \frac{6 b^* }{C_1^2 (1 + 2\lambda)^4 \Delta_k^2 } \times \log \Big[ 1 + C_1^2 \Delta_k^2 (1 + 2\lambda)^4  (h_1^* + h_2^*)\Big] \log T,
   \end{aligned}
\]
where $b^*$, $h_1^*$, and $h_2^*$ free of $s$ are the parameters in Lemma \ref{lem_subE_ineq}.

Now combine all results in Section C.2, we get that 
\[
    a_{k} \leq \underbrace{\max_{s = 0, \ldots, T -  1} a_{k, s}}_{\text{bound by a constant in \eqref{a_k_constant}}} \cdot \overbrace{\max _{j = 1, 2, 3} s_{a, j}}^{\text{with order } \log T} + 3 T^{-1} \cdot\big(T - \underbrace{\max_{j = 1, 2, 3} s_{a, j}}_{\text{with order } \log T}\big) \lesssim \log T.
\]
\begin{remark}
   Here, we do not give the explicit expression of the bound as it involves several numeric integral terms, while its order is exactly $\log T$. However, for some explicit reward distributions, the expectation in Lemma \ref{lem_subE_ineq} can be bounded by some simpler expression, and hence we have the explicit expression for the bounds in these cases, see Corollary \ref{cor_ber} and Corollary \ref{cor_gau}.
\end{remark}

\subsection{Bounding $b_k$}

For bounding $b_k = \sum_{s = 0}^{T - 1} b_{k, s}$, we decompose $b_{k, s} = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1}\big) + 1$ as $b_k = b_{k, s, 1} + b_{k, s, 2} + b_{k, s, 3} + 1$ with
\[
    b_{k, s, 1} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-1}) 1(A_{k, s}^{\prime c} )\big],
\]
\[
    b_{k, s, 2} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-1}) 1(A'_{k, s}) 1(G_{k, s}^c)\big],
\]
\[
    b_{k, s, 3} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-1}) 1(A'_{k, s}) 1(G_{k, s})\big].
\]
Since $b_{k, s}$ is naturally bounded $2$, it only needs to bound $s_{b, j}: = \max \{ s: b_{k, s, j} \geq T^{-1}\}$ up to $\log T$ for $j = 1, 2, 3$.

It is easy to see that $\mathbb{P} (A^{\prime c}_{k, s}) \leq T^{-2} $ when $s \geq \frac{1}{2}s_{a, 2}'$. Here $s_{\cdot}'$ is the value of $s_{\cdot}$ but in it $C_1$ is replaced by $C_2$. Then we have $b_{k,s, 1} \vee b_{k, s, 2} \leq T^{-2} < T^{-1}$ as long as $s \geq (s'_{a, 1}  / 2 \vee s'_{a, 2}) \asymp \log T$. It remains to tackle $b_{k, s, 1}$. We will first bound $\Gamma_k = \overline{Y}_{k, s}  - \overline{R}_{k, s}^*$ on $A_{k, s}' \, \cap \, G_{k, s}$. Decompose
\[
    A'_{k, s} = \bigg\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2\lambda} < C_2 \Delta_k \bigg\} = \bigg\{ \tau_k - \frac{\mu_k + \lambda}{1 + 2\lambda} - C_2 \Delta_k  < \Gamma_k \bigg\},
\]
and
\[
    \begin{aligned}
        G_{k, s} = \big\{ -C_3 \Delta_k \leq \overline{Y}_{k, s} - \overline{R}_{k, s}^* \leq C_3 \Delta_k \big\} & = \big\{ \tau_k -C_3 \Delta_k \leq \overline{Y}_{k, s} + \Gamma_k \leq \tau_k + C_3 \Delta_k \big\} \\
        & = \big\{ \tau_k - 2 C_3 \Delta_k - \overline{R}_{k, s}^* \leq \Gamma_k \leq \tau_k + 2 C_3 \Delta_k - \overline{R}_{k, s}^* \big\} \\
        & \subseteq \bigg\{ \tau_k - 2 C_3 \Delta_k - \frac{1 + \lambda}{1 + 2\lambda} \leq \Gamma_k \leq \tau_k + 2 C_3 \Delta_k - \frac{\lambda}{1 + 2\lambda}\bigg\},
    \end{aligned}
\]
hence, we conclude that 
\[
    \begin{aligned}
        A'_{k, s} \, \cap \, G_{k, s} & \subseteq  \bigg\{ \tau_k - \frac{\mu_k + \lambda}{1 + 2\lambda} - C_2 \Delta_k  < \Gamma_k \leq\tau_k + C_2 \Delta_k - \frac{\lambda}{1 + 2\lambda} \bigg\} \\
        & \subseteq \bigg\{ \tau_k  - C_2 \Delta_k - \frac{\mu_k}{2}  < \Gamma_k < \tau_k + C_2 \Delta_k \bigg\}
    \end{aligned}
\]
if
\begin{equation}\label{requirement_3}
    2 C_3 \geq C_2.
\end{equation}
Under this establishment, we need $\tau_k - C_2 \Delta_k - \frac{\mu_k }{2} \leq 0$, then we can take
\begin{equation}\label{requirement_4}
    \tau_k = 2 C_2 \Delta_k + \mu_k / 2
\end{equation}
and we will obtain that the bound for $\Gamma_k$ is
\[
    C_2 \Delta_k = \tau_k  - C_2 \Delta_k - \frac{\mu_k}{2} < \Gamma_k < \tau_k + C_2 \Delta_k = 3 C_2 \Delta_k + \frac{\mu_k}{2} \leq \frac{7}{2} < 4,
\]
i.e. $\mathbb{R}_+ \ni \Gamma_k \in (C_2 \Delta_k, 4)$. Similar to previous, 
\[
    \begin{aligned}
        & Q_{k, s}\left(\tau_{k}\right) 1\left(A_{k, s}^{\prime}\right) 1\left(G_{k, s}\right) = {\mathbb{E}} \mathbb{P} \left(\overline{Y}_{k, s}-\overline{R}_{k, s}^{*}>\Gamma_{k} \mid \mathcal{H}_{k, T}\right) 1\left(A_{k, s}^{\prime}\right) 1\left(G_{k, s}\right) \\
        = & {\mathbb{E}} \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s \big[ (1 + 2\lambda) R_i - (\overline{R}_{1, s} + \lambda) - \Gamma_k (1 + 2 \lambda)\big] \omega_i \geq s \lambda \big[ 2 \Gamma_k (1 + 2 \lambda) + 2 \overline{R}_{1, s} - 1 \big]\Bigg\}\\
        \leq &  T \exp \left\{ - \frac{C_2^2 (1 + 2\lambda)^4 \Delta_k^2 s}{8 b^{**}}\right\} \times \Big[ 1 + C_2^2 (1 + 2\lambda)^4 \Delta_k^2 (h_1^{**} + h_2^{**} )\Big].
    \end{aligned}
\]
where $b^{**}$, $h_1^{**}$, and $h_2^{**}$ are the parameters in Lemma \ref{lem_subE_ineq} similarly defined as previous. Therefore, one can just take 
\[
    \begin{aligned}
        s \geq s_{b, 3} & :=  \frac{8 b^{**}}{C_2^2 (1 + 2\lambda)^4 \Delta_k^2 s} \times \log \Big[ 1 + C_2^2 \Delta_k^2 (1 + 2\lambda)^4  (h_1^{**} + h_2^{**})\Big] \log T
    \end{aligned}
\]
and then by using the similar argument, we have 
\[
    b_{k} \leq 2 \cdot \overbrace{\max _{j = 1, 2, 3} s_{b, j}}^{\text{with order } \log T} + 3 T^{-1} \cdot\big(T - \underbrace{\max_{j = 1, 2, 3} s_{b, j}}_{\text{with order } \log T}\big) \lesssim \log T.
\]
\begin{remark}
   Similarly, for specific reward distributions, we can bound $b_k$ in $\log T$ with some simple and specific coefficients, see Corollary \ref{cor_ber} and Corollary \ref{cor_gau}.
\end{remark}
Finally, what remains is to show the restrictions (\ref{requirement_0}), (\ref{requirement_1}), (\ref{requirement_2}), (\ref{requirement_3}), and (\ref{requirement_4}) are compatible. Indeed, they will build the following system of equations,
\[
    \left\{ \begin{array}{l}
         \frac{\mu_1}{2} - 2 C_1 \Delta_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \\
         2 C_2 \Delta_k + \frac{\mu_k}{2} \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \\
        \tau_k - \frac{\mu_1}{2} - C_1 \Delta_k  \leq \tau_k + C_2\Delta_k - \frac{\mu_k }{2} \\
        C_1 \geq 2C_3 \\
        2 C_3 \geq C_2 \\
        \frac{\mu_1}{2} - 2 C_1 \Delta_k = 2 C_2 \Delta_k + \frac{\mu_k}{2}
    \end{array}\right..
\]
We can see that 
\[
    C_1 = C_2 = \frac{1}{8}, \qquad C_3 = \frac{1}{16},
\]
is the unique solution for the above system. Up to now, we have finished the first part of our proof. For the second part, 
\begin{equation*}
    \begin{aligned}
        \operatorname{Reg}_T & = \sum_{\Delta_k : \Delta_k < \Delta} \Delta_k {\mathbb{E}} J_{k, T} + \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k {\mathbb{E}} J_{k, T} \\
        & \lesssim T \Delta + \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k  +  \sum_{\Delta_k : \Delta_k \geq \Delta} \frac{\log T}{\Delta_k} \\
        & \lesssim T \Delta + \frac{ K \log T}{\Delta} \lesssim \sqrt{K T \log T}.
    \end{aligned}
\end{equation*}
for any previously specified $\Delta \in (0, 1)$ by Cauchy inequality. Therefore, we finish the proof of Theorem \ref{thm.MAB}.





\section{Proof of Corollary \ref{cor_ber}}

As stated in Section \ref{proof_thm}, for bounded Bernoulli rewards, we still have $a_{k, s} \leq e^{9 /8} + \big[ 1 - \Phi (\sqrt{2 \pi}) \big]^{-1}$ as long as the tuning parameter $(\lambda, \sigma_{\omega}^2)$ satisfying \eqref{tuning_condition_exact}.
And similar, one can prove that for bounding $a_{k, s, j}$, one can easily show that
\begin{itemize}
    \item For $a_{k, s, 1}$, we have $a_{k, s, 1} \leq T^{-1}$ as long as
    \[
        s \geq s_{a, 2} := \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2} \log T.
    \]
    \item For $a_{k, s, 2}$, one can following the same steps to get that 
    \[
        \begin{aligned}
            a_{k, s, 2} & \leq {\mathbb{E}} \Big[ T  1(G_{1, s}^c)\Big] =  T {\mathbb{E}} \Big[ \mathbb{P}(G_{1, s}^c \mid \mathcal{H}_{1, s}) \Big] \\
            & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{ C_3^2 \Delta_k^2 (1 + 2 \lambda)^4 s^2 }{2 \sigma_{\omega}^2 \big[ \lambda(2 \overline{R}_{1, s} - 1) - C_3 \Delta_k (1 + 2 \lambda)\big]^4} \right\} \\
            & = 2 T {\mathbb{E}} \exp \left\{ - \frac{ C_3^2 \Delta_k^2 (1 + 2 \lambda)^4 s^6 }{2 \sigma_{\omega}^2 \big[ s \lambda (2 \overline{R}_{1, s} - 1) - s C_3 \Delta_k (1 + 2 \lambda)\big]^4} \right\}  =: 2 T {\mathbb{E}} f(s \overline{R}_{1, s})
        \end{aligned}
    \]
    where the function $f(x)$ is
    \[
        f(x) := \exp \left\{ - \frac{ C_3^2 \Delta_k^2 (1 + 2 \lambda)^4 s^6 }{2 \sigma_{\omega}^2 \big[  \lambda (2 x - s) - s C_3 \Delta_k (1 + 2 \lambda)\big]^4} \right\}.
    \]
    Then $f(x)$ is a decreasing function on $0 \leq x \leq s / 2$ and an increasing function on $x \geq s / 2$. By symmetric technique of Binomial distribution and Lemma 2 in \cite{kveton2019garbage}, we have
    \[
        a_{k, s, 2} \leq 4 T \sum_{x = 0}^{\lfloor s \mu_1 \rfloor} B(x ; s, \mu_1) f(x) \leq 4 T \left[ \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[-2 i^{2}\right] f(\mu_1 s -(i+1) \sqrt{s})+\exp \left[-2 s^{2} \mu_1^2 \right] f(0) \right].
    \]
    Similarly to \cite{kveton2019garbage}, we can show that 
    \[
        \begin{aligned}
            & \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[-2 i^{2}\right] f(\mu_1 s -(i+1) \sqrt{s}) +\exp \left[-2 s^{2} \mu_1^2 \right] f(0) \\
            = & \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[-2 i^{2}\right] \exp \left\{-  \frac{C_3^2 \Delta_k^2 (1 + 2 \lambda)^4 s^4}{2 \sigma_{\omega}^2 \left\{ \sqrt{s}\left[ \lambda (2 \mu_1 - 1) -C_3 (1 + 2\lambda)\right] - 2 \lambda (i + 1) \right\}^4}\right\} + \exp \left[-2 s^{2} \mu_1^2 \right] f(0) \\
            \leq & \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[- \left(2 i^{2} + \frac{  \Delta_k^2 (1 + 2\lambda)^4 s^4}{ 2 \sigma_{\omega}^2 c_1^{* 4} s^2} \right) \right] + \exp \left[-2 s^{2} \mu_1^2 \right]  \\
            \leq & \exp \left\{ - (2 \mu_1^2 \wedge \sigma_{\omega}^{-2} c_1^{* - 4}  \Delta_k^2 (1 + 2 \lambda)^4) s^2 \right\} \left( 1 + \sum_{i=0}^{\lfloor s \mu_1 \rfloor}  \exp \left[-2 i^{2}\right] \right) \\
            = & \left( 1 + \sqrt{\pi / 8}\right) \exp \left\{ - (2 \mu_1^2 \wedge  \sigma_{\omega}^{-2} c_1^{* - 4} \Delta_k^2 (1 + 2 \lambda)^4) s^2 \right\}
           
           
           
        \end{aligned}
    \]
    with $c_1^*$ is defined as
    \begin{equation}\label{c_1_star}
        c_1^{*} = c_1 (\lambda, \Delta_k) :=  \min_{0 \leq i \leq \lfloor s\mu_1 \rfloor, \, i \in \mathbb{N}} \frac{1}{\sqrt{s}} \big| \sqrt{s}\left[ \lambda (2 \mu_1 - 1) - (1 + 2\lambda) / 16 \right] - 2 \lambda (i + 1) \big|  > 0
    \end{equation}
    \red{In general, we have
    \[
        c_1^{*4} \geq \max_{k \in [K] / 1} \frac{2 \sigma_{\omega}^2 \mu_1^2}{\Delta_k^2 (1 + 2 \lambda)^4},
    \]
    and thus, $a_{k, s, 2} \leq 4 T (1 + \sqrt{\pi / 8}) \exp \left\{ - \sigma_{\omega}^{-2} c_1^{*-4} \Delta_k^2 (1 + 2 \lambda)^4 s^2 \right\}$.
    }
    Therefore, by taking 
    \[
        s_{a, 2} \geq \frac{5 \sigma_{\omega} \log T}{2 c_1^{* 2} \Delta_k (1 + 2 \lambda)^2} \geq \frac{\sigma_{\omega}}{c_1^{* 2} \Delta_k (1 + 2 \lambda)^2} \cdot \sqrt{\log \big( 4 (1 + \sqrt{\pi / 8}) T^2 \big)}
    \]
    one will have $ a_{k, s, 2} \leq T^{-1}$.\\
    
    \item By taking the same technique as $a_{k, s, 2}$, 
    one can show that
    \[
        \begin{aligned}
            \big[ 1 - Q_{1, s}(\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \leq 2 \sum_{x = 0}^{\lfloor s \mu_1 \rfloor} B(x; s, \mu_1) g(x) \leq 2 s \mu_1 \exp \{ - s \mu_1\} \leq T^{-1}
        \end{aligned}
    \]
    as long as $s \geq s_{a, 3} := s_{a, 2}$. And hence in this case, one will have $a_{k, s, 3} \leq T^{-1}$.
\end{itemize}

Combining these results, we obtain that 
\[
    \begin{aligned}
        a_k & \leq \max_{s = 0, \ldots, T -  1} a_{k, s} \cdot \max _{j = 1, 2, 3} s_{a, j} + 3 T^{-1} \cdot\big(T - \max_{j = 1, 2, 3} s_{a, j}\big) \\
        & \leq \left[ e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \right] \left[ \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2} + \frac{5}{2  (1 + 2 \lambda)^2 c_1^{* 2} \Delta_k }\right] 
        \log T + 3.
    \end{aligned}
\]
Specially, if $\sigma_{\omega}^2 \geq 1$ and $\lambda \geq 1$, by using the fact that $\lambda^2\big[ (\lambda - 1)^2 + \lambda^2\big] \geq \lambda (\lambda + 1)$, one will have 
\[
    a_k \leq  \left[ e^{ 9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \right] \left[ \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2} + \frac{5}{2  (1 + 2 \lambda)^2 c_1^{* 2} \Delta_k } \right] \log T + 3.
\]
By using the same technique, one can show that 
\[
    b_k \leq 2 \cdot \left[ \frac{2}{(1 + 2 \lambda)^2 \Delta_k^2} + \frac{5}{2  (1 + 2 \lambda)^2 c_k^{* 2} \Delta_k } \right] \log T + 3
\]
for any $\lambda > 0$ with
\begin{equation}\label{c_k_star}
    c_k^{*}  :=  \min_{0 \leq i \leq \lfloor s\mu_k \rfloor, \, i \in \mathbb{N}} \frac{1}{\sqrt{s}} \big| \sqrt{s}\left[ \lambda (2 \mu_k - 1) - (1 + 2\lambda) / 16 \right] - 2 \lambda (i + 1) \big|  > \max_{k \in [K] / 1} \frac{2 \sqrt{\sigma_{\omega} \mu_k}}{\sqrt{\Delta_k} (1 + 2 \lambda)}.
\end{equation}
Therefore, we complete the proof.

\section{Proof of Corollary \ref{cor_gau}}

Follow the steps in Proof of Corollary \ref{cor_ber}. We claim the following statements:
\begin{itemize}
    \item[1.] Note the fact that $\sigma_{1}^{-2} s^2 (\overline{R}_{1, s} - \mu_1)^2 \sim \chi^2(s)$, here one give more precise condition for the tuning parameter. Indeed, one can show that
    \[
        \begin{aligned}
            a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (1 + 2 \lambda)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}  \\
            (\text{by using the MGF of } \chi^2(s))& = \left[ 1 - \frac{3 \sigma_1^2 (2 \lambda + 1)^2}{ s \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right]^{- \frac{s}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \\
            (\text{take } s = 1) & \leq \left[ 1 - \frac{3 \sigma_1^2 (2 \lambda + 1)^2}{ \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} \big]^2}\right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}.
        \end{aligned}
    \]
   
    since $s = 0$ naturally satisfies the above inequality. This further implies we will need
    \[
       \frac{3 \sigma_1^2 (2 \lambda + 1)^2}{ \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} \big]^2} < 1
    \]
    with its sufficient condition as 
    \[
        3 (2 \lambda + 1)^2 < \frac{12}{(2 \mu_1 - 1)^2}  \big[ (2 \mu_1 - 1) \lambda + \sqrt{2 \pi} \big]^2
    \]
    i.e. 
    \begin{equation}\label{cor_2_tuning_suf}
        4 \left[ \frac{2 \sqrt{2 \pi}}{2 \mu_1 - 1} - 1\right] \lambda > 1 - \frac{8 \pi}{(2 \mu_1 - 1)^2}
    \end{equation}
    if we take $\sigma_{\omega}^2 \geq \frac{12 \sigma_1^2}{(2 \mu_1 - 1)^2}$. Note that $2 \mu_1 - 1 \in (0, 1]$, we have $\left[ \frac{2 \sqrt{2 \pi}}{2 \mu_1 - 1} - 1\right] > 0$ and $1 - \frac{8 \pi}{(2 \mu_1 - 1)^2} < 0$, \eqref{cor_2_tuning_suf} always hold for any $\lambda > 0$. 
    In a word, under the assumptions in Corollary \ref{cor_gau}, one will have
    \[
        \begin{aligned}
            a_{k, s} \leq \left[ 1 - \frac{3 (2\mu_1 - 1)^2 (2 \lambda + 1)^2}{ 4 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} \big]^2}\right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}.
        \end{aligned}
    \]
    with $\sigma_{\omega}^2 \geq \frac{12 \sigma_1^2}{(2 \mu_1 - 1)^2}$ for $s \in \{ 0, \ldots, T - 1\}$.
    
    \item[2.] For controlling $a_{k}$, one can easily show that 
    \[
        \begin{aligned}
            a_k & \leq \left\{ \left[ 1 - \frac{3 (2\mu_1 - 1)^2 (2 \lambda + 1)^2}{ 4 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} \big]^2}\right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \right\} \\
            & \qquad \qquad \left[ \frac{64 \sigma_1^2}{(1 + 2 \lambda)^2 \Delta_k^2} + (1 + \sqrt{2})\sqrt[4]{3 \times 2^{13}} \frac{\lambda \sigma_{\omega}^{3/2}}{(1 + 2\lambda) \sqrt{\Delta_k}}  \right] \log T + 3.
        \end{aligned}
    \]
    by the following statements.
    \begin{itemize}
        \item[2.1] Still use sub-Gaussian inequality to get that we have $a_{k, s, 1} \leq T^{-1}$ when
        \[
            s \geq s_{a, 1} = \frac{64 \sigma_1^2}{(1 + 2 \lambda)^2 \Delta_k^2} \log T.
        \]
        
        \item[2.2] For dealing with $a_{k, s, 2}$, we again use the fact that
        \[
            \begin{aligned}
                \big[ \lambda(2 \overline{R}_{1, s} - 1) - C_3 \Delta_k (1 + 2 \lambda)\big]^4 & = \big[  2 \lambda (\overline{R}_{1, s} - \mu_1) + \lambda (2 \mu_1 - 1) - C_3 \Delta_k (1 + 2 \lambda) \big]^4 \\
                & =: \big[ \xi + \lambda (2 \mu_1 - 1) - C_3 \Delta_k (1 + 2 \lambda) \big]^4
            \end{aligned}
        \]
        with $\xi \sim N (0, 4 s^{-1} \lambda^2)$. Denote
        \[
            A: = C_3^2 (1 + 2 \lambda)^4 \Delta_k^2, \qquad B:= \lambda(2 \mu_1 - 1) - C_3 \Delta_k (1 + 2 \lambda),
        \]
        then it will lead to the result that 
        \[
            \begin{aligned}
                a_{k, s, 2} & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{C_3^2 (1 + 2 \lambda)^4 \Delta_k^2 s^2}{ 2 \sigma_{\omega}^2 \big[ \xi + \lambda (2 \mu_1 - 1) - C_3 \Delta_k (1 + 2 \lambda) \big]^4}\right\} = 2 T {\mathbb{E}} \exp \left\{ - \frac{A s^2}{  2 \sigma_{\omega}^2 \big[ \xi + B \big]^4}\right\} \\
                & = \frac{T}{\lambda} \sqrt{\frac{s}{2 \pi}} \int_{- \infty}^{\infty} \exp \left\{ - \left[ \frac{A s^2}{ 2 \sigma_{\omega}^2(u + B)^4} + \frac{s u^2}{32 \sigma_{\omega}^2 \lambda^2}\right] \right\} \exp \left\{ - \frac{s u^2}{ 32 \sigma_{\omega}^2 \lambda^2} \right\} \, du \\
                & \leq \frac{3 T}{\lambda}  \sqrt{\frac{s}{2 \pi}} \exp\left\{ - 3 \left[ \frac{A s^4}{2^7 \sigma_{\omega}^6 \lambda^4}\right]^{1 / 3} \right\} \int_{- \infty}^{\infty} \exp \left\{ - \frac{s u^2}{32 \sigma_{\omega}^2 \lambda^2} \right\} \, du = 12 \sigma_{\omega} T \exp\left\{ - 3 \left[ \frac{A s^4}{2^7 \sigma_{\omega}^2 \lambda^4}\right]^{1 / 3} \right\}.
            \end{aligned} 
        \]
        Therefore, for any $T \geq \lceil 2 \sigma_{\omega} \rceil $, take
        \[
            s \geq s_{a, 2} := \sqrt[4]{3 \times 2^{15}} \frac{\lambda \sigma_{\omega}^{3/2}}{(1 + 2\lambda) \sqrt{\Delta_k}} \log T \geq \sqrt[4]{\frac{2^{15}}{3^3}} \frac{\lambda \sigma_{\omega}^{3/2}}{(1 + 2\lambda) \sqrt{\Delta_k}} \log^{3 / 4} \big( 2 \sigma_{\omega} T^2 \big)
        \]
        we will get that $a_{k, s, 2} \leq T^{-1}$.\\
        
        \item[2.3] Just by replacing $C_3 \Delta_k$ by $(- \Gamma_1)$ which is bounded in $(C_1 \Delta_k, 4)$, and we will get that
        \[
            \begin{aligned}
                \big[ 1 - Q_{1, s}(\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \leq T^{-1}
            \end{aligned}
        \]
        i.e. $a_{k, s, 3} \leq T^{-1}$ whenever 
        \[
            s \geq s_{a, 3} := \sqrt[4]{3 \times 2^{13}} \frac{\lambda \sigma_{\omega}^{3/2}}{(1 + 2\lambda) \sqrt{\Delta_k}} \log T .
        \]
    \end{itemize}
    
    \item[3.] Exactly the same as 2., we can get that 
    \[
        b_{k} \leq 2 \left[ \frac{32 \sigma_k^2}{(1 + 2 \lambda)^2 \Delta_k^2} + (1 + \sqrt{2})\sqrt[4]{3 \times 2^{13}} \frac{\lambda \sigma_{\omega}^{3/2}}{(1 + 2\lambda) \sqrt{\Delta_k}}  \right] \log T + 3.
    \]
\end{itemize}
These results give Corollary \ref{cor_gau}.
\section{Proof of Theorem \ref{thm.MAB}}\label{proof_thm}


Before starting our proof, we first give the definition of sub-exponential variables. To be specific, we call a \textit{zero-mean} variable $X$ sub-exponential with parameters $\lambda$ and $\alpha$ if
\[
    {\mathbb{E}} \exp \{ t X \} \leq \exp \left\{ \frac{t^2 \lambda^2}{2}\right\}, \qquad |t| \leq \frac{1}{\alpha} ,
\]
denoted as $X \sim \operatorname{subE}(\lambda, \alpha)$. Specially, we denote $\operatorname{subE}(\lambda) := \operatorname{subE}(\lambda, \lambda)$. 
Similar to the sub-Gaussian variables, we have the following concentration {property}
\[
    \mathbb{P} (X \geq t) \leq \exp \left\{ - \frac{1}{2} \left( \frac{t^2}{\lambda^2} \wedge \frac{t}{\alpha}\right) \right\}, \qquad  {\forall} \, t \geq 0.
\]

   

   



\subsection{Basic Idea}

Our proof relies on the following decomposition of the cumulative regret from \cite{kveton2019garbage}. 
We will recap its proof in Appendix \ref{sec:proof_lemma} for completeness. Let the history of arm $k$ after $s$ pulls be a set $\mathcal{H}_{k, s}$ of cardinality $s$, $Q_{k, s} (\tau) := \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_{k, s} \big)$ be the tail probability that $\overline{Y}_{k, s}$ conditioned on history $\mathcal{H}_{k, s}$ is a least $\tau$ for some tunable parameter $\tau$, and $N_{k, s} (\tau) := 1 / Q_{k, s} (\tau) - 1$.
\begin{lemma}\label{lem.a.b}
Suppose in MAB we select arms according to the rule 
$A_t = \arg \max_{k \in [K]} \overline{Y}_{k, t}$. 
Then for any $\{ \tau_k \}_{k = 2}^K \subseteq \mathbb{R}^{K - 1}$, 
the expected $T$-round regret can be bounded above as
\[
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
\]
where
\[
    a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] =: \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \sum_{s = 0}^{T - 1} \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1 =: \sum_{s = 0}^{T - 1} b_{k, s} + 1.
\]
\end{lemma}
Recall the summation index $s$ is the time we pull the $k$-th arm in the total $T$ round{s}. 
The definition of $a_{k}$ and $b_{k}$ have important meaning, $a_{k}$ represents the expected number of rounds that optimal arm $1$ has been being underestimted, while $b_{k}$ is the probability that the suboptimal arm $k$ is being overestimated.
Here we only need to consider the lower bound of the tail of the distribution of the rewards from the optimal arm. 
The intuition is: (i) we only need the rewards from the optimal arm taking a relatively large value with a probability that is not too small; (ii) we do not care about the tiny probability of a large reward from suboptimal arms. 


In the following steps, we will bound $a_k$ and $b_k$ separately up to order $\log T$. Denote $\mathbb{P}_{\xi}(A)$ is the probability of event $A$ concerning the law of $\xi$ and taking other quantities as deterministic. Recall that we denote two functions $a(s, T) \lesssim b(s, T)$ if $a(s, T) \leq c \times b(s, T)$ for some constant $c$ free of $s$ and $T$. 
And we write $a(s, T) \asymp b(s, T)$ if both $a(s, T) \lesssim b(s, T)$ and $a(s, T) \gtrsim b(s, T)$. 
The proof of all lemmas can be found in Appendix \ref{sec:proof_lemma}. 

\subsection{Bounding $a_k$}
For bounding $a_{k}$ with $C\times\log T$ for some constant $C > 0$, we will need first prove a weaker statement: $a_{k, s}$ as a function of $T$ can be upper bounded by some constants free of $T$. As we can see later, this is a critical intermediate step for proving $a_{k, s}$ has an order $\log T$.
   


Note that if we take
\begin{equation}\label{requirement_0}
    \tau_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda},
\end{equation}
then we will have
\begin{equation}\label{constant_bound}
    \begin{aligned}
    	a_{k, s} & = {\mathbb{E}} \left[ \left( \frac{1}{Q_{1, s}(\tau_k)}  - 1\right) \wedge T \right] \\
    	(\text{by } Q_{1, s}(\cdot) \text{ is decreasing})&\leq {\mathbb{E}} \frac{1}{Q_{1, s}(\tau_k)} \leq {\mathbb{E}} Q_{1, s}^{-1} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right),
    \end{aligned}
\end{equation}
Hence, we need to find a lower bound of
the tail probability $Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right) = \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right)$. And throughout the proof, we will use the choice of \eqref{requirement_0} for $\tau_k$, and we can take advantage of the bound \eqref{constant_bound}. 

For further analyzing \eqref{constant_bound}, we write it as the probability with respect to the weighted random summation of Gaussian random variable $\{ \omega_i, \omega_i', \omega_i''\}$.

\begin{lemma}\label{lemma:gaussian_weight}
    Suppose $\{x_i, y_i, z_i\}_{i = 1}^n \subseteq \mathbb{R}$ and $\{ \omega_i, \omega_i', \omega_i''\}_{i = 1}^n$ i.i.d. $\mathcal{N} (1, \sigma_{\omega}^2)$ random variables, then 
    \[
        \begin{aligned}
            & \mathbb{P} \left( \sum_{i = 1}^n x_i \omega_i > \sum_{ i = 1}^n y_i \omega_i' + \sum_{i = 1}^n z_i \omega_i'' \right)\\
            \geq & \exp \left\{-\frac{3\left(S_{x}-S_{y}-S_{z}\right)^{2}}{2 \sigma_{\omega}^{2}\left(T_{x}+T_{y}+T_{z}\right)}\right\}\times\mathbb{I} \left(S_{x} \geq S_{y}+S_{z}+\sqrt{2 \pi}\right)+(1-\Phi(\sqrt{2 \pi})) \times\mathbb{I} \left(S_{x} \leq S_{y}+S_{z}+\sqrt{2 \pi}\right),
    \end{aligned}
    \] 
    if $T_x + T_y + T_z \neq 0$. Furthermore, if $S_x + S_y + S_z \neq 0$, we will have 
    \[
        \begin{aligned}
            & \mathbb{P} \left( \sum_{i = 1}^n x_i \omega_i > \sum_{ i = 1}^n y_i \omega_i' + \sum_{i = 1}^n z_i \omega_i'' \right)\\
            \geq & \exp \left\{-\frac{3\left(S_{x}-S_{y}-S_{z}\right)^{2}}{2 \sigma_{\omega}^{2} / n \left(S_x + S_y + S_z\right)^2}\right\} \times\mathbb{I} \left(S_{x} \geq S_{y}+S_{z}+\sqrt{2 \pi}\right)+(1-\Phi(\sqrt{2 \pi})) \times\mathbb{I} \left(S_{x} \leq S_{y}+S_{z}+\sqrt{2 \pi}\right),
        \end{aligned}
    \] 
    where $S_x = \sum_{i = 1}^n x_i$ and $T_x = \sum_{i = 1}^n x_i^2$. And $S_y, S_z, T_y, T_z$ are similarly defined. 
\end{lemma}

And we first consider the case that $\mu_1 \neq {1} / {2}$. By applying Lemma \ref{lemma:gaussian_weight}
\[
    \begin{aligned}
        & Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right) \\
        = & \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \big( (2 \lambda + 1) R_{1, i} - (\mu_1 + \lambda) \big)\omega_i > \lambda (\mu_1 - 1 - \lambda) \sum_{i = 1}^s \omega_i' + \lambda (\lambda + \mu_1) \sum_{i = 1}^s \omega_i'' \right).
    \end{aligned}
\]
Let $x_i := (2 \lambda + 1) R_{1, i} - (\mu_1 + \lambda)$, $y_i := \lambda (\mu_1 - 1 - \lambda) $, $z_i :=  \lambda (\lambda + \mu_1) $ for $i \in [s]$, then
\[
    S_x = \sum_{i = 1}^s x_i = s \big( (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big), \quad S_y = \sum_{i = 1}^s y_i = s \lambda (\mu_1 - 1 - \lambda), \quad S_z = \sum_{i = 1}^s z_i = s \lambda (\lambda + \mu_1),
\]
with $S_x - S_y - S_z = s (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1)$ and
\[
    T_x = \sum_{i = 1}^s x_i^2 = \sum_{i = 1}^s \big[  (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big]^2, \quad T_y = \sum_{i = 1}^s y_i^2 = s \lambda^2 (\mu_1 - 1 - \lambda)^2, \quad T_z = \sum_{i = 1}^s z_i^2 = s \lambda^2 (\lambda + \mu_1)^2,
\]
with $T_x + T_y + T_z = \sum_{i = 1}^s \big[  (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big]^2  + s \big[ \lambda^2 (\mu_1 - 1 - \lambda)^2 + \lambda^2 (\lambda + \mu_1)^2\big]$. Hence, by applying the first inequality in Lemma \ref{lemma:gaussian_weight} and use the fact that $T_x + T_y + T_z \geq T_x \geq S_x^2 / s$\footnote{If one use $T_x + T_y + T_z \geq T_x$, one cannot give a valid tuning condition.}.
\[
    \begin{aligned}
        & Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s x_i \omega_i > \sum_{i = 1}^s y_i \omega_i' + \sum_{i = 1}^s z_i \omega_i'' \right) \\
        \geq & \exp \left\{-\frac{3s^2 (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^{2}\left(T_{x}+T_{y}+T_{z}\right)}\right\}\times\mathbb{I} \left(s (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1) \geq \sqrt{2 \pi}\right) \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~ + (1-\Phi(\sqrt{2 \pi})) \times\mathbb{I} \left(s (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1) \leq \sqrt{2 \pi} \right) \\
        \alignedoverset{T_x + T_y + T_z \geq S_x / s}{\geq} \exp \left\{-\frac{3(2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^{2} s \big( (2 \lambda + 1) \overline{R}_{1, s} - (\mu_1 + \lambda) \big)^2 }\right\}\times\mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}   \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~ + (1-\Phi(\sqrt{2 \pi})) \times\mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} 
    \end{aligned}
\]
Then this result combined with the fact \eqref{constant_bound} gives the upper bound for $a_{k, s}$ as
   
       
       
       
   
\begin{equation}\label{a_k_constant_2}
    \begin{aligned}
        a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times\mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \\
        & ~~~~~~~~~~~~~~ + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \times \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 \leq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right) \\ 
        & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2  \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times\mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  \mathbb{I} \left( \mu_1 > \frac{1}{2} \right) \\
        & ~~~~ + {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right)  \mathbb{I} \left( \mu_1 < \frac{1}{2} \right)  + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \\
        & = : I + II + III, 
       
       
       
       
       
       
       
    \end{aligned}
\end{equation}
where $III = \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}$ is a constant. We will bound $I$ and $II$ as follows. 
\begin{itemize}
	\item Bounding $I$: 
	\[
	    \begin{aligned}
	    	I & = {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2  \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2  }\right\} \times\mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right)  \mathbb{I} \left(  \mu_1 > \frac{1}{2} \right) \\
	    	   \alignedoverset{\overline{R}_{1, s} > \mu_1 > \frac{\mu_1 + \lambda}{1 + 2 \lambda} \text{ when } \mu_1 > \frac{1}{2}}{\leq} {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2  \left(  \mu_1 +\frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 \mathbb{I} \left( \overline{R}_{1, s}  \geq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right) }\right\} \\
	    	  
	    	  
	    	 
	    	 
	    	 
	    	  & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ \times\mathbb{I} \left( \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right)  \mathbb{I} \left(  \mu_1 > \frac{1}{2} \right)\\
	    	  \alignedoverset{\text{by using } \mu_1 > \frac{1}{2}}{\leq} {\mathbb{E}} \exp \left\{ \frac{3 (2 \lambda + 1)^2 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\}  \mathbb{I} \left( \overline{R}_{1, s }- \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right) \mathbb{I} \left(  \mu_1 > \frac{1}{2} \right).
	    \end{aligned}
	\]
	
	\item Bounding $II$: 
	

	


	   
	    
	    
	    
	    
	    
	   

	
	

	We first note under the indicator $\mathbb{I}\left(\overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right)$, we have
	\[
	    \overline{R}_{1, s} > \mu_1 + \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} > \mu_1.
	\]
	The basic idea is the fact that ${\mathbb{E}} X = \int_0^{\infty} \mathbb{P} (X > r) \, \mathrm{d} r$ for any positive $X$. Hence, for any $r > 0$,  we consider set that 
	\[
	    \begin{aligned}
	    	E(r, s) & := \left\{ \overline{R}_{1, s} - \mu_1: \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} > r \right\} \\
	    	& = \left\{  \overline{R}_{1, s} - \mu_1 : \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2} > \log r\right\}.
	    \end{aligned}
	\]
	We need to divide the analysis into two cases: (i) $\overline{R}_{1, s} \geq \frac{\mu_1 + \lambda}{1 + 2 \lambda} $ and (ii)
	$\overline{R}_{1, s} < \frac{\mu_1 + \lambda}{1 + 2 \lambda} $ respectively. We first prove the case (i) since it is easier. Note that 
	\begin{equation}\label{a_k_constant_II_case1}
		\overline{R}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} > \mu_1
	\end{equation}
	when $\mu_1 < \frac{1}{2}$. Hence we can write $E(r, s)$ on condition \eqref{a_k_constant_II_case1} as
	\[
	    \begin{aligned}
	    	E(r, s) \alignedoverset{\text{by \eqref{a_k_constant_II_case1}}}{=} \left\{  \overline{R}_{1, s} - \mu_1 : \frac{\sqrt{3 s} (\overline{R}_{1, s} - \mu_1)}{\sqrt{2} \sigma_{\omega} \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)} > \sqrt{\log r}\right\} \\
	    	& = \left\{  \overline{R}_{1, s} - \mu_1 : \sqrt{2 \log r} \sigma_{\omega} \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} > \left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right) (\overline{R}_{1, s} - \mu_1) \right\} \\
	    	& = \left\{ \begin{array}{ll}
	    		\left\{  \overline{R}_{1, s} - \mu_1 : \sqrt{2 \log r} \sigma_{\omega} \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} > 0 >  \left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right) (\overline{R}_{1, s} - \mu_1) \right\}, & r \leq \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}, \\
	    		\left\{  \overline{R}_{1, s} - \mu_1 : \frac{\sqrt{2 \log r} \sigma_{\omega} \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda}}{\left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right)} > (\overline{R}_{1, s} - \mu_1) \right\}, & r > \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\} 
	    	\end{array}\right. \\
	    	\alignedoverset{\text{by \eqref{a_k_constant_II_case1} again}}{=} \left\{ \begin{array}{ll}
	    		\left\{ \overline{R}_{1, s} - \mu_1 \in \mathbb{R} \right\}, & r \leq \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}, \\
	    		\left\{  \overline{R}_{1, s} - \mu_1 : \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} \frac{\sqrt{2 \log r} \sigma_{\omega}} {\left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right)} > (\overline{R}_{1, s} - \mu_1) \right\}, & r > \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\} 
	    	\end{array} \right. 
	    \end{aligned}
	\]
	Then,
	\[
	    \begin{aligned}
	    	II & = \int_0^{\infty} \mathbb{P} \left( E(r, s) \, \cap \, \left\{ \overline{R}_{1,s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right\} \right) \, \mathrm{d} r \\
	    	& = \left[ \int_0^{ \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}} + \int_{\exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}}^{\infty} \right] \mathbb{P} \left( E(r, s) \, \cap \, \left\{ \overline{R}_{1,s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right\} \right) \, \mathrm{d} r \\
	    	& = II_1 + II_2.
	    \end{aligned} 
	\]
	We first bound $II_2$. Write $II_2$ as
	\[
	    II_2 = \int_{\exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}}^{\infty} \mathbb{P} \left( \frac{\lambda(1 - \mu_1)}{1 + 2 \lambda} < \overline{R}_{1,s} - \mu_1 <  \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} \frac{\sqrt{2 \log r} \sigma_{\omega}} {\left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right)} \right) \, \mathrm{d} r.
	\]
	By letting $s \longrightarrow \infty$,
	\[
	    \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\} \, \uparrow \, \infty, \qquad \left\{ \frac{\lambda(1 - \mu_1)}{1 + 2 \lambda} < \overline{R}_{1,s} - \mu_1 <  \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} \frac{\sqrt{2 \log r} \sigma_{\omega}} {\left( \sqrt{2 \log r} \sigma_{\omega} - \sqrt{3 s}\right)} \right\}  \, \downarrow \, \varnothing,
	\]
	hence we have $II_2 \longrightarrow 0$. This implies  there exist $T_1 \in \mathbb{N}$ such that $II_2 \leq 1$ for any $s \geq T_1$. And for $II_1$, we take advantage of sub-Gaussian inequality,
	\[
	    \begin{aligned}
	    	II_1 & = \int_0^{ \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}} \mathbb{P} \left( E(r, s) \mathbb{I} \left\{ \overline{R}_{1,s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right\} \right) \, \mathrm{d} r \\
	    	& \leq \int_0^{ \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}} \mathbb{P} \left( \overline{R}_{1,s} > \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right) \, \mathrm{d} r \\
	    	& \leq \int_0^{ \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}} \mathbb{P} \left( \overline{R}_{1,s} - \mu_1 > \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda} \right) \, \mathrm{d} r \\
	    	\alignedoverset{\text{by sub-Gaussian inequality}}{\leq} \exp \left\{ \frac{3s}{2 \sigma_{\omega}^2}\right\}  \cdot \exp \left\{ - \frac{\lambda^2 (1 - 2 \mu_1)^2 s}{2 (1 + 2 \lambda)^2 \sigma_1^2}\right\} \\
	    	& = \exp \left\{ \left[ \frac{3}{2 \sigma_{\omega}^2} - \frac{\lambda^2 (1 - 2 \mu_1)^2}{2 (1 + 2 \lambda)^2 \sigma_1^2} \right] s \right\}.
	    \end{aligned}
	\]
	Therefore, for making $II_1$ have a upper a constant    free of $s$, a sufficient condition is
	\[
	    \frac{3}{2 \sigma_{\omega}^2} - \frac{\lambda^2 (1 - 2 \mu_1)^2}{2 (1 + 2 \lambda)^2 \sigma_1^2} \geq 0
	\]
	i.e.
	\begin{equation}\label{tuning_case_1}
		\lambda > \frac{\sqrt{3}\sigma_1}{(1 - 2\mu_1) \sigma_{\omega} - 2 \sqrt{3}\sigma_1} \qquad  \text{ with } \qquad \sigma_{\omega} > \frac{2 \sqrt{3} \sigma_1}{1 - 2\mu_1}.
	\end{equation}
	Combining these results, we conclude that if take \eqref{tuning_case_1}, we have $II \leq 2$ for any $s \geq T_1$. Now switching to case (ii). In case (ii), we will have:
	\begin{equation}\label{a_k_constant_II_fact1}
		\frac{\mu_1 + \lambda}{1 + 2 \lambda} - \overline{R}_{1, s} > 0 \qquad \text{ and } \qquad \frac{\sqrt{2 \pi}}{s(2 \lambda + 1)} < \overline{R}_{1, s} - \mu_1 < \frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda}.
	\end{equation}
	\[
	    \begin{aligned}
	    	E(r, s) & = \left\{  \overline{R}_{1, s} - \mu_1 : \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2} > \log r\right\} \\
	    	(\text{use \eqref{a_k_constant_II_fact1}}) & \subseteq \left\{  \overline{R}_{1, s} - \mu_1 : \frac{3 s (\overline{R}_{1, s} - \mu_1)^{2 - \epsilon}}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2} > \log r \left[\frac{\lambda(1 - 2 \mu_1)}{1 + 2 \lambda}\right]^{-\epsilon}\right\} \\
	    	& =  \left\{  \overline{R}_{1, s} - \mu_1 : \left[ \frac{(3 s)^{\frac{1}{2 - \epsilon}} (\overline{R}_{1, s} - \mu_1)}{\log^3 r}\right]^{2 - \epsilon} > \left[ \frac{\sqrt{2} \sigma_{\omega}\left( \frac{1 + 2 \lambda}{\lambda(1 - 2 \mu_1)}\right)^{\frac{\epsilon}{2}}\left( \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \overline{R}_{1, s}\right)}{\log^{\frac{5 - 3\epsilon}{2}} } \right]^2\right\} \\
	    	& =  \left\{  \overline{R}_{1, s} - \mu_1 : \left[ \frac{(3 s)^{\frac{1}{2 - \epsilon}} (\overline{R}_{1, s} - \mu_1)}{\log^3 r}\right]^{1 - \frac{\epsilon}{2}} > \frac{\sqrt{2} \sigma_{\omega}\left( \frac{1 + 2 \lambda}{\lambda(1 - 2 \mu_1)}\right)^{\frac{\epsilon}{2}}\left( \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \overline{R}_{1, s}\right)}{\log^{\frac{5 - 3\epsilon}{2}} } \right\}
	    \end{aligned}
	\]
	For any $0 < \epsilon < 2$. Specially, we take $\epsilon = 1$, and 

	\[
	    F(r, s) := \left\{ \frac{(3 s)^{\frac{1}{2 - \epsilon}} (\overline{R}_{1, s} - \mu_1)}{\log^{3} r} \geq 1 \right\},
	\]
	then
	\[
	    \begin{aligned}
	    	II & = \int_0^{\infty} \mathbb{P} \left\{ E (r, s)  \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right) \right\} \, \mathrm{d} r \\
	    	& \leq \int_0^{\infty} \left[\mathbb{P} \left\{ \big[ E (r, s) \, \cap  \, F(r, s) \big] \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right) \right\} + \mathbb{P} \left\{ \big[ F(r, s)^c \big] \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right)\right\} \right]\, \mathrm{d} r \\
	    	& =: II_1 + II_2.
	    \end{aligned}
	\]
	For $II_2$, we write that
	\[
	    \begin{aligned}
	    	II_2 & \leq \int_0^{\infty} \mathbb{P} \left\{ \big[ F(r, s)^c \big] \, \cap \, \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right)\right\}\, \mathrm{d} r  \\
	    	& = \int_0^{\infty} \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \, \mathrm{d}r \\
	    	& = \left[ \int_0^{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}} + \int_{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}}^{\infty} \right] \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \, \mathrm{d}r \\
	    	& = \int_{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}}^{\infty}  \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right)  \, \mathrm{d}r.
	    \end{aligned}
	\]
	For $II_2$, we write that
	\[
	    \begin{aligned}
	    	II_2 & = \int_0^{\infty} \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \, \mathrm{d}r \\
	    	& = \left[ \int_0^{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}} + \int_{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}}^{\infty} \right] \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \, \mathrm{d}r \\
	    	& = \int_{\exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\}}^{\infty}  \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right)  \, \mathrm{d}r.
	    
	    
	    
	    
	    \end{aligned}
	\]
	By letting $s \longrightarrow \infty$, we have 
	\[
	    \exp\left\{ \frac{2 \cdot 3^{\frac{2}{2 - \epsilon}}}{(1 + 2 \lambda)} s^{\frac{2(\epsilon - 1)}{2 - \epsilon}}\right\} \longrightarrow \infty, \qquad \mathbb{P} \left( \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}  < \overline{R}_{1, s} - \mu_1 < \frac{\log^{3} r}{(3s)^{\frac{1}{2 - \epsilon}}}\right) \longrightarrow 0.
	\]
	Thus, $II_2$ shows we must have $II_2 \leq 1$ bounded by a constant free of $s$ for enough big $s$, say $s \geq T_2$.
	And for $II_1$, we take advantage that $x \geq x^{1 - \frac{\epsilon}{2}}$ for $x > 1$ and $0 < 1 - \frac{\epsilon}{2}< 1$, 
	\[
	    \begin{aligned}
	    	& E(r, s) \, \cap \, F(r, s) \\
	    	\subseteq & \left\{  \overline{R}_{1, s} - \mu_1 :  \frac{(3 s)^{\frac{1}{2 - \epsilon}} (\overline{R}_{1, s} - \mu_1)}{\log^3 r} > \frac{\sqrt{2} \sigma_{\omega}\left( \frac{1 + 2 \lambda}{\lambda(1 - 2 \mu_1)}\right)^{\frac{\epsilon}{2}}\left( \frac{\mu_1 + \lambda}{1 + 2 \lambda} - \overline{R}_{1, s}\right)}{\log^{\frac{5 - 3\epsilon}{2}} } \right\} \\
	    	= & \left\{  \overline{R}_{1, s} - \mu_1 : \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r}\right\}.
	    \end{aligned}
	\]
	and in turn, we have
	\[
	    \begin{aligned}
	    	 & \big[ E(r, s) \, \cap \, F(r, s) \big] \mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right\} \\
	    	
	    	 \subseteq & \left\{ \begin{array}{ll}
	    	 	\varnothing, & r \leq \overline{r}_1 (s) \\
	    	 	\left\{ \overline{R}_{1, s} - \mu_1 : \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r}\right\}, & r > \overline{r}_1(s) 
	    	 \end{array}\right..
	    \end{aligned}
	\]
	where
	\[
	    \overline{r}_1 (s) := \exp \left\{ \left[ \frac{\sqrt{2 \pi}}{\left[ \sqrt{2} \lambda (1 - 2 \mu_1) - 2 \sqrt{\pi} / s\right] \sigma_{\omega}}\right]^{\frac{2}{1 + 3 \epsilon}} \left( \frac{\lambda (1 - 2\mu_1)}{1 + 2\lambda} \right)^{\frac{\epsilon}{1 + 3 \epsilon}} (3s)^{\frac{2(\epsilon - 1)}{(2 - \epsilon)(1 + 3 \epsilon)}}  \right\}.
	\]
	Therefore, 
	\[
	    \begin{aligned}
	    	II_1 & = \left[ \int_{0}^{\overline{r}_1(s)} + \int_{\overline{r}_1(s)}^{\infty} \right] \mathbb{P} \left( \big[ E(r, s) \, \cap \, F(r, s) \big] \mathbb{I} \left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right\} \right) \, \mathrm{d} r \\
	    	& \leq \int_{0}^{\overline{r}_1(s)} \mathbb{P} \left(  \varnothing \right) \, \mathrm{d} r + \int_{\overline{r}_1(s)}^{\infty} \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r} \right) \, \mathrm{d} r \\
	    	& = \int_{\overline{r}_1(s)}^{\infty} \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r} \right) \, \mathrm{d} r.
	    \end{aligned}
	\]
	Then, by sub-Gaussian inequality,
	\[
	    \begin{aligned}
	    	II_1 & \leq \int_{\overline{r}_1(s)}^{\infty} \exp \left( - \frac{s}{2 \sigma_1^2} \left[ \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r} \right]^2\right) \, \mathrm{d} r \\
	    	\alignedoverset{\text{Cauchy inequality}}{\leq}\int_{\overline{r}_1(s)}^{\infty}  \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{3^{\frac{2}{2-\epsilon}} s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r + 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r\\ 
	    	& = \left[ \int_{\overline{r}_1(s)}^{\overline{r}_2(s)} + \int_{\overline{r}_2(s)}^{\infty} \right] \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{3^{\frac{2}{2-\epsilon}} s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r + 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r \\
	    	& =: II_{11} + II_{12},
	    \end{aligned}
	\]
	where
	\[
	    \overline{r}_2(s) := \exp \left\{ \left[ \left( \frac{\lambda(1 - 2\mu)}{1 +2 \lambda} \right)^{\epsilon}\frac{3^{\frac{2}{2 - \epsilon}}}{2 \sigma_1^2}\right]^{\frac{1}{1 + 3 \epsilon}} s^{\frac{2}{(2 - \epsilon)(1 + 3 \epsilon)}}\right\} 
	\]
	is a higher order of $\overline{r}_1(s)$ due to $0 < 2 (\epsilon - 1) < 2$. For $II_{12}$, 
	\[
	    \begin{aligned}
	    	II_{12} & \leq \int_{\overline{r}_2(s)}^{\infty} \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{2 \cdot 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r \\
	    	& = \int_{\overline{r}_2(s)}^{\infty} \exp \left\{ - \frac{s}{8 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2} \right\} \, \mathrm{d} r \, \longrightarrow \, 0
	    \end{aligned}
	\]
	when $s \, \longrightarrow \, \infty$. Thus, there exists a $T_3$ such that, $II_{12} \leq 1$ whenever $s \geq T_3$. Finally, for $II_{11}$, 
	\[
	    \begin{aligned}
	    	II_{11} & \leq \int_{\overline{r}_1(s)}^{{\overline{r}_2(s)}} \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{2 \cdot 3^{\frac{2}{2-\epsilon}} s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r}\right) \, \mathrm{d} r \\
	    	& = \int_{\overline{r}_1(s)}^{{\overline{r}_2(s)}} \exp \left\{  - \frac{\sigma_{\omega}^2}{4 \cdot 3^{\frac{2}{2-\epsilon}}  \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \log^{1 + 3\epsilon} r s^{\frac{- \epsilon}{2 - \epsilon}} \right\} \\
	    	& \leq \overline{r}_2(s) \exp \left\{  - \frac{\sigma_{\omega}^2}{4 \cdot 3^{\frac{2}{2-\epsilon}}  \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} s^{\frac{- \epsilon}{2 - \epsilon}} \big[ \log \overline{r}_2(s) \big]^{1 + 3\epsilon} \right\} \\
	    	& = \exp \left\{ \left[ \left( \frac{\lambda(1 - 2\mu)}{1 +2 \lambda} \right)^{\epsilon}\frac{3^{\frac{2}{2 - \epsilon}}}{2 \sigma_1^2}\right]^{\frac{1}{1 + 3 \epsilon}} s^{\frac{2}{(2 - \epsilon)(1 + 3 \epsilon)}} - \frac{\sigma_{\omega}^2}{4 \cdot 3^{\frac{2}{2-\epsilon}}  \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} s^{\frac{- \epsilon}{2 - \epsilon}} \big[ \log \overline{r}_2(s) \big]^{1 + 3\epsilon}  \right\} \\
	    	& = \exp \left\{C_1 \cdot s^{\frac{2}{(2 - \epsilon)(1 + 3 \epsilon)}} - C_2 \cdot s \right\} \, \longrightarrow \, 0
	    \end{aligned}
	\]
	when $s \, \longrightarrow \, \infty$, where $C_1$ and $C_2$ are some positive constants free of $s$. Again, when $s \, \longrightarrow \, \infty$. Thus, there exists a $T_4$ such that, $II_{11} \leq 1$ whenever $s \geq T_4$.
	\iffalse
	\red{
	Therefore, by sub-Gaussian inequality,
	\[
	    \begin{aligned}
	    	& \mathbb{P} \left( E(r, s) \, \cap \, F(r, s)  \, \cap \,\left\{ \overline{R}_{1, s} - \mu_1 > \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)} \right\} \right)\\
	    	& = \mathbb{P} \\
	    	& \leq 1 + \int_1^{\infty} \exp \left( - \frac{s}{2 \sigma_1^2} \left[ \frac{\sqrt{2}\sigma_{\omega} \log^3 r\left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{1 - \frac{\epsilon}{2}}} {(3s)^{\frac{1}{2 - \epsilon}} \log^{\frac{5 - 3\epsilon}{2}} r + \sqrt{2}\sigma_{\omega} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{ - \frac{\epsilon}{2}} \log^3 r}\right]^2 \right) \, \mathrm{d}r \\
	    \alignedoverset{\text{Cauchy inequality}}{\leq} 1 + \int_1^{\infty} \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{3 s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r + 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r\\
	    	& = 1 + \left[ \int_1^{\overline{r}(s)} +\int_{\overline{r}(s)}^{\infty}\right] \exp \left(  - \frac{\sigma_{\omega}^2}{2 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \frac{s \log^6 r}{3 s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r + 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}}\right) \, \mathrm{d} r \\
	    	& = : 1 + II_{21} + II_{22}:
	    \end{aligned}
	\]
	where $\overline{r}(s)$ is defined as 
	\[
	     \overline{r}(s) := \exp \left\{ \left[ \left( \frac{\lambda(1 - 2\mu)}{1 +2 \lambda} \right)^{\epsilon}\frac{3 s^{\frac{2}{2 - \epsilon}}}{2 \sigma_1^2}\right]^{\frac{1}{1 + 3 \epsilon}}\right\}.
	\]
	For $r \leq \overline{r}(s)$, we have $3 s^{\frac{2}{2-\epsilon}} \log^{5 - 3 \epsilon} r \geq 2 \sigma_{\omega}^2 \log^6 r \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{- \epsilon}$, hence
	\[
	    \begin{aligned}
	    	II_{21} & \leq \int_1^{\overline{r}(s)} \exp \left(  - \frac{\sigma_{\omega}^2}{12 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \log^{1 + 3\epsilon} rs^{\frac{2(1 - \epsilon)}{2 - \epsilon}}\right) \, d r \\
	    	& \leq \int_1^{\infty} \exp \left(  - \frac{\sigma_{\omega}^2}{12 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \log^{1 + 3\epsilon} r s^{\frac{2(1 - \epsilon)}{2 - \epsilon}}\right) \, d r \\
	    	& \overset{\text{take } s = 1}{\leq}  \int_1^{\infty} \exp \left(  - \frac{\sigma_{\omega}^2}{6 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon} \log^{1 } r\right) \, d r \\
	    	& = \int_1^{\infty} r^{-  \frac{\sigma_{\omega}^2}{12 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2 - \epsilon}} \, d r
	    \end{aligned}
	\]
	Therefore, for making $II_{21}$ have a upper bound free of $s$, a sufficient condition is 
	\[
		\frac{\sigma_{\omega}^2}{12 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda} \right]^1 > 1
	\]
	i.e.
	\begin{equation}\label{tuning_sigma}
	    \lambda > \frac{12 \sigma_1^2}{(1 - 2\mu_1) \sigma_{\omega}^2 - 24 \sigma_1^2} \qquad \text{ and } \qquad \sigma_{\omega} > \frac{2 \sqrt{6} \sigma_1}{1 - 2\mu_1}
	\end{equation}	
	For $II_{22}$, we have
	\[
	    \begin{aligned}
	    	II_{22} \leq \int_{\overline{r}(s)}^{\infty} \exp \left\{ - \frac{1}{4 \sigma_1^2} \left[ \frac{\lambda (1 - 2 \mu_1)}{1 + 2\lambda}\right]^{2} s^2\right\} \, dr \longrightarrow 0
	    \end{aligned}
	\]
	when $s \longrightarrow \infty$, {there exist a $T_1$ such that $s \geq T_1$, $II_{22} < 1$.}}
	\fi
\end{itemize}
From the above expression, for proving $a_{k, s}$ is bounded by a constant free of $s$, it remains to show the expectation
\begin{equation}\label{a_k_constant_3}
	{\mathbb{E}} \exp \left\{ \frac{3 (2 \lambda + 1)^2 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} \mathbb{I} \left(  \mu_1 > \frac{1}{2} \right)
\end{equation}

	

We need the following lemma.

\begin{lemma}\label{lem_subGtosubE}
    Suppose $\{X_i\}_{i = 1}^n$ are i.i.d. zero-mean sub-Gaussian with variance proxy $\sigma^2$, and $\overline{X} = \frac{1}{n} \sum_{i = 1}^n X_i$, then
    \[
        {\mathbb{E}} \exp\left\{ \lambda\overline{X}^2 \right\} \leq e^{9 / 8}
       
    \]
    for any $|\lambda| \leq \frac{n}{8 \sigma^2}$.
\end{lemma}
Lemma \ref{lem_subGtosubE} ensures the expectation in \eqref{a_k_constant_3}
   
    
    
   
\[
    \begin{aligned}
        & {\mathbb{E}} \exp \left\{ \frac{3 (2 \lambda + 1)^2 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2}\right\} \mathbb{I} \left( \mu_1 > {1} / {2} \right) \\
        = & {\mathbb{E}} \exp \left\{ \frac{3 (2 \lambda + 1)^2 s }{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2} \times(\overline{R}_{1, s} - \mu_1)^2 \right\} \mathbb{I} \left( \mu_1 > {1} / {2} \right)  \leq e^{9/8} \times\mathbb{I} \left( \mu_1 > {1} / {2} \right)
    \end{aligned}
\]
as long as we take
   
\begin{equation}
\label{cond.1}
    \frac{3 s (1 + 2 \lambda)^2}{2 \sigma_{\omega}^2 \big[ \lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s\big]^2} \leq \frac{s}{8 \sigma_1^2}.
\end{equation}
   
  
The condition \eqref{cond.1} is equal to 
   

   
   
\begin{equation}\label{tuning_condition}
    \frac{\sqrt{3} (1 + 2 \lambda)}{\lambda (2 \mu_1 - 1) + \sqrt{2 \pi} / s } \leq \frac{\sigma_{\omega}}{2 \sigma_1}.
\end{equation}
Hence it is enough to ask $\lambda (2 \mu_1 - 1) > {2 \sqrt{3} \sigma_1} (1 + 2 \lambda) / \sigma_{\omega}$, i.e.
    \[
        \lambda \left[ 2 \mu_1 - 1 - \frac{4 \sqrt{3} \sigma_1}{\sigma_{\omega}}\right] >  \frac{2 \sqrt{3} \sigma_1}{\sigma_{\omega}}.
	\]
Solving the above inequality for positive $\lambda$ and $\sigma_{\omega}$ and we get that 
	\[
	    \lambda > \frac{2 \sqrt{3} \sigma_1}{(2 \mu_1  - 1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {2 \mu_1 - 1}.
	\]
\iffalse
\begin{itemize}
	\item If $\mu_1 > 1 / 2$, the above inequality is equal to 
	\[
	    \lambda (2 \mu_1 - 1) + \frac{\sqrt{2\pi}}{s} > \frac{2 \sqrt{3} \sigma_1}{\sigma_{\omega}} (1 + 2 \lambda).
	\]
	And hence it is enough to ask $\lambda (2 \mu_1 - 1) > {2 \sqrt{3} \sigma_1} (1 + 2 \lambda) / \sigma_{\omega}$, i.e.
    \[
        \lambda \left[ 2 \mu_1 - 1 - \frac{4 \sqrt{3} \sigma_1}{\sigma_{\omega}}\right] >  \frac{2 \sqrt{3} \sigma_1}{\sigma_{\omega}}.
	\]
	Solving the above inequality for positive $\lambda$ and $\sigma_{\omega}$ and we get that 
	\[
	    \lambda > \frac{2 \sqrt{3} \sigma_1}{(2 \mu_1  - 1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {2 \mu_1 - 1}.
	\]
	\item If $\mu_1 < 1 / 2$, the above inequality is equal to 
	\[
	    \lambda (1 - 2\mu_1) + \frac{\sqrt{2\pi}}{s} > \frac{2 \sqrt{3} \sigma_1}{\sigma_{\omega}} (1 + 2 \lambda).
	\]
	Similar to the case of $\mu_1 > 1 / 2$, the solution is 
	\[
	    \lambda > \frac{2 \sqrt{3} \sigma_1}{(1 - 2 \mu_1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {1 - 2 \mu_1}.
	\]
\end{itemize}
\fi

   
  
  
  
  
   
   
  
  
Hence, in the case $\mu_1 \neq \frac{1}{2}$, we can take the tuning condition as 
\begin{equation}\label{tuning_condition_exact}
    \begin{aligned}
    	& \lambda > \frac{2 \sqrt{3} \sigma_1}{(2 \mu_1  - 1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {2 \mu_1 - 1}, \qquad \text{ for } \qquad \mu_1 > \frac{1}{2} \\
    	& \lambda > \frac{\sqrt{3}\sigma_1}{(1 - 2\mu_1) \sigma_{\omega} - 2 \sqrt{3}\sigma_1} \qquad  \text{ with } \qquad \sigma_{\omega} > \frac{2 \sqrt{3} \sigma_1}{1 - 2\mu_1}, \qquad \text{ for } \qquad \mu_1 < \frac{1}{2}.
    \end{aligned}
\end{equation}	

\iffalse
\begin{equation}\label{tuning_condition_exact}
	\begin{aligned}
		 & \text{if the distribution of reward is symmetric:} \\
		 & ~~~~~~~~~~~~~~~~~~~~~ \lambda > \frac{2 \sqrt{3} \sigma_1}{|2 \mu_1 - 1| \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {|2 \mu_1 - 1|}  \\
	    & \text{if the distribution of reward is not symmetric} \\
	    & ~~~~~~~~~~~~~~~~~~~~~ \lambda > \frac{2 \sqrt{3} \sigma_1}{(2 \mu_1 - 1) \sigma_{\omega} - 4 \sqrt{3} \sigma_1} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{4 \sqrt{3} \sigma_1} {2 \mu_1 - 1} \qquad \text{ for } \mu_1 > \frac{1}{2} \\
	    & ~~~~~~~~~~~~~~~~~~~~~ \lambda >  \frac{12 \sigma_1^2}{(1 - 2\mu_1) \sigma_{\omega}^2 - 24 \sigma_1^2} \qquad \text{ with } \qquad \sigma_{\omega} > \frac{2 \sqrt{6} \sigma_1}{1 - 2\mu_1} \qquad \text{ for } \mu_1 < \frac{1}{2}
	\end{aligned}
\end{equation}
\fi
   
   

For the case $\mu_1 = \frac{1}{2}$, we analyze it separately, $Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right)$ will become
\[
    \begin{aligned}
        Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2 \lambda}\right) & = Q_{1, s} \left(\frac{1}{2}\right) \\
        & = \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \left( R_{1, i} - \frac{1}{2} \right) \omega_i > \frac{\lambda}{2} \sum_{i = 1}^s (\omega_i'' - \omega_i')  \right) \\
        & = \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \big( R_{1, i} - \mu_1 \big) \omega_i > \frac{\lambda}{2} \sum_{i = 1}^s (\omega_i'' - \omega_i')  \right) \\
        & = \mathbb{P}_Z \left( Z > \frac{- s(\overline{R}_{1, s} - \mu_1)}{\sigma_{\omega}\sqrt{\sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + \lambda^2 s  / 2}} \right) \\
        & \geq \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 \geq 0 \right) \times\frac{1}{2} + \mathbb{I} \left( \overline{R}_{1, s} - \mu_1 < 0 \right) \times\mathbb{P}_Z \left( Z > \frac{ s(\mu_1 - \overline{R}_{1, s})}{\sigma_{\omega}\sqrt{\sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + \lambda^2 s  / 2}} \right).
    \end{aligned}
\]
where $Z \sim \mathcal{N}(0, 1)$ is the standard Gaussian variable. 
And then by the inequality $\mathbb{P} (Z \geq x) \geq e^{-x^2} / 4$ for any $x > 0$ in \cite{zhang2022sharper}, we will have
\[
     \begin{aligned}
        & \mathbb{P}_Z \left( Z > \frac{s(\mu_1 - \overline{R}_{1, s})}{\sigma_{\omega}\sqrt{\sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + \lambda^2 s  / 2}} \right) \\
        (\text{by Cauchy inequality})\geq & \mathbb{P} \left( Z \geq \frac{s(\mu_1 - \overline{R}_{1, s})}{\sigma_{\omega} \sqrt{s (\mu_1 - \overline{R}_{1, s})^2 + \lambda^2 s  / 2}} \right)  \\
        \geq & \frac{1}{4} \exp\left\{ - \frac{s(\mu_1- \overline{R}_{1, s})^2}{4 (\mu_1- \overline{R}_{1, s})^2 + 2 \lambda^2} \right\}.
     \end{aligned}
\]
Therefore, by the fact \eqref{constant_bound} and Lemma \ref{lem_subGtosubE} again,
\[
    \begin{aligned}
        a_{k, s} & \leq Q_{1, s} \left(\frac{1}{2}\right) \\
        & \leq 2 \mathbb{P} \left( \overline{R}_{1, s} - \mu_1 \geq 0 \right) + 4 {\mathbb{E}} \exp \left\{ \frac{s(\mu_1- \overline{R}_{1, s})^2}{4 (\mu_1- \overline{R}_{1, s})^2 + 2 \lambda^2} \right\} \\
        & \leq 2 + 4 {\mathbb{E}} \exp \left\{ \frac{s(\mu_1- \overline{R}_{1, s})^2}{ 2 \lambda^2} \right\} \\
        \alignedoverset{\text{apply Lemma \ref{lem_subGtosubE}}}{\leq} 2 \left( 1 + 2 e^{9 / 8} \right),
    \end{aligned}
\]
as long as we take
\[
    \frac{s}{2 \lambda^2} \leq \frac{s}{8 \sigma_1^2}
\]
which is equivalent to $\lambda \geq 2 \sigma_1$. 
   

Combine the results for case of $\mu_1 \neq \frac{1}{2}$ and $\mu_1 = \frac{1}{2}$, $a_{k, s}$ has a constant upper bound as follows:
\begin{equation}\label{a_k_constant_final}
    \begin{aligned}
        a_{k, s} & \leq \big( e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \big) \mathbb{I} ( \mu_1 \neq 1 / 2 )  + 2 \big( 1 + 2 e^{9 / 8} \big) \mathbb{I} ( \mu_1 = 1/ 2 ) .
    \end{aligned}
\end{equation}

This step derives the restriction conditions of the tuning parameters in Theorem \ref{thm.MAB}. 







  

  

Next, we decompose $a_{k, s}$ as $a_{k, s} = a_{k, s, 1} + a_{k, s, 2} + a_{k, s, 3}$, and we can focus on these simpler components $a_{k, s, j}$ for $j = 1, 2, 3$. 

Following the idea in \cite{kveton2019garbage}, we are going to try to bound 
\[
    s_{a, j} = \max \{ s: a_{k, s, j} \geq T^{-1}\}, \qquad j = 1, 2, 3
\]
with a constant times $\log T$. Indeed, if we successfully do so, then 
\[
    a_{k} \leq \max_{s \in [T]} a_{k, s} \times s_{a, j} + T^{-1} \times (T - s_{a, j}) \asymp O(1) \times O(\log T) + O(1) = O  (\log T),
\]
which does bound $a_{k} = \sum_{s = 0}^{T - 1} a_{k, s} \lesssim \log T$.
\todohw{I still need to go through the following part. I will get this done tonight and tomorrow.}
For the decomposition, a common approach is using indicators on "good" events to decompose. Denote the shifted version mean reward defined as 
\[
    \overline{R}_{k, s}^* := \frac{\sum_{i = 1}^s R_{k, i} + \lambda s}{s(1 + 2 \lambda)} = \frac{\lambda}{1 + 2 \lambda} + \frac{1}{1 + 2 \lambda} \overline{R}_{k, s}.
\]
Then we can define the following "good" events for $k$-th arm as
\[
    A_{k, s} := \left\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2 \lambda}  > - C_1 \Delta_k \right\}, \qquad A_{k, s}^{\prime} := \left\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2 \lambda}  < C_2 \Delta_k \right\},
\]
and
\[
    G_{k', s} := \left\{ - C_3 \Delta_k \leq \overline{Y}_{k', s} - \overline{R}_{k', s}^* \leq C_3 \Delta_k \right\}.
\]
The definitions of $A_{k, s}$ and $G_{k, s}$ are intuitive: $A_{k, s} \, \cap \, A_{k, s}'$ represent the sample mean does not derivative from the true mean too far; and events on $G_{k', s}$ means $\overline{Y}_{k', s}$ is not too far away from the scaled-shifted sample mean $\overline{R}_{k, s}^*$.

Therefore, by using these good events, we decompose $a_{k, s}$ as the following three parts:
\[
    a_{k, s, 1} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}^c)\Big],
\]
\[
    a_{k, s, 2} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s}^c)\Big], \qquad a_{k, s, 3} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s})\Big].
\]
As we can see, $a_{k, s, 1}$ and $a_{k, s, 2}$ are defined on complement sets of the good events or say "bad" events,
and we will bound them by bounding the probability $\mathbb{P} (A_{1, s}^c)$ and $\mathbb{P} (G_{1, s}^c)$; while $a_{k, s, 3}$ defined on the "good" events, we will need extra steps to bound them, which we will specify later. 

\subsubsection{Bounding $a_{k, s, 1}$ by analyzing $s_{a, 1}$}

We will bound $a_{k, s, 1}$ by bounding the probability of $A_{k, s}^c$. Write
\begin{equation}\label{a_k_1_bound}
    a_{k, s, 1} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) \mathbb{I} (A_{1, s}^c)\Big] \leq {\mathbb{E}} \Big[ T  \mathbb{I}(A_{1, s}^c)\Big] = T \mathbb{P} (A_{1, s}^c) 
\end{equation}
Since the summation of independent sub-Gaussian variables is still sub-Gaussian, \eqref{lem_subGtosubE_1} in Lemma \ref{lem_subGtosubE} gives $\overline{R}_{1, s} - \mu_1 \sim \operatorname{subG} (\sigma^2 /  s)$. And by the concentration inequality of the sub-Gaussian variable,
\[
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \left( \mu_1 - \overline{R}_{1, s} \geq (1 + 2\lambda) C_1 \Delta_k \right) \\
        & \leq \exp \left\{ - \frac{(1 + 2\lambda)^2 C_1^2 \Delta_k^2 s}{2 \sigma_1^2}\right\} \\
        (\text{take} s \geq s_{a, 1} :=  \begin{matrix} \frac{\sigma_1^2}{C_1^2 (1 + 2 \lambda)^2 \Delta_k^2} \times\log T \end{matrix}) & \leq \frac{1}{T^2}
    \end{aligned}
\]
Hence, we obtain that
\[
    a_{k, s, 1} \leq  T \mathbb{P}(A_{1, s}^c) \leq T^{-1}, \qquad \text{ for any } s \geq s_{a, 1} \asymp \log T.
\]

  

\subsubsection{Tackling $a_{k, s, 2}$ with $s_{a, 2}$}

For bounding the probability of $G_{1, s}^c$ conditioning on $\mathcal{H}_{1, s}$, we use the fact that $\omega_i, \omega_i', \omega_i''$ is symmetric about $1$: 
\[
    \begin{aligned}
        \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s}) & = \mathbb{P} \big( |\overline{Y}_{k', s} - \overline{R}^*_{k', s}| > C_3 \Delta_k \mid \mathcal{H}_{1, s}\big)\\
        & = 2 \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) \omega_i + \lambda \sum_{i = 1}^s (2 R_{1, i} \omega_i - \overline{R}_{1, s} \omega_i' - \overline{R}_{1, s} \omega_i'') 
        \\
        & \qquad \qquad  + \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s (\omega_i' - \omega''_i) - C_3 (1 + 2\lambda) \Delta_k \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] > 0\Bigg\}.
    \end{aligned}
\]
To analyze the above probability, define the following functions of $\{ R_{1,i}\}_{i = 1}^s$:
\[
   f \big(\{ R_{1,i}\}_{i = 1}^s\big) := f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) + f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) - f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big)
\]
with 
\[
    f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) := \sum_{i = 1}^s \big[ (R_{1, i} - \overline{R}_{1, s}) + \lambda (2 \overline{R}_{1, i} - 1) - (1 + 2 \lambda ) C_3 \Delta_k  \big] \omega_i,
\]
   
\[
    f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) := \sum_{i = 1}^s \big[ \lambda (\lambda + 1  - \overline{R}_{1, s}) - (1 + 2 \lambda ) C_3 \Delta_k  \big] \omega_i',
\]
   
\[
    f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big) = \sum_{i = 1}^s \big[ \lambda (\overline{R}_{1, s} + \lambda) + (1 + 2 \lambda ) C_3 \Delta_k  \big] \omega_i''.
\]
   
Then we can write $\mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s}) = 2 \mathbb{P}_{\omega, \omega', \omega'} (f_1 + f_2 - f_3 > 0)$. Note that $f_1, f_2$, and $f_3$ are mutually independent. The expectation is
\[
    {\mathbb{E}} (f_1 + f_2 - f_3) = - 3 C_3 (1 + 2 \lambda) \Delta_k s,
\] 
and the variance is
\begin{equation}\label{bound_var_f}
    \begin{aligned}
        & \operatorname{var}(f_1 + f_2 - f_3) \\
        = & \sigma_{\omega}^2 \Bigg[ \overbrace{(1 + 2 \lambda)^2 \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s})^2 + 2 s \lambda (1 + 2 \lambda) (2 \overline{R}_{1, s} - 1 ) (\overline{R}_{1, i} - \overline{R}_{1, s}) + 2 s \lambda^2 \big(3 \overline{R}_{1, s}^2 - 3 \overline{R}_{1, s} + \lambda^2 + \lambda + 1 \big)}^{\text{the variance coming from the difference between $\overline{Y}_{k, s}$ and $\overline{R}_{k, s}$.}} \\
        & \qquad \qquad \qquad + \underbrace{3 s (1 + 2\lambda)^2 C_3^2 \Delta_k^2}_{\text{the maximal variance expansion allowed on the event $G_{1, k}^c$ (the cross term cancelling out).}} \Bigg] \\
        & = \sigma_{\omega}^2 \bigg[ (1 + 2 \lambda)^2 \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s})^2 +  2 s \lambda^2 \big(3 \overline{R}_{1, s}^2 - 3 \overline{R}_{1, s} + \lambda^2 + \lambda + 1 \big) + 3 s (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \bigg] \\
        & =: \sigma_{\omega}^2 \bigg[ \mathbb{G}_1 (s, \lambda) + \mathbb{G}_2 (s, \lambda) + g_3 (s, \lambda) \bigg]
    \end{aligned}
\end{equation}
where we define
\[
    \mathbb{G}_1(s, \lambda) := (1 + 2 \lambda)^2 \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s})^2, \qquad \mathbb{G}_2 (s, \lambda) := 6 s \lambda^2 \overline{R}_{1, s} (\overline{R}_{1, s} - 1),
\]
and
\[
    g_3(s, \lambda) = s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2  \big].
\]
\iffalse
\[
    \begin{aligned}
        & \operatorname{var}(f_1 + f_2 - f_3) = \operatorname{var} f_1 + \operatorname{var} f_2 + \operatorname{var} f_3 \\
        = & \bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s})^2 + 4 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + 4 \lambda \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) (R_{1, i} - \mu_1) \\
        & \qquad + 2 \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big] \sum_{i = 1}^s (R_{1,i} - \overline{R}_{1, s}) \\
        & \qquad + 4 \lambda \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big] \sum_{ i =1}^s (R_{1, i} - \mu_1) + s \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big]^2 \bigg\} \\
        & + \lambda^2 \bigg\{ s^2 (\overline{R}_{1, s} - \mu_1)^2 + 2 \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big] s (\overline{R}_{1, s} - \mu_1) +  s \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \bigg\} \\
        & + \lambda^2 \bigg\{ s^2 (\overline{R}_{1, s} - \mu_1)^2 + 2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] s (\overline{R}_{1, s} - \mu_1) + s \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \bigg\} \\
        = & (1 + 4 \lambda^2 + 4 \lambda) \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + (-s - 4 \lambda s + \lambda^2 s^2 + \lambda^2 s^2) (\overline{R}_{1, s} - \mu_1)^2 + \Big\{ 4 \lambda \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big] \\
        & + 2 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] + 2 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big\} \sum_{i = 1}^s (R_{1, s} - \mu_1) \\
        & + \Big\{ s \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big]^2 + s \lambda^2 \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \\
        & + s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \Big\} \\
        =: & g_1(\lambda) \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + g_2(s, \lambda) (\overline{R}_{1, s} - \mu_1)^2 + g_3(\lambda) \sum_{i = 1}^s (R_{i, s} - \mu_1) + g_4(s, \lambda) \\
        =: & \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + \mathbb{G}_{3}(s, \lambda) + g_4(s, \lambda).
    \end{aligned}
\]
\fi

The $\mathbb{G}_j$ for $j = 1, 2$ and $g_3$ have relatively complex expressions, so we want to give some simpler upper bounds of them. We calculate it as
\begin{equation}\label{G_function_bound}
    \begin{aligned}
        & \mathbb{G}_1 (s, \lambda) + \mathbb{G}_2 (s, \lambda) + g_3 (s, \lambda) \\
        = & (1 + 2 \lambda)^2 \bigg[ \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 - s (\overline{R}_{1, s} - \mu_1)^2\bigg] \\
        & \qquad + 6 s \lambda^2 \Big[ (\overline{R}_{1, s} - \mu_1)^2 + 2 (\mu_1 - 1) (\overline{R}_{1,  s} - \mu_1) + \mu_1 (\mu_1 - 1)\Big] + s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) \\
        & \qquad + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2  \big] \\
        = & \bigg[ (1 + 2 \lambda)^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + s (2 \lambda^2 - 4 \lambda - 1) (\overline{R}_{1, s} - \mu_1)^2 - 2s (1 - \mu_1) (\overline{R}_{1, s} - \mu_1) \bigg] \\
        & \qquad \qquad + s \big[ - 6 \lambda^2 \mu_1 (1 - \mu_1) + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big] \\
        \overset{\text{dropping the negative terms}}{\leq} & \bigg[ (1 + 2 \lambda)^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + s (2 \lambda^2 - 4 \lambda - 1) (\overline{R}_{1, s} - \mu_1)^2  \bigg]  \\
        & + s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big] \\
        \overset{\text{by Cauchy inequality}}{\leq} & \big[ (1 + 2 \lambda)^2 + (2 \lambda^2 - 4 \lambda - 1)  \big] \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 +  s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big] \\
        = & 6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big]. 
    \end{aligned}
\end{equation}
As one can see, in this step, we bound $\mathbb{G}_1 (s, \lambda) + \mathbb{G}_2 (s, \lambda) + g_3 (s, \lambda)$ by a sub-Exponential variable plus a strictly positive constant.
\iffalse
\begin{equation}\label{G_function_bound}
    \begin{aligned}
        & \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + \mathbb{G}_{3}(s, \lambda) + g_4(s, \lambda) \\
        = & s^2 g_1(\lambda) (\overline{R}_{1, s} - \mu_1)^2 + g_2(s, \lambda) (\overline{R}_{1, s} - \mu_1)^2  + s g_3(\lambda) (\overline{R}_{1, s} - \mu_1) + g_4(s, \lambda) \\
        = & \big[ s^2 g_1(\lambda) + g_2(s, \lambda)\big](\overline{R}_{1, s} - \mu_1)^2 + s g_3(\lambda) (\overline{R}_{1, s} - \mu_1) + g_4(s, \lambda) \\
        \leq & \Big\{ s^2(2 \lambda + 1)^2 + (2 \lambda^2 s^2 - 4 \lambda s - 1)\Big\} (\overline{R}_{1, s} - \mu_1)^2 \\
        & \qquad + \Big\{ s \times4 \lambda \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + \lambda) + \lambda (\lambda + \mu_1) + C_3 \lambda (1 + 2 \lambda) \Delta_k \big] \Big\} (\overline{R}_{1, s} - \mu_1) \\
        & \qquad + s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \\
        & \qquad + s \lambda^2 \Big\{ \big[ (2 \mu_1 - 1) - C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \Big\} \\
        \leq & 2 s^2 (6 \lambda^2 + 4 \lambda + 1) (\overline{R}_{1, s} - \mu_1)^2 + 4 s \lambda \big[ (2 C_3 \Delta_k + 1) \lambda^2 + (3 \mu_1 - 1 - C_3 + C_3 \Delta_k) \lambda - C_3 \big] (\overline{R}_{1, s} - \mu_1) \\
        & \qquad + s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \\
        & \qquad + s \lambda^2 \Big\{ \big[ (2 \mu_1 - 1) - C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \Big\} \\
        \leq & 2s^2 (4 \lambda^2 + 4 \lambda + 1) (\overline{R}_{1, s} - \mu_1)^2  + 2 s \lambda (2 \lambda + 1) \times\lambda \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \sqrt{s} (\overline{R}_{1, s} - \mu_1) \\
        & \qquad + s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + g_5(s, \lambda)\\
        \leq & 2 s \Big[ \sqrt{s} (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1) +  \lambda \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big]^2 + g_5(s, \lambda),
    \end{aligned}
\end{equation}
where the last two $\leq$ use the fact that $s \geq 1$ and $\mu_1, C_3 \in (0, 1]$. And furthermore, $g_5(s, \lambda) = s \lambda^2 \Big\{ \big[ (2 \mu_1 - 1) - C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \Big\}$ is a strict positive function for at least $\Delta_{k_1}, \ldots, \Delta_{k_{K - 3}}$, since
\[
    \frac{C_3 \Delta_k}{2(\mu_1 - C_3 \Delta_k) - 1} = \frac{1 - \mu_1 - C_3 \Delta_k}{2 C_3 \Delta_k - 1}.
\]
is a quadratic equation for $\Delta_k$ and cannot hold for all $k \in [K] / \{ 1\}$. And for $g_5(s, \lambda)$, it can be bounded by
\[
    \begin{aligned}
        g_5(s, \lambda) & \leq s \lambda^2 \Big( \big[ 1 + C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ 1 + C_3 (1 + 2 \lambda) \Delta_k + (1 + \lambda) \big]^2 \Big) \\
        & \leq 2 s \lambda^2 \left[ 1 + C_3^2 (\lambda^{-1} + 2)^2 \Delta_k^2 + (2 + \lambda)^2 + C_3^2 (1 + 2 \lambda)^2 \Delta_k^2\right] \\
        & \leq 2 s \big[ \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 \big].
    \end{aligned}
\]


   
       
       
       
       
       
   
\fi

For the standard Gaussian variable $Z$, we have $\mathbb{P} (Z \geq x)\leq \exp \{ - {x^2} / {2}\}$ for $x > 0$. Then
\[
    \begin{aligned}
        \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s}) & = 2 \mathbb{P} \left(\frac{f_1 + f_2 - f_3 - {\mathbb{E}} (f_1 + f_2 - f_3)}{\sqrt{\operatorname{var} (f_1 + f_2 - f_3)}} > - \frac{{\mathbb{E}} (f_1 + f_2 - f_3)}{\sqrt{\operatorname{var} (f_1 + f_2 - f_3)}} \right) \\
        & \leq  2\exp \left\{ - \frac{{\mathbb{E}}^2 (f_1 + f_2 - f_3)}{2 \operatorname{var} (f_1 + f_2 - f_3)} \right\} \\
        (\text{by \eqref{bound_var_f}} )& \leq 2 \exp \left\{ - \frac{9 C_3^2 (1 + 2 \lambda)^2 \Delta_k^2 s^2 }{2 \sigma_{\omega}^2 \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + g_3(s, \lambda) \big]}\right\}.
    \end{aligned}
\]
Therefore, $a_{k, s, 2}$ will be bounded by 
\[
    \begin{aligned}
        a_{k, s, 2} & \leq {\mathbb{E}} \Big[ T  \mathbb{I}(G_{1, s}^c)\Big] =  T {\mathbb{E}} \Big[ \mathbb{P}(G_{1, s}^c \mid \mathcal{H}_{1, s}) \Big] \\
        & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{9 C_3^2 (1 + 2 \lambda)^2 \Delta_k^2 s^2 }{2\sigma_{\omega}^2 \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + g_3(s, \lambda) \big]}\right\}.
    \end{aligned}
\]
Recall $R_{1, i} - \mu_1 \sim \operatorname{subG}(\sigma_1^2)$, and then $(R_{1, i} - \mu_1)^2 - \operatorname{var} (R_{1, i}) \sim \operatorname{subE}(8 \sqrt{2} \sigma_1^2)$, hence by Corollary 4.2 (a) in \cite{zhang2020concentration},
\[
    \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 - s \operatorname{var} (R_{1, i}) \, \sim \, \operatorname{subE} (8 \sqrt{2 s} \sigma_1^2).
\]
Therefore, 
\begin{equation}\label{G_function_bound_2}
    \begin{aligned}
        2 \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + g_3(s, \lambda) \big] & \overset{\text{by \eqref{G_function_bound}}}{\leq} 12 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big] \\
        & \sim 12 \lambda^2 \operatorname{subE} (8 \sqrt{2 s} \sigma_1^2) + 12 \lambda s \operatorname{var} (R_{1, 1}) +  s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) \\
        & ~~~~~~~~~~~~~~ + 6  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big] \\
        & \overset{\text{by } \operatorname{var}(R_{1, 1}) \leq \sigma_1^2}{\leq} 12 \lambda^2 \operatorname{subE} (8 \sqrt{2 s} \sigma_1^2) \\
        & ~~~~~~~~~~~~~ + s \big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big].
    \end{aligned}
\end{equation}
Here we use the $\operatorname{subE}(\lambda)$ to represent a random variable $Z \sim \operatorname{subE}(\lambda)$. And the distribution notation in the inequalities following will also have a similar meaning.

\iffalse
For tackling the expectation above, we first note that $\sqrt{s} (\overline{R}_{1, s} - \mu_1) \sim \operatorname{subG} (\sigma_1^2)$, then $\sqrt{s} (2\lambda + 1) (\overline{R}_{1, s} - \mu_1) \sim \operatorname{subG} \big((2\lambda + 1)^2\sigma_1^2 \big)$, and
hence
\[
    \begin{aligned}
        & \Big[ \sqrt{s} (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1) + \lambda \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big]^2 \\
        & \quad - \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 - (2 \lambda + 1 )^2 \operatorname{var} (R_{1,s}) \,  \sim \, \operatorname{subE} \big(8\sqrt{2}(2\lambda + 1)^2\sigma_1^2, 8(2\lambda + 1)^2\sigma_1^2 \big),
    \end{aligned}
\]
\fi
Then applying Lemma \ref{lem_subE_ineq} in \eqref{G_function_bound} will furthermore give
\begin{equation}\label{a_2_bound}
    \begin{aligned}
        a_{k, s, 2} & \leq 2 T  {\mathbb{E}} \exp \left\{ - \frac{9 C_3^2 (1 + 2\lambda)^2 \Delta_k^2 s^2 }{2 \sigma_{\omega}^2 \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + g_3 (s, \lambda) \big]}\right\} \\
        & \overset{\text{by \eqref{G_function_bound_2}}}{\leq} 2 T {\mathbb{E}}\exp \left\{ - \frac{9 \sigma_{\omega}^{-2} C_3^2 (1 + 2\lambda)^2 \Delta_k^2 s^2 }{12 \lambda^2 \operatorname{subE} (8 \sqrt{2 s} \sigma_1^2) + s \big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big]}\right\} \\
        & \overset{\text{by } a \operatorname{subE} (\lambda) = \operatorname{subE} (a^2 \lambda)}{=} 2 T {\mathbb{E}}\exp \left\{ - \frac{9 \sigma_{\omega}^{-2} C_3^2 (1 + 2\lambda)^2 \Delta_k^2 s }{ \operatorname{subE} ( 1152 \sqrt{2} \lambda^4 \sigma_1^2) + \big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big]}\right\} \\
        & \overset{\text{Lemma \ref{lem_subE_ineq}}}{\leq} 2 T \Big[ 1 + 9 \sigma_{\omega}^{-2} C_3^2 (1 + 2\lambda)^2 \Delta_k^2 s ( h_1 + h_2) \Big] \exp \left\{  - \frac{9 \sigma_{\omega}^{-2} C_3^2 (1 + 2\lambda)^2 \Delta_k^2 s }{\big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big]}\right\} \\
        & \overset{\text{by } s \leq T}{\leq} 2 T^2 \Big[ 1 + 9 \sigma_{\omega}^{-2} C_3^2 (1 + 2\lambda)^2 \Delta_k^2 ( h_1 + h_2) \Big] \exp \left\{  - \frac{9 \sigma_{\omega}^{-2} C_3^2 (1 + 2\lambda)^2 \Delta_k^2 s }{\big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big]}\right\},
    \end{aligned}
\end{equation}
\iffalse
\begin{equation}\label{a_2_bound}
    \begin{aligned}
        a_{k, s, 2} & \leq 2 T  {\mathbb{E}} \exp \left\{ - \frac{9 C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s^2 }{2 \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + g_3 (s, \lambda) \big]}\right\} \\
        & \leq 2 T \times \\
        & {\mathbb{E}} \exp \left\{ - \frac{9 C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s}{4 \big[\operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2) + 2\lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + (2 \lambda + 1 )^2 \operatorname{var} (R_{1,s})\big] + 2 s^{-1}g_5(s, \lambda)}\right\} \\
        & \leq 2 T \exp \left\{ - \frac{9 C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s}{8 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + 4 (2 \lambda + 1 )^2 \operatorname{var} (R_{1,s}) + 2 s^{-1} g_5(s, \lambda)}\right\} \\
        & \qquad \times \Big[ 1 + C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s (h_1 + h_2) \Big]\\
        & \leq 2 T^2 \Big[ 1 + C_3^2 (1 + 2\lambda)^4 \Delta_k^2 (h_1 + h_2) \Big] \\
        & \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s}{8 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + 4 (2 \lambda + 1 )^2 \sigma_1^2 + 4 \big[ \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 \big]}\right\}.
    \end{aligned}
\end{equation}
\fi
where $h_1$ and $h_2$ are finite and free of $s$ by following Lemma \ref{lem_subE_ineq}. 

\iffalse
\begin{lemma}[Need to be corrected!]\label{lem_subE_ineq}
    Suppose zero-mean random variable $X \sim \operatorname{subE}(\lambda, \alpha)$, and positive constants $a, b, c$ satisfying $a X + b > 0$,
    then
    \[
        {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\}  \leq e^{- c / b} \big[1 + c h_1(a, b, \alpha) + c h_2(a, b, \lambda) \big]
    \]
    where $h_1$ and $h_2$ are finite positive functions.
\end{lemma}

\begin{proof}
    Note that differentiable $f$ is positive, we have $0 \leq {\mathbb{E}} f(X) \leq f(0) + \int_0^{\infty} \big[ f'(u) - f'(-u) \big]\mathbb{P} (|X| > u) \, du$, then 
    \[ 
        \begin{aligned}
            & {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\} \\
            \leq & e^{-c / b} + \int_0^{\infty} \bigg[ \exp \left\{ - \frac{c}{a u + b}\right\}  \frac{ca }{ (a u + b)^2} - \exp \left\{ - \frac{c}{ - a u + b}\right\}  \frac{ca }{ (- a u + b)^2} \bigg] \, \mathbb{P} (|X| > u) \, du \\
            \leq & \int_0^{\infty} \bigg[ \exp \left\{ - \frac{c}{a u + b}\right\}  \frac{ca }{ (a u + b)^2} - \exp \left\{\frac{c}{ a u - b}\right\}  \frac{ca }{ (a u - b)^2} \bigg] \, \exp \left\{ - \frac{1}{2} \left( \frac{u^2}{\lambda^2} \wedge\frac{u}{\alpha} \right) \right\} \, du \\
            = & \int_0^{\infty} \exp \left\{ - \frac{c}{a u + b}\right\} \bigg[   \frac{ca }{ (a u + b)^2} - \exp \left\{\frac{b + au}{ b - au}\right\}  \frac{ca }{ (a u - b)^2} \bigg] \, \exp \left\{ - \frac{1}{2} \left( \frac{u^2}{\lambda^2} \wedge\frac{u}{\alpha} \right) \right\} \, du \\
            \leq & \exp (-c / b) \int_0^{\infty} \Bigg[ \frac{ca}{(a u + b)^2} \exp \left\{ - \frac{u}{2 \alpha}\right\} + \frac{ca}{(a u - b)^2} \exp \left\{ - \frac{1}{2}\frac{u^2}{ \lambda^2}  \wedge \frac{u}{ \alpha} + \frac{b + au}{b - au}\right\} \Bigg] \, du \\
            =: &  \exp (-c / b) \big[ I_1 + I_2 \big].
        \end{aligned}
    \]
    Denote $\Gamma(\cdot, \cdot)$ as incomplete gamma function, then one can calculate that
    \[
        I_1 = \frac{c}{b} - \frac{c e^{\frac{b}{2 a \alpha}}\Gamma(0, \frac{b}{2 a \alpha})}{2 a \alpha} =: c h_1 (a, b, \alpha),
    \]
    which is finite as long as $a, \alpha$ is positive.
    For calculating $I_2$, we write $I_2$ as
    \[
        \begin{aligned}
            I_2 = & c a \int_{0}^{\lambda^2 / \alpha} \frac{1}{(a u - b)^2} \exp \left\{ - \frac{u^2}{2 \lambda^2}  + \frac{b + a u}{b - au}\right\} \, du  + \int_{\lambda^2 / \alpha}^{\infty}  \frac{1}{(a u - b)^2} \exp \left\{ - \frac{u}{2 \alpha} + \frac{b + a u}{b - au}\right\} \, du \\
            =: & ca ( I_{21} + I_{22}).
        \end{aligned}
    \]
    As $I_{21}$ and $I_{22}$ are both free of $c$, we only need to care its irregular point $\frac{b}{a}$ and $\infty$. We separate it as the following two cases:
    \begin{itemize}
        \item[(i)] If $\frac{b}{a} \leq \frac{\lambda^2}{\alpha}$, then 
        \[
            \lim_{u \rightarrow \frac{b}{a}} (a u - b)^{1 / 2}\frac{1}{(a u - b)^2} \exp \left\{ - \frac{u^2}{2 \lambda^2} + \frac{b + au}{b - au}\right\} = 0,
        \]
        and 
        \[
            \lim_{u \rightarrow \infty} u^{3 / 2}\exp \left\{ - \frac{u}{2 \alpha} + \frac{b + au}{b - au}\right\} = 0.
        \]
        \item[(ii)] If $\frac{b}{a} \geq \frac{\lambda^2}{\alpha}$, then $I_{21}$ will be strictly finite. And for $I_{22}$, it will be
        \[
            \lim_{u \rightarrow \frac{b}{a}} (a u - b)^{1 / 2}\frac{1}{(a u - b)^2} \exp \left\{ - \frac{u}{2 \alpha} + \frac{b + au}{b - au}\right\} = \lim_{u \rightarrow \infty},
        \]
        
    \end{itemize}
    Hence, $\exists \, I_2 \in \mathbb{R}$, and then
    \[
        \exists \, I_2 = c \int_0^{\infty} \frac{a}{(a u - b)^2} \exp \left\{ - \frac{u}{2 \alpha} + \frac{b + au}{b - au}\right\} =: c h_2(a, b, \lambda) \in c \times (0, \infty)
    \]
    with $h_1$ and $h_2$ are finite positive functions.
\end{proof}
\fi


\begin{lemma}[sub-Exponential concentration]\label{lem_subE_ineq}
    Suppose zero-mean random variable $X \sim \operatorname{subE}(\lambda, \alpha)$, and positive constant $a, b, c$ satisfies $a X + b > 0$. If $b \leq 2 c a\lambda^2 \alpha^2 \wedge \sqrt{2 c a\alpha}$, then
    \[
        {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\}  \leq e^{- c / b} \big[1 + c h_1(a, b, \alpha, \lambda) + c h_2(a, b, \alpha) \big] := e^{- c / b} [1 + ch_1 + ch_2],
    \]
    where $h_1$ and $h_2$ are finite positive functions free of $c$.
\end{lemma}

Let us take
\[
    \begin{aligned}
        s \geq s_{a, 2} := \frac{ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_3^2 \Delta_k^2}{9 \sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2} \log \Big[ 2 T^3 \big( 1 + 9 \sigma_{\omega}^{-2} C_3^2 (1 + 2\lambda)^2 \Delta_k^2 ( h_1 + h_2) \big) \Big]
    \end{aligned}
\]
   
       
       
   
in the last step of \eqref{a_2_bound}, we will obtain $a_{k, s, 2} \leq T^{-1}$.

\subsubsection{Tackling $a_{k, s, 3}$ with $s_{a, 3}$}

In the previous section, we control the probability on bad events to bound $a_{k, s, 1}$ and $a_{k, s, 2}$. But since $a_{k, s, 3}$ is defined on good events, we need a different technique to deal with $a_{k, s, 3}$.

Different from dealing with $a_{k, s, 1}$ and $a_{k, s, 2}$, in which we control the probability of the bad events with $O(T^{-2})$,
$a_{k, s, 3}$ is defined on good events. Hence, 
We need a special technique to deal with $a_{k, s, 3}$ . Note that 
\begin{equation}\label{a_k_3_sw}
    \begin{aligned}
        \{N_{1, s}(\tau_{k})<T^{-1}\} & \supseteq \{N_{1, s}(\tau_{k}) < (T^{2}-1)^{-1}\} = \big\{Q_{1, s}\left(\tau_{k}\right)^{-1}<1+\left(T^{2}-1\right)^{-1}\big\} \\
        &=\big\{Q_{1, s}(\tau_{k})^{-1} < (1-T^{-2})^{-1}\big\} = \big\{\left[1-Q_{1, s}(\tau_{k})\right]<T^{-2}\big\},
    \end{aligned}
\end{equation}
and $a_{k, s, 3} \leq {\mathbb{E}} \big[ N_{1, s} (\tau_k) 1(A_{1, s} \cap G_{1, s}) \big]$. 
Thus, similar to 
\cite{wang2020residual}, it is sufficient to find $s \geq s_{a,3}$ satisfying $\big\{[1-Q_{1,s}(\tau_{k})]<T^{-2}\big\}$ on the event $\{A_{1, s} \cap G_{1, s}\}$.

We write 
\[
    \big[1-Q_{1, s}(\tau_{k})\big]  \mathbb{I} (A_{1, s})\mathbb{I}  (G_{1,s}) = \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Gamma_1 \, | \, \mathcal{H}_{1, s} \big)  \mathbb{I} (A_{1, s} \, \cap \, G_{1, s})
\]
with $\Gamma_1 := \Gamma_1(\tau_1) = \tau_1 - \overline{R}_{1, s}^*$ measuring the difference between $\overline{Y}_{1, s}$ and $\overline{R}_{1,s}^*$. And on the event $A_{1, s} \, \cap \, G_{1, s}$, $\Gamma_1$ can be bounded in an interval.
To see this, we write 
\[
    A_{1, s} = \bigg\{ \overline{R}_{1, s}^* - \frac{\mu_1 + \lambda}{1 + 2\lambda} > - C_1 \Delta_k \bigg\} = \bigg\{ \Gamma_1 < C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} + \tau_1 \bigg\} 
\]
and
\[
    \begin{aligned}
        G_{1, s} = \big\{ -C_3 \Delta_k \leq \overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq C_3 \Delta_k \big\} & = \big\{ \tau_1 -C_3 \Delta_k \leq \overline{Y}_{1, s} + \Gamma_1 \leq \tau_1 + C_3 \Delta_k \big\} \\
        & = \big\{ \tau_1 - C_3 \Delta_k - \overline{Y}_{1, s} \leq \Gamma_1 \leq \tau_1 + C_3 \Delta_k - \overline{Y}_{1, s}\big\} \\
        (\text{by plugging in the inequality for } \overline{Y}_{1, s})& \subseteq \big\{ \tau_1 - 2 C_3 \Delta_k - \overline{R}_{1, s}^* \leq \Gamma_1 \leq \tau_1 + 2 C_3 \Delta_k - \overline{R}_{1, s}^* \big\},
       
    \end{aligned}
\]
then
\[
    \begin{aligned}
        A_{1, s} \, \cap \, G_{1, s} \\
       
        (\text{by plugging the inequality for $\overline{R}_{1, s}^*$ in } A_{1, s})\subseteq & \bigg\{ \tau_1 -  2 C_3 \Delta_k - 1 \leq \Gamma_1 < C_1 \Delta_k + \tau_1 - \frac{\mu_1}{2}\bigg\} \\
        (\text{take } C_1 \geq 2 C_3 ) \subseteq & \bigg\{ \tau_1 -  C_1 \Delta_k - 1 \leq \Gamma_1 < C_1 \Delta_k + \tau_1 - \frac{\mu_1}{2}\bigg\}.
    \end{aligned}
\]
Since $\tau_1 - C_1 \Delta_k - 1 < 0$, to control $\Gamma_1$ within a small interval, we add conditions: 
\begin{equation}\label{requirement_1}
     \Delta_k + \tau_1 - \frac{\mu_1}{2} \leq 0 \qquad \text{ together with }  \qquad C_1 \geq 2C_3.
\end{equation}
Hence, we obtain the range of the positive quantity $-\Gamma_1$ lies in
\[
    C_1 \Delta_k = \frac{\mu_1 }{2} - \tau_1 - C_1 \Delta_k <  - \Gamma_1 \leq 1 + C_1 \Delta_k  - \tau_k = 1 - \frac{\mu_1}{2} + 3 C_1 \Delta_k < 4.
\]
Now fix
\begin{equation}\label{requirement_2}
    \tau_1 = \frac{\mu_1}{2} - \Delta_k,
\end{equation}
we get $ (- \Gamma_1) \in (C_1 \Delta_k, 4) \subset (0, + \infty)$.

Returning to the quantity we care about, $ \big[ 1 - Q_{1,s} (\tau_k)\big] 1(A_{1, s}) 1(G_{1, s})$, we write it as 
\[
    \begin{aligned}
        & \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I} (A_{1, s}) \mathbb{I} (G_{1, s}) \\
        = & \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Gamma_1 \, | \, \mathcal{H}_{1, s} \big)  \mathbb{I} (A_{1, s} \, \cap \, G_{1, s})\\
        = & {\mathbb{E}} \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) \omega_i + \lambda \sum_{i = 1}^s (2 R_{1, i} \omega_i - \overline{R}_{1, s} \omega_i' - \overline{R}_{1, s} \omega_i'') 
        \\
        & \qquad \qquad + \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s (\omega_i' - \omega''_i) - \Gamma_1 (1 + 2\lambda) \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] < 0\Bigg\} \mathbb{I} (A_{1, s} \, \cap \, G_{1, s}).
    \end{aligned}
\]
Similar to the technique we use to bound conditional probability $\mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s})$, we just need to replace $C_3 \Delta_k$ by $(-\Gamma_1)$ in \eqref{a_2_bound}. We get that 
\[
    \begin{aligned}
         & \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I} (A_{1, s}) \mathbb{I}(G_{1, s}) \\
         \overset{\text{apply steps in  \eqref{G_function_bound_2}}}{\leq} & {\mathbb{E}} \exp \left\{ - \frac{9 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 (- \Gamma_1)^2 s^2 }{12 \lambda^2 \operatorname{subE} (8 \sqrt{2 s} \sigma_1^2) + s \big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 (- \Gamma_1)^2  \big]}\right\} \\
         \overset{\text{by the bound of } \Gamma_1}{\leq} & {\mathbb{E}} \exp \left\{ - \frac{144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 s }{12 \lambda^2 \operatorname{subE} (8 \sqrt{2} \sigma_1^2) + \big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_1^2 \Delta_k^2  \big]}\right\}\\
         \overset{\text{Lemma \ref{lem_subE_ineq}}}{\leq} & \Big[ 1 + 144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2  s ( h_1^* + h_2^*) \Big] \exp \left\{  - \frac{144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2  s }{\big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_1^2 \Delta_k^2 \big]}\right\} \\
         \overset{s \leq T}{\leq} & T \Big[ 1 + 144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 ( h_1^* + h_2^*) \Big] \exp \left\{  - \frac{144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2  s }{\big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_1^2 \Delta_k^2 \big]}\right\}.
    \end{aligned}
\]
\iffalse
\begin{footnotesize}
\[
    \begin{aligned}
        & \big[ 1 - Q_{1,s} (\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \\
        \leq & {\mathbb{E}} \exp \left\{ - \frac{\Gamma_1^2 (1 + 2\lambda)^4 s}{4 (\operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2) + 2\lambda^2 \big[ (\lambda + \mu_1) + (1 + 2 \lambda) \Gamma_1 \big]^2 + (2 \lambda + 1 )^2 \sigma_1^2 + 2 \lambda^2(\sqrt{5} + \lambda)^2 + 2 (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 ) }\right\} \\
        \leq & {\mathbb{E}} \exp \left\{ - \frac{C_1^2 \Delta_k^2 (1 + 2\lambda)^4 s}{4 (\operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2) + 2\lambda^2 \big[ (\lambda + \mu_1) -  (1 + 2 \lambda) 4 \big]^2 + (2 \lambda + 1 )^2 \sigma_1^2 + \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2) }\right\} \\
        \leq &  T  \exp \left\{ - \frac{C_1^2 (1 + 2\lambda)^4 \Delta_k^2 s}{8 \lambda^2 (7 \lambda + 4 - \mu_1)^2 + 4 (2 \lambda + 1 )^2 \sigma_1^2 + 4 [ \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2]}\right\} \\
        & \times \big[1 + C_1^2 \Delta_k^2 (1 + 2\lambda)^4  (h_1^* + h_2^*)\big] 
    \end{aligned}
\]
\end{footnotesize}
\fi
Therefore, to find $s$ such that $[1 - Q_{1, s} (\tau_k)] < T^{-2}$ on $\{ A_{1, s} \, \cap \, G_{1, s}\}$, let
\[
   \begin{aligned}
       s \geq s_{a, 3} := \frac{ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_1^2 \Delta_k^2}{144 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} \log \Big[ T^3 \big( 1 + 144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 ( h_1^* + h_2^*) \big) \Big],
   \end{aligned}
\]
where $h_1^*$ and $h_2^*$ free of $s$ are $h_1(\cdot, \cdot, \cdot)$ and $h_2(\cdot, \cdot, \cdot)$ in Lemma \ref{lem_subE_ineq} by replacing the corresponding parameters. By the argument in \eqref{a_k_3_sw}, we have $a_{k, s, 3} \leq T^{-1}$ whenever $s \geq s_{a,3}$ defined above.

Now, combining all the results in Section C.2, we get that 
\[
    a_{k} \leq \underbrace{\max_{s = 0, \ldots, T -  1} a_{k, s}}_{\text{bound by a constant in \eqref{a_k_constant_3}}} \times\overbrace{\max _{j = 1, 2, 3} s_{a, j}}^{\text{with order } \log T} + 3 T^{-1} \cdot\big(T - \underbrace{\max_{j = 1, 2, 3} s_{a, j}}_{\text{with order } \log T}\big) \lesssim \log T.
\]
\begin{remark}
   Here we do not give the explicit expression of the bound as it involves several complicated integrals, but its order is exactly $\log T$. However, for some explicit distributions, the expectation in Lemma \ref{lem_subE_ineq} can be bounded by some simpler displays, and hence we can get explicit expressions for the bounds in these cases. See Corollary \ref{cor_ber} and Corollary \ref{cor_gau} for instance.
\end{remark}

\subsection{Bounding $b_k$}

Similar to the previous sections, for bounding $b_k = \sum_{s = 0}^{T - 1} b_{k, s}$, we decompose $b_{k, s} = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1}\big) + 1$ as $b_k = b_{k, s, 1} + b_{k, s, 2} + b_{k, s, 3} + 1$ with
\[
    b_{k, s, 1} := {\mathbb{E}} \big[ \mathbb{I} ( Q_{k, s} (\tau_k) > T^{-1}) \mathbb{I} (A_{k, s}^{\prime c} )\big],
\]
\[
    b_{k, s, 2} := {\mathbb{E}} \big[ \mathbb{I} ( Q_{k, s} (\tau_k) > T^{-1}) \mathbb{I} (A'_{k, s}) \mathbb{I} (G_{k, s}^c)\big],
\]
and
\[
    b_{k, s, 3} := {\mathbb{E}} \big[ \mathbb{I} ( Q_{k, s} (\tau_k) > T^{-1}) \mathbb{I} (A'_{k, s}) \mathbb{I} (G_{k, s})\big].
\]
Since $b_{k, s}$ is naturally bounded {the} constant $2$, it only needs to bound $s_{b, j}: = \max \{ s: b_{k, s, j} \geq T^{-1}\}$ with the $C \log T$ for $j = 1, 2, 3$ with some positive constant $C$.



For any $\star \in \{ (a, j), (b, j): j \in 1, 2, 3\}$, we denote $s_{\star}'$ as the quantity where we {replaces} $C_2$ in $s_{\star}$  by $C_1$. 

It is easy to see that $\mathbb{P} (A^{\prime c}_{k, s}) \leq T^{-2} $ when $s \geq s_{a, 2}' / 2$. Then by the results on bounding $a_{k, s, 2}$,
we have $\max\{ b_{k,s, 1}, \, b_{k, s, 2}\} \leq T^{-2} < T^{-1}$ as long as $s \geq \max \{ s'_{a, 1}  / 2 , \, s'_{a, 2} \} \asymp \log T$. 

It remains to deal with $b_{k, s, 3}$. We will first bound $\Gamma_k = \overline{Y}_{k, s}  - \overline{R}_{k, s}^*$ on the event $A_{k, s}' \, \cap \, G_{k, s}$. Write
\[
    A'_{k, s} = \bigg\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2\lambda} < C_2 \Delta_k \bigg\} = \bigg\{ \tau_k - \frac{\mu_k + \lambda}{1 + 2\lambda} - C_2 \Delta_k  < \Gamma_k \bigg\},
\]
and
\[
    \begin{aligned}
        G_{k, s} = \big\{ -C_3 \Delta_k \leq \overline{Y}_{k, s} - \overline{R}_{k, s}^* \leq C_3 \Delta_k \big\} & = \big\{ \tau_k -C_3 \Delta_k \leq \overline{Y}_{k, s} + \Gamma_k \leq \tau_k + C_3 \Delta_k \big\} \\
        & = \big\{ \tau_k - 2 C_3 \Delta_k - \overline{R}_{k, s}^* \leq \Gamma_k \leq \tau_k + 2 C_3 \Delta_k - \overline{R}_{k, s}^* \big\}.
       
    \end{aligned}
\]
Hence, we conclude that 
\[
    \begin{aligned}
        A'_{k, s} \, \cap \, G_{k, s}\\
        (\text{by plugging the inequality for $\overline{R}_{k, s}^*$ in } A'_{k, s}) \subseteq & \bigg\{ \tau_k - \frac{\mu_k + \lambda}{1 + 2\lambda} - C_2 \Delta_k  < \Gamma_k \leq\tau_k + C_2 \Delta_k - \frac{\lambda}{1 + 2\lambda} \bigg\} \\
        (\text{let } 2 C_3 \geq C_2) \subseteq & \bigg\{ \tau_k  - C_2 \Delta_k - \frac{\mu_k}{2}  < \Gamma_k < \tau_k + C_2 \Delta_k \bigg\}
    \end{aligned}
\]
Exactly the same as before, we require $\tau_k - C_2 \Delta_k - \frac{\mu_k }{2} \leq 0$ to ensure $\Gamma_k$ falls in a relatively small interval. To achieve this, we can just take
\begin{equation}\label{requirement_4}
    \tau_k = 2 C_2 \Delta_k + \mu_k / 2
\end{equation}
together with
\begin{equation}\label{requirement_3}
    2 C_3 \geq C_2.
\end{equation}
Then we will obtain that a bound for $\Gamma_k$ is
\[
    C_2 \Delta_k = \tau_k  - C_2 \Delta_k - \frac{\mu_k}{2} < \Gamma_k < \tau_k + C_2 \Delta_k = 3 C_2 \Delta_k + \frac{\mu_k}{2} \leq \frac{7}{2} < 4,
\]
i.e. $\Gamma_k \in (C_2 \Delta_k, 4) \subset (0, + \infty)$. Similar to the technique in \eqref{a_2_bound}, we have
\[
    \begin{aligned}
        & Q_{k, s}\left(\tau_{k}\right) \mathbb{I}\left(A_{k, s}^{\prime}\right) \mathbb{I}\left(G_{k, s}\right) = {\mathbb{E}} \mathbb{P} \left(\overline{Y}_{k, s}-\overline{R}_{k, s}^{*}>\Gamma_{k} \mid \mathcal{H}_{k, T}\right) \mathbb{I}\left(A_{k, s}^{\prime}\right) \mathbb{I}\left(G_{k, s}\right) \\
        = & {\mathbb{E}} \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s (R_{k, i} - \overline{R}_{k, s}) \omega_i + \lambda \sum_{i = 1}^s (2 R_{k, i} \omega_i - \overline{R}_{k, s} \omega_i' - \overline{R}_{k, s} \omega_i'')
        \\
        & \qquad \qquad \qquad + \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s (\omega_i' - \omega''_i) - \Gamma_k (1 + 2\lambda) \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] < 0\Bigg\}\\
        \overset{\text{apply steps in  \eqref{G_function_bound_2}}}{\leq} & {\mathbb{E}} \exp \left\{ - \frac{9 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 \Gamma_k^2 s^2 }{12 \lambda^2 \operatorname{subE} (8 \sqrt{2 s} \sigma_k^2) + s \big[ 12 \lambda \sigma_1^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 \Gamma_k^2  \big]}\right\} \\
        \overset{\text{by the bound of } \Gamma_k}{\leq} & {\mathbb{E}} \exp \left\{ - \frac{144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 s }{12 \lambda^2 \operatorname{subE} (8 \sqrt{2} \sigma_k^2) + \big[ 12 \lambda \sigma_k^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_2^2 \Delta_k^2  \big]}\right\}\\
        \overset{\text{Lemma \ref{lem_subE_ineq}}}{\leq} & \Big[ 1 + 144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2  s ( h_1^{**} + h_2^{**}) \Big] \exp \left\{  - \frac{144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2  s }{\big[ 12 \lambda \sigma_k^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_2^2 \Delta_k^2 \big]}\right\} \\
        \overset{\text{by }s \leq T}{\leq} & T \Big[ 1 + 144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 ( h_1^{**} + h_2^{**}) \Big] \exp \left\{  - \frac{144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2  s }{\big[ 12 \lambda \sigma_k^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_2^2 \Delta_k^2 \big]}\right\}.
    \end{aligned}
\]
where $h_1^{**}$ and $h_2^{**}$ are defined in Lemma \ref{lem_subE_ineq} by replacing the corresponding parameters again. Therefore, one can just take 
\[
    \begin{aligned}
        s \geq s_{b, 3} & := \frac{ 12 \lambda \sigma_k^2 + 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 6  (1 + 2\lambda)^2 C_2^2 \Delta_k^2}{144 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} \log \Big[ T^3 \big( 1 + 144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 ( h_1^{**} + h_2^{**}) \big) \Big]
    \end{aligned}
\]
to ensure $b_{k, s, 3} \leq T^{-1}$.
And then, by using a similar argument, we have 
\[
    b_{k} \leq 2 \times\overbrace{\max _{j = 1, 2, 3} s_{b, j}}^{\text{with order } \log T} + 3 T^{-1} \cdot\big(T - \underbrace{\max_{j = 1, 2, 3} s_{b, j}}_{\text{with order } \log T}\big) \lesssim \log T.
\]
\begin{remark}
   Similarly, for specific reward distributions, we can bound $b_k$ in $\log T$ with some simple and specific coefficients. See Corollary \ref{cor_ber} and Corollary \ref{cor_gau}.
\end{remark}

Finally, {what} remains is to show {that} the restrictions (\ref{requirement_0}), (\ref{requirement_1}), (\ref{requirement_2}), (\ref{requirement_3}), and (\ref{requirement_4}) are compatible. Indeed, they will build the following system of constraints,
\[
    \left\{ \begin{array}{l}
         \frac{\mu_1}{2} - 2 C_1 \Delta_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \\
         2 C_2 \Delta_k + \frac{\mu_k}{2} \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \\
        \tau_k - \frac{\mu_1}{2} - C_1 \Delta_k  \leq \tau_k + C_2\Delta_k - \frac{\mu_k }{2} \\
        C_1 \geq 2C_3 \\
        2 C_3 \geq C_2 \\
        \frac{\mu_1}{2} - 2 C_1 \Delta_k = 2 C_2 \Delta_k + \frac{\mu_k}{2}
    \end{array}\right..
\]
We can see that 
\[
    C_1 = C_2 = \frac{1}{8}, \qquad C_3 = \frac{1}{16},
\]
is the unique solution of the above system. Up to now, we have finished the first part in Theorem \ref{thm.MAB}. For the second part, 
\begin{equation*}
    \begin{aligned}
        \operatorname{Reg}_T & = \sum_{\Delta_k : \Delta_k < \Delta} \Delta_k {\mathbb{E}} J_{k, T} + \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k {\mathbb{E}} J_{k, T} \\
        & \lesssim T \Delta + \sum_{\Delta_k : \Delta_k \geq \Delta} \Delta_k  +  \sum_{\Delta_k : \Delta_k \geq \Delta} \frac{\log T}{\Delta_k} \\
        & \lesssim T \Delta + \frac{ K \log T}{\Delta} \lesssim \sqrt{K T \log T}.
    \end{aligned}
\end{equation*}
for any previously specified $\Delta \in (0, 1)$ by Cauchy inequality. Therefore, we finish the proof of Theorem \ref{thm.MAB}.





\section{Proof of Corollary \ref{cor_ber}}

The similar as stated in Section \ref{proof_thm}. Take advantage of Bernoulli rewards are bounded in $[0, 1]$, one can get that 
\[
    \begin{aligned}
        a_{k, s} & \leq \big( e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \big) \mathbb{I} ( \mu_1 \neq 1 / 2 )  + 2 \big( 1 + 2 e^{9 / 8} \big) \mathbb{I} ( \mu_1 = 1/ 2 ).
    \end{aligned}
\]
\todohw{Need to revise later}
as long as the parameters satisfying the restrictions in Theorem \ref{thm.MAB}. Then we need to control $a_{k, s, j}$ with $\log T$ for $j \in \{ 1, 2, 3\}$. These steps will be completed in the following.
\begin{itemize}
    \item For $a_{k, s, 1}$, recall the inequality \eqref{a_k_1_bound}. We use the fact that the rewards are bounded in $[0,1]$, Hoeffding inequality will give
    \[
        \mathbb{P} (A_{1, s}^c) = \mathbb{P} (\mu_1 - \overline{R}_{1, s} \geq (1 +2  \lambda) C_1 \Delta_k) \leq \exp \left\{ - 2 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2 s\right\}.
    \]
    Hence, take
    \[
        s \geq s_{a, 1} := \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2} \log T,
    \]
    we have $\mathbb{P} (A_{1, s}^c) \leq T^{-2}$ , and $a_{k, s, 1} \leq T^{-1}$ then.
    
    \item For $a_{k, s, 2}$, one can {follow} the same steps as in the proof of Theorem \ref{thm.MAB}, while constants in the upper bound can be simpler and sharper due to the distribution of rewards being Bernoulli.
    To be specific, the variance $\sigma_{\omega}^{-2} \operatorname{var}(f_1 + f_2 - f_3)$ in \eqref{bound_var_f} can be bound by a deterministic quantity:
    
   
    \[
        \begin{aligned}
            & \mathbb{G}_1 (s, \lambda) + \mathbb{G}_2 (s, \lambda) + g_3 (s, \lambda) \\
            \overset{\text{by the same steps in} \eqref{G_function_bound_2}}{\leq} & 6 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big] \\
            \overset{\text{use the fact } R_{1, i} \in [0, 1]}{\leq} & 6 \lambda^2 s + s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 1 ) + 3  (1 + 2\lambda)^2 C_3^2 \Delta_k^2 \big].
        \end{aligned}
    \]
    Then use \eqref{a_2_bound} again, 
    \[
        \begin{aligned}
            a_{k, s, 2} & \leq 2 T  {\mathbb{E}} \exp \left\{ - \frac{9 C_3^2 (1 + 2\lambda)^2 \Delta_k^2 s^2 }{2 \sigma_{\omega}^{2} \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + g_3 (s, \lambda) \big]}\right\} \\
            & \leq 2 T \exp \left\{ - \frac{9 \sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2 s}{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }\right\}.
        \end{aligned}
    \]
    Hence, if we take 
    \[
        \begin{aligned}
            s \geq s_{a, 2} & := \frac{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{3 \sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2} \log T \\
            (
            \text{require } T \geq 2) & \geq \frac{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{9\sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2} \log (2 T^2)
        \end{aligned}
    \]
    and then $a_{k, s, 2} \leq T^{-1}$.
    \iffalse
    Then if we denote 
    \[
        f(x) := \exp \left\{ - \frac{64^{-1} (1 + 2 \lambda)^4 \Delta_k^2 s}{4 \Big[ s^{- 1 / 2} (2 \lambda + 1) (x - s \mu_1) + \lambda \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big] \Big]^2 + 2 s^{-1} g_5(s, \lambda) \big]^2} \right\},
    \]
    then $f(x)$ is a decreasing function on $0 \leq x \leq s \mu_1$ and an increasing function on $x \geq s \mu_1$. By symmetric technique of Binomial distribution and Lemma 2 in \cite{kveton2019garbage}, we have
    \[
        a_{k, s, 2} \leq {\red{4}} T \sum_{x = 0}^{\lfloor s \mu_1 \rfloor} B(x ; s, \mu_1) f(x) \leq 4 T \left[ \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[-2 i^{2}\right] f(\mu_1 s -(i+1) \sqrt{s})+\exp \left[-2 s^{2} \mu_1^2 \right] f(0) \right].
    \]
    Similarly to \cite{kveton2019garbage}, we can show that 
    \[
        \begin{aligned}
            & \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[-2 i^{2}\right] f(\mu_1 s -(i+1) \sqrt{s}) +\exp \left[-2 s^{2} \mu_1^2 \right] f(0) \\
            \leq & \sum_{i=0}^{\lfloor s \mu_1 \rfloor} \exp \left[-2 i^{2}\right] \exp \left\{- \frac{64^{-1} (1 + 2 \lambda)^4 \Delta_k^2 s}{4 \left[ \lambda \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big] - (2 \lambda + 1) (i + 1)\right]^2 + 2 s^{-1} g_5(s, \lambda)} \right\} \\
            & \qquad +\exp \left[-2 s^{2} \mu_1^2 \right] f(0) \\
            \leq & \int_1^{s \mu_1 + 1} \exp \{ - 2 (u - 1)^2 \} \exp \left\{ - \frac{c_1 s}{4 (c_2 - (2 \lambda + 1) u )^2 + 2 c_3}\right\}  \, du \\
            = & \int_1^{s \mu_1 + 1} \exp \left\{ - \left[ 2 (u - 1)^2 + \frac{c_1 s}{4 (c_2 - (2 \lambda + 1) u )^2 + 2 c_3} \right] \right\}  \, du \\
            \leq & \int_1^{s \mu_1 + 1} \exp \left\{- \left[ 2 s^2 \mu_1^2 + \frac{c_1 s}{4(c_2 - (2 \lambda + 1) (s \mu_1 + 1))^2}\right] \right\} \, du \\
            \leq & \int_1^{s \mu_1 + 1} \exp \left\{ - 2 s \mu_1^2 \right\} \, du \leq T \mu_1 \exp \left\{ - 2 s \mu_1^2 \right\}
           
           
           
        \end{aligned}
    \]
    with $c_1 = 64^{-1} (1 + 2 \lambda)^4 \Delta_k^2$, $c_2 = \lambda \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big]$, and $c_3 = \lambda^2 \big\{ \big[ (2 \mu_1 - 1) - C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \big\}$. Therefore, if $T \geq 2$, by taking 
    \[
        \red{s_{a, 2} \geq \frac{5 \log T}{\mu_1^2}\geq \frac{\log 4 T^3 \mu_1}{\mu_1^2}}
    \]
    one will have 
    \[
        a_{k, s, 2} \leq T^{-1}
    \]
    \fi
    \item To deal with $a_{k, s, 3}$,
   
    by taking the same technique as $a_{k, s, 2}$, we have
    \[
        \begin{aligned}
             \big[ 1 - Q_{1,s} (\tau_k)\big] \mathbb{I}(A_{1, s}) \mathbb{I}(G_{1, s}) & \leq {\mathbb{E}} \exp \left\{ - \frac{9 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 (- \Gamma_1)^2 s^2 }{ s \big[ 4 \lambda^2 (\lambda^2 + \lambda + 4)  + 6  (1 + 2\lambda)^2 (- \Gamma_1)^2  \big]}\right\} \\
             & \leq \exp \left\{ - \frac{144 \sigma_{\omega}^{-2} (1 + 2\lambda)^2 s }{ 4 \lambda^2 (\lambda^2 + \lambda + 4)  + 6  (1 + 2\lambda)^2 C_1^2 \Delta_k^2 }\right\} \leq T^{-2}
        \end{aligned}
    \]
    as long as 
    \[
        s \geq s_{a, 3} := \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2}{36 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} \log T.
    \]
    \iffalse
    \[
        g(x) = \exp \left\{ - \frac{64^{-1} (1 + 2 \lambda)^4 \Delta_k s}{4 \Big[ s^{- / 2} (2 \lambda + 1) (x - s \mu_1) + \lambda \big[ (\lambda + \mu_1) + 4 (1 + 2 \lambda)\big] \Big]^2 + 2 s^{-1} g_5(s, \lambda)}\right\}
    \]
    one can show that
    \[
        \begin{aligned}
            \big[ 1 - Q_{1, s}(\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \leq 2 \sum_{x = 0}^{\lfloor s \mu_1 \rfloor} B(x; s, \mu_1) g(x) \leq 2 s \mu_1 \exp \{ - s \mu_1\} \leq T^{-1}
        \end{aligned}
    \]
    as long as $s \geq s_{a, 3} := s_{a, 2}$. And hence, in this case, one will have $a_{k, s, 3} \leq T^{-1}$.
    \fi
\end{itemize}

Combining these results above with 
\[
    \begin{aligned}
        &\max \left\{ \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2}, \frac{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{3\sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2}, \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2}{36 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} \right\} \\
        \leq & \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2} + \frac{4 \lambda^2 (\lambda^2 + \lambda + 4) + 6 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{3 \sigma_{\omega}^{-2} C_3^2 (1 + 2 \lambda)^2 \Delta_k^2} + \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2}{36 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2}\\
        = & \frac{36 (1 + 2 \lambda)^2 + \lambda^2 (\lambda^2 + \lambda + 4)}{18 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} + \frac{32^2 \lambda^2 \sigma_{\omega}^{2}  (\lambda^2 + \lambda + 4) + 12}{3 (1 + 2 \lambda)^2 \Delta_k^2} + \frac{\sigma_{\omega}^{2} \Delta_k^2}{3 \times 16^2},
    \end{aligned}
\]
we obtain that 
\[
    \begin{aligned}
        a_k & \leq \max_{s = 0, \ldots, T -  1} a_{k, s} \times\max _{j = 1, 2, 3} s_{a, j} + 3 T^{-1} \cdot\big(T - \max_{j = 1, 2, 3} s_{a, j}\big) \\
        & \leq \Big[\big( e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \big) \mathbb{I} ( \mu_1 \neq 1 / 2 )  + 2 \big( 1 + 2 e^{9 / 8} \big) \mathbb{I} ( \mu_1 = 1/ 2 ) \Big] \\
        & \qquad \qquad \times \left[  \frac{36 (1 + 2 \lambda)^2 + \lambda^2 (\lambda^2 + \lambda + 4)}{18 \sigma_{\omega}^{- 2}  (1 + 2 \lambda)^2} + \frac{32^2 \lambda^2 \sigma_{\omega}^{2}  (\lambda^2 + \lambda + 4) + 12}{3 (1 + 2 \lambda)^2 \Delta_k^2} + \frac{\sigma_{\omega}^{2} \Delta_k^2}{3 \times 16^2} \right] 
        \log T + 3.
    \end{aligned}
\]
   
By using the same technique, one can show that 
\[
    \begin{aligned}
        b_k & \leq 2 \times\max _{j = 1, 2, 3} s_{b, j} + 3 T^{-1} \cdot\big(T - \max_{j = 1, 2, 3} s_{b, j}\big) \\
        & \leq 2 \times \left\{ \frac{4}{(1 + 2 \lambda)^2 \Delta_k^2}, \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }{3 \sigma_{\omega}^{- 2}  C_3^2 (1 + 2 \lambda)^2 \Delta_k^2}, \frac{2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_2^2 \Delta_k^2}{36 \sigma_{\omega}^{-2} (1 + 2 \lambda)^2} \right\} \log T + 3 \\
        & \leq  2 \left[  \frac{18(1 + 2 \lambda)^2 + \lambda^2 (\lambda^2 + \lambda + 4)}{18 \sigma_{\omega}^{- 2}  (1 + 2 \lambda)^2} + \frac{2 \times16^2  \sigma_{\omega}^{2}  \lambda^2 (\lambda^2 + \lambda + 4) + 12}{3 (1 + 2 \lambda)^2 \Delta_k^2} + \frac{\sigma_{\omega}^{2} \Delta_k^2}{3 \times 16^2} \right] 
        \log T + 3
    \end{aligned}
\]
These results together with Lemma \ref{lem.a.b} give Corollary \ref{cor_ber}.

\section{Proof of Corollary \ref{cor_gau}}

The basic proof idea is the same as the proof of Corollary \ref{cor_ber}. Since the Gaussian variable is still sub-Gaussian, the constant bound in \eqref{a_k_constant_3} is still available for this case. And next we will bound $s_{a,  j} :=  \min \{ s: a_{k, s, j} \leq T^{-1}\}$ with $\log T$ under the Gaussian case.
\iffalse
\begin{itemize}
    \item[1.] By using the fact that $s^2 (\overline{R}_{1, s} - \mu_1) \sim \chi^2(s)$,
    \[
        \begin{aligned}
            a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \times1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\} \\
            & \leq 
        \end{aligned}
    \]
    \[
        \begin{aligned}
            a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 s (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left(  \overline{R}_{1, s} - \frac{\mu_1 + \lambda}{1 + 2 \lambda} \right)^2 }\right\} \times1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \times1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  \\
            (\text{by using the MGF of } \chi^2(s))& = \left[ 1 - \frac{3 (2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^2 [(\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2]s^2}\right]^{- \frac{s}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \\
            (\text{take } s = 1) & \leq \left[ 1 - \frac{3 (2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^2 [(\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2]}\right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}.
        \end{aligned}
    \]
    since $s = 0$ naturally satisfies the above inequality. This further implies we will need
    \[
        \frac{3 (2 \lambda + 1)^2}{\sigma_{\omega}^2 \lambda^2 [(\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2]} < 1
    \]
    with its sufficient condition as 
    \[
        \frac{3(2 \lambda + 1)^2}{\lambda^2[\lambda^2 + (\lambda + 1)^2]} \leq 1
    \]
    i.e. $\lambda \geq 1 + \sqrt{2}$. In a word, under the assumptions in Corollary \ref{cor_gau}, one will have
    \[
        \begin{aligned}
            a_{k, s} & \leq \left[ 1 - \frac{3 (2 \lambda + 1)^2}{\sigma_{\omega}^2 [(\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2]}\right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \\
            & \leq \left[ 1 - \frac{3(2 \lambda + 1)^2}{\lambda^2[\lambda^2 + (\lambda + 1)^2]} \right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}. 
        \end{aligned}
    \]
    with $\sigma_{\omega}^2 \geq 1$ for $s \in \{ 0, \ldots, T - 1\}$.
    
    \item[2.] For controlling $a_{k}$, one can easily show that 
    \[
        a_k \leq \left\{ \left[ 1 - \frac{3(2 \lambda + 1)^2}{\lambda^2[\lambda^2 + (\lambda + 1)^2]} \right]^{- \frac{1}{2}} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \right\} \left[ \frac{65}{(1 + 2 \lambda)^2 \Delta_k^2} + \frac{6}{\log 2} \right] \log T + 3.
    \]
    by the following statements.
    \begin{itemize}
        \item[2.1] Still use sub-Gaussian inequality to get that we have $a_{k, s, 1} \leq T^{-1}$ when
        \[
            s \geq s_{a, 1} = \frac{64 }{(1 + 2 \lambda)^2 \Delta_k^2} \log T.
        \]
        
        \item[2.2] For dealing with $a_{k, s, 2}$, we again use the fact that 
        \[
            \Big[ \sqrt{s} (\overline{R}_{1, s} - \mu_1) + \lambda \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big] \Big]^2 \, \sim \, \chi^2(s; s^{-1}\lambda^2 \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k \big]^2 )
        \]
        is a noncentral chi-squared distribution, then it will lead to the result that 
        \[
            \begin{aligned}
                a_{k, s, 2} & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{64^{-1}(1 + 2 \lambda)^4 \Delta_k^2 s}{4\big[ \chi^2(s; s^{-1}\lambda^2 \big[ (\lambda + \mu_1) + 16^{-1} (1 + 2 \lambda) \Delta_k) + 2 s^{-1}g_5(s, \lambda)\big]}\right\} \\
                & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{64^{-1} (1 + 2\lambda)^4 \Delta_k^2 s}{4  \big[ \chi^2(s) + s^{-1}g_5(s, \lambda)\big]}\right\} \\
                & \leq 2 T \int_0^{\infty} \frac{2^{- s / 2}}{\Gamma(s / 2)} u^{-s / 2-1} e^{-1 /(2 u)} e^ { - 256^{-1} (1 + 2\lambda)^4 \Delta_k^2 s u} \, du \\
                (\text{by Cauchy inequality})& \leq  2 T \frac{2^{- s / 2}}{\Gamma(s / 2)}  \int_0^{\infty} u^{- s / 2 - 1} e^{- 1 / u} e^{- 16 (1 + 2 \lambda)^4 \Delta_k \sqrt{s}} \, du  \\
                & =  2 T \frac{2^{- s / 2}}{\Gamma(s / 2)}  \Gamma(s /2 ) e^{- 16 (1 + \lambda)^2 \Delta_k \sqrt{s}} \\
                & \leq 2 T \exp \left\{ - \left[ \frac{\log 2}{2} s + 16 (1 + 2 \lambda)^2 \Delta_k \sqrt{s}\right]\right\} \leq T^{-1}
            \end{aligned} 
        \]
        if 
        \[
            s \geq s_{a, 2} :=  \left[ \frac{1}{8(1 + 2 \lambda)^2 \Delta_k} + \frac{4}{\log 2}\right] \log T
        \]
        \item[2.3] Just by replacing $C_3 \Delta_k$ by $(- \Gamma_1)$ which is bounded in $(C_1 \Delta_k, 4)$, and we will get that
        \[
            \begin{aligned}
                \big[ 1 - Q_{1, s}(\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \leq  \exp \left\{ - \left[ \frac{\log 2}{2} s + 16 (1 + 2 \lambda)^2 \Delta_k \sqrt{s}\right]\right\} \leq T^{-1}
            \end{aligned}
        \]
        i.e. $a_{k, s, 3} \leq T^{-1}$ whenever 
        \[
            s \geq s_{a, 3} := \left[ \frac{1}{16(1 + 2 \lambda)^2 \Delta_k} + \frac{2}{\log 2}\right] \log T.
        \]
    \end{itemize}
    
    \item[3.] Exactly the same as 2., we can get that 
    \[
        b_{k} \leq 2 \left[ \frac{33}{(1 + 2 \lambda)^2 \Delta_k^2} + \frac{6}{\log 2}\right] \log T + 3.
    \]
\end{itemize}
These results give Corollary \ref{cor_gau}.
\fi

\begin{itemize}
    \item For $a_{k, s, 1}$, we still use the same steps of proof in Theorem \ref{thm.MAB} directly: take
    \[
        s \geq s_{a, 1} = \frac{\sigma_1^2}{C_1^2 (1 + 2 \lambda)^2 \Delta_k^2} \log T,
    \]
    we have $a_{k, s, 1} \leq T^{-1}$.
    
    \item For $a_{k, s, 2}$, we note that
    \begin{equation}\label{chi_square}
    	\sum_{i = 1}^s (R_{1, i} - \mu_1)^2 = \sigma_1^2 \sum_{i = 1}^s \left( \frac{R_{1, i} - \mu_1}{\sigma_1} \right)^2 \, \sim \, \sigma_1^2 \chi^2(s).
    \end{equation}
    is a chi-square distribution.
    By using the density of chi-square distribution, we have a bound of \eqref{a_2_bound} as
    \[
        \begin{aligned}
            a_{k, s, 2} & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{9 C_3^2 (1 + 2 \lambda)^2 \Delta_k^2 s^2}{2 \sigma_{\omega}^{2} \big[ \mathbb{G}_{1} (s, \lambda) + \mathbb{G}_{2} (s, \lambda) + g_3(s, \lambda)\big]}\right\} \\
            & \overset{\text{use the bound in \eqref{G_function_bound_2} and \eqref{chi_square}}}{\leq} 2 T {\mathbb{E}} \exp \left\{ - \frac{9 \sigma_{\omega}^{- 2}  C_3^2 (1 + 2 \lambda)^2 \Delta_k^2 s^2}{12 \lambda^2 \sigma_1^2 \chi^2(s) + 2 s \big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2\big] }\right\} \\
            & =  2 T \int_0^{\infty} \exp \left\{ - \frac{D_1 s^2}{D_2 u + D_3}\right\} \frac{1}{2^{s / 2} \Gamma(s / 2)} u^{{s} / {2} - 1} e^{- u / 2} \, du \\
            & = 2 T \int_0^{\infty} \exp \left\{ - \left[ \frac{D_1 s^2}{D_2 u + D_3} + \frac{u}{4} \right]\right\} \frac{1}{2^{s / 2} \Gamma(s / 2)} u^{{s} / {2} - 1} e^{- u / 4} \, du
        \end{aligned}
    \]
   
    with
    \[
        D_1 = 9 \sigma_{\omega}^{- 2}  C_3^2 (1 + 2 \lambda)^2 \Delta_k^2, \qquad D_2 = 12 \lambda^2 \sigma_1^2, \qquad D_3 = 2 \big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2\big].
    \]
    Write 
    \[
        \frac{D_1 s^2}{D_2 u + D_3} + \frac{u}{4} = D_4 s^2 + \frac{D_2 u^2 + (D_3 - 4 D_2 D_4 s^2) u + 4 (D_1 - D_3 D_4) s^2}{4 (D_2 u + D_3) }
    \]
    with $D_4 > 0$ is a constant making
    \begin{equation}\label{gau_D_4}
        D_2 u^2 + (D_3 - 4 D_2 D_4 s^2) u + 4 (D_1 - D_3 D_4) s^2 \geq 0
    \end{equation}
    strictly holds on $u \in (0, \infty)$. A sufficient condition for \eqref{gau_D_4} is
    \[
         \begin{aligned}
             \boldsymbol\Delta & = (D_3 - 4 D_2 D_4 s^2)^2 - 4 D_2 \big[ 4 (D_1 - D_3 D_4) s^2 \big] \\
             & = 16 D_2^2 D_4^2 s^4 + 8 D_2 (D_3 D_4 - 2) s^2 + D_3^2 > 0, \qquad s \in [T].
         \end{aligned}
    \]
    For ensuring the above holds, one only needs to ask
    \[
        D_3 D_4 -  2 \geq 0.
    \]
    Therefore, we choose 
    \[
        D_4 := \frac{2}{D_3} = \frac{1}{\big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2\big]}.
    \]
    and hence 
    \[
        \frac{D_1 s^2}{D_2 u + D_3} + \frac{u}{4} \geq \frac{2}{D_3} s^2, \qquad \forall \, u > 0.
    \]
    It will furthermore give
    \[
        \begin{aligned}
            a_{k, s, 2} & \leq 2 T \int_0^{\infty} \exp \left\{ - \left[ \frac{D_1 s^2}{D_2 u + D_3} + \frac{u}{4} \right]\right\} \frac{1}{2^{s / 2} \Gamma(s / 2)} u^{{s} / {2} - 1} e^{- u  / 4}\, du \\
            & \leq 2 T \int_0^{\infty} \exp \left\{ - \frac{2 s^2 }{D_3} \right\} \frac{1}{2^{s / 2} \Gamma(s / 2)} u^{{s} / {2} - 1} e^{- u /4} \, du \\
            (\text{use the density of Gamma distribution})& = 2 T \exp \left\{ - \frac{2 s^2}{D_3} \right\} 2^{s / 2} \\
            & = 2 T \exp \left\{ -  \frac{2 s^2}{D_3} + \frac{s \log 2} {2} \right\} \leq 2 T \exp \left\{ -  \frac{2 s^2}{D_3} \right\} .
        \end{aligned}
    \]
    Hence, we can take
    \[
        s \geq s_{a, 2} := \frac{3 D_3 \log T}{2} = 3 \sigma_{\omega}^{2} \big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_3^2 \Delta_k^2\big] \log T.
    \]
    to guarantee $a_{k, s, 2} \leq T^{-1}$ for any $T \geq 2$.
    \item The approach to deal with $a_{k, s, 3}$ is the same as above, and we can get we have $a_{k, s, 3} \leq T^{-1}$ whenever
    \[
        s \geq s_{a, 3} = 3  \sigma_{\omega}^{2}  \big[ 2 \lambda^2 (\lambda^2 + \lambda + 4) + 3 (1 + 2 \lambda)^2 C_1^2 \Delta_k^2\big] \log T.
    \]

   
\end{itemize}

Combining the expressions of $a_{k, s, j}$, $j \in 1, 2, 3$, we obtain 
    \[
        \begin{aligned}
            a_{k} & \leq \Big[\big( e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \big) \mathbb{I} ( \mu_1 \neq 1 / 2 )  + 2 \big( 1 + 2 e^{9 / 8} \big) \mathbb{I} ( \mu_1 = 1/ 2 ) \Big] \\
            & \qquad \qquad \times \left[  \frac{16 \sigma_1^2}{(1 + 2 \lambda)^2 \Delta_k^2} + 12 \lambda^2  \sigma_{\omega}^{2} (\lambda^2 + \lambda + 4)  + \frac{153}{256} (1 + 2 \lambda)^2  \sigma_{\omega}^{2}  \Delta_k^2 \right] 
        \log T + 3.
        \end{aligned}
    \]
And similarly the bound of $b_k$ can be also proven as follows:
\[
     b_k \leq 2 \left[  \frac{16 \sigma_k^2}{(1 + 2 \lambda)^2 \Delta_k^2} + 9 \lambda^2  \sigma_{\omega}^2 (\lambda^2 + \lambda + 4)  + \frac{297}{512} (1 + 2 \lambda)^2 \sigma_{\omega}^{2} \Delta_k^2 \right] \log T + 3.
\]

\section{Introduction}

We start our discussion on \name{MBTS} with the multi-armed bandit (MAB) problem. 
For any positive integer $M$, let $[M] = \{1, \dots, M\}$. 
At each round $t \in [T]$,  the agent needs to choose an arm $A_t \in [K]$, and then receive a stochastic reward $R_t$. 
Denote the mean reward of each arm $k$ as $u_k = {\mathbb{E}}\big[ R_t | A_t = k \big]$. 
The goal is to minimize the cumulative regret 
\begin{equation}\label{eq:REG}
    \operatorname{Reg}_{T} := {\mathbb{E}} \operatorname{Reg}_{T}(Y,\mathcal{A}) = {\mathbb{E}} \sum_{t=1}^{T} [\mu_1-\mu_{A_{t}}]=\sum_{k=2}^{K} \Delta_{k} {\mathbb{E}} \Big[\sum_{t=1}^{T} 1 \left\{A_{t}=k\right\}\Big],
\end{equation}
where we assume, w.o.l.g., the first arm is the optimal one, i.e. $\mu_1 = \max_{k \in [K]} \mu_k$. From \cite{kveton2019garbage}, we know that the optimal rate for $\operatorname{Reg}_{T} $ is exactly $\log T$. Hence, researchers' aim is finding an algorithm making the cumulative regrets achieve the optimal rate. 
\vspace{2ex}

Generally speaking, there are two popular algorithmic frameworks, Upper Confidence Bound (UCB) and Thompson Sampling (TS):

\begin{itemize}
    \item UCB adopts a frequentist viewpoint and construct an upper confidence bound for $\mu_k$ as $\widehat{U}_k$, for every $k \in [K]$, UCB then chooses $I_t = \arg \max_{k \in [K]} \widehat{U}_{k}$;
    
    \item TS adopts a Bayesian viewpoint, maintain a posterior over the model, and sample action from it. Specifically, we can maintain a posterior over $k \in [K]$ as $P_{\omega_k} \in \mathcal{P}$, sample $\omega_k \sim P_{\omega_k}$, and then choose $I_t = \arg \max_{k \in [K]} Y(k, \omega_k)$ with some target function $Y(\cdot, \omega_k): [K] \rightarrow \mathbb{R}$ similar to $\widehat{U}_k$ in UCB. This is equivalent to directly maintaining a posterior over $I^*_t = \arg \max_{k \in [K]} Y_k$ and then sample $I_t$.
\end{itemize}

Here have been several articles touch this problem, and as we said before, they either base on UCB or TS. It is worthy to note that for a specific problem, i.e. MAB with some particular type of random rewards, the UCB method is partly restricted. This is by the fact that once we got the Bootstrap upper bound with weights $\mathbf{w}_n = (w_1, \ldots, w_n)^{\top} \in \mathbb{R}^n$ such that
\begin{equation*}
    \mathbb{P}_{\mathbf{R}_{k, n}} \left[ \frac{1}{n} \sum_{i = 1}^n R_{k, i} - \mu_k > q_{\alpha} (\mathbf{R}_{k, n}, \mathbf{w}_n)\right] \leq \alpha,
\end{equation*}
then that the algorithm will be exactly become $I_t = \arg \max_{k \in [K]}  \left[ \frac{1}{n} \sum_{i = 1}^n R_{k, i} + q_{\alpha} (\mathbf{R}_{k, n}, \mathbf{w}_n) \right]$ with some pre-specific $\alpha \asymp o(T^{-1})$. In the contrast, TS method can vary widely since there are many choices of the Bayesian posterior $P_{\omega_k}$, we conclude the existing literature is listed in chronological order as follows
\begin{itemize}
    
    \item \cite{lu2017ensemble} is the first one to that put TS and try to theoretically analyze the regret in the Bandit problem. However, their results are neither nonasymptotic and problem specific.
    
    \item \cite{vaswani2018new} throw an insight for how to theoretically analyze the cumulative regret under nonasymptotic sense at Bootstrap framework. Here the nonasymptotic means we can get the relationship of the regret and the round $T$ exactly. 
    
    \item \cite{kveton2019garbage} is the pioneer, which exactly use the TS to achieve the optimal rate. It adds $a$ pseudo rewards from $(0, 1)$ with tuning parameter $a$ on each arm's history, and prove this method guarantees an $\log T$ under Bernoulli rewards on $\{ 0, 1\}$;
    
    
    \item \cite{kveton2019perturbed} again uses another Bayesian posterior to deal with the same problem with \cite{kveton2019perturbed}: it adds $a$ Bernoulli perturbation at each reward step, the $a$ is again the tuning parameter. It is worthy to note that this method is quite similar to \cite{kveton2019garbage} sharing with the same condition (under some transformation);
    
    \item \cite{wang2020residual} construct another Bayesian posterior based on the residuals $R_{k, i} - \overline{R}_{k}$, extended the existent theory and the optimal rate $\log T$ to a specific unbound reward, Gaussian setting. Their theory is exactly based on the exact tail of Gaussian distribution.
\end{itemize}

We have to say that the existent literature, which achieves the optimal regret rate, are ALL computationally heavy. As we can see, \cite{kveton2019garbage} and \cite{kveton2019perturbed} need to add $a$ pseudo random quantile at each round, their storage cost and computation cost are $\mathcal{O}(aT), \mathcal{O} (aT^2)$ and $\mathcal{O}(aT), \mathcal{O} (aKT)$ representatively. While \cite{wang2020residual} achieves the rate $\mathcal{O}(T), \mathcal{O} (KT)$, they need EXTRA steps to compute the residuals.
\vspace{2ex}

\textbf{Advances of the proposed method. }
Compared with the existing methods, our main advantages include
\begin{itemize}
     \item[1.] Our method is directly base on TS method, no other transformation is required. And our storage cost and computation cost are exactly $\mathcal{O}(T), \mathcal{O} (KT)$, no any other constants will be included;
    
    \item[2.] Our theoretical analysis is only dependent on the tail probability of the Bootstrap weights $\boldsymbol\omega$. As long as the rewards are bounded, our method can be applied. (instinct uncertainty or \cite{kveton2019perturbed} will use the pseudo rewards based on the known reward property, feasibility and empiricality)
    
    \item[3.] Our method can be easily extended to linear bandit and more complex setting.
    
\end{itemize}

\section{Algorithm and Theoretical Analysis}

\begin{algorithm}[!h]
\SetAlgoLined
\KwData{
number of arms $K$, 
multiplier weight distribution $\rho(\omega)$
}

Set $\mathcal{H}_{k, s}$ is the history of the arm $k$ with exact $s$ rewards. and $\overline{Y}_{k, 0} = -\infty, \forall k \in [K]$  \blue{why -inf?}

\For{t = 1, \dots, T}{

    Offer $I_t = \arg \max_{k \in [K]} \overline{Y}_{k, s}$ (break tie randomly)

    Observe reward $R_t$

    \For{k = 1, \dots, K}{
    Sample the multiplier weights $\{\omega_l\}_{l=0}^{s} \sim \rho(\omega)$. Note that ${\mathbb{E}} \omega_{\ell}$ should be equal to one to ensure the consistency.
    
    Update the mean reward $\overline{Y}_{k, s} = \left(\omega _0 {\color{red}\lambda_k} s + \sum_{\ell = 1}^{s} \omega_{\ell} R_{k, \ell} \right) / \left( \sum_{\ell = 0}^{s} \omega_{\ell} \right)$, where $R_{k, l}$ is the $l$-th non-zero element of $\mathcal{H}_{k, t}$ with some tranformation $L$ and $s = |\mathcal{H}_{k, t}|$. 
    
    \blue{We should have well-defined notations such as $s$ and $\mathcal{H}_{k,t}$ (at first it is $\mathcal{H}_{k,s}$)}
    }
    
} 
\caption{Multiplier Bootstrap-based Thompson Sampling for MAB with $[0, 1]$ rewards
\blue{[infeasible version]}
}\label{alg:MBTS-MAB}
\end{algorithm}



\begin{algorithm}[!h]
\SetAlgoLined
\KwData{
number of arms $K$, 
multiplier weight distribution $\rho(\omega)$, 
tuning parameter $\lambda$
}

Set $\mathcal{H}_{k} = \{\}$ be the history of the arm $k$, $c_k = 0$ ,and $\overline{Y}_{k} = +\infty, \forall k \in [K]$

\For{$t = 1, \dots, T$}{

    Offer $I_t = \arg \max_{k \in [K]} \overline{Y}_{k}$ (break tie randomly)

    Observe reward $R_t$
    
    Set $\mathcal{H}_{k} = \mathcal{H}_{k} \cup \{R_t\}$

    \For{$k = 1, \dots, K$}{
    \If{$|\mathcal{H}_{k}| > 0$}{
    
    Sample the multiplier weights $\{\omega_l, \omega'_l, \omega''_l\}_{l=1}^{|\mathcal{H}_{k}|} \sim \rho(\omega)$. 
    
    Update the mean reward $\overline{Y}_{k} = \left(\sum_{\ell = 1}^{|\mathcal{H}_{k}|} (\omega_{\ell} R_{k, \ell}+ \omega'_{\ell} * \lambda + 0 * \lambda *   \omega''_l ) \right) / \left( \sum_{\ell = 1}^{|\mathcal{H}_{k}|} (\omega_{\ell} + \lambda\omega'_{\ell} + \lambda\omega''_l) \right)$, where $R_{k, l}$ is the $l$-th element of $\mathcal{H}_{k}$ 
   

    }
    }
\blue{without the pseudo-reward $0$, the algorithm is not valid.}

    
\blue{$\overline{Y}_{k}$ should concentrate to $\frac{0 + \lambda + \mu_k}{2\lambda + 1}$, which preserves the order between arms}
   
} 
\caption{Multiplier Bootstrap-based Thompson Sampling for MAB with $[0, 1]$ rewards}\label{alg:MBTS-MAB}
\end{algorithm}

\begin{algorithm}[!h]
\SetAlgoLined
\KwData{
number of arms $K$, 
multiplier weight distribution $\rho(\omega)$, 
tuning parameter $\lambda$
}

Set $\mathcal{H}_{k} = \{\}$ be the history of the arm $k$, $c_k = 0$ ,and $\overline{Y}_{k} = +\infty, \forall k \in [K]$

\For{$t = 1, \dots, T$}{

    Offer $I_t = \arg \max_{k \in [K]} \overline{Y}_{k}$ (break tie randomly)

    Observe reward $R_t$
    
    Set $\mathcal{H}_{k} = \mathcal{H}_{k} \cup \{R_t\}$

    \For{$k = 1, \dots, K$}{
    \If{$|\mathcal{H}_{k}| > 0$}{
    
    Sample the multiplier weights $\{\omega_l\}_{l=1}^{|\mathcal{H}_{k}|} \sim \rho(\omega)$. 
    
    Update the mean reward $\overline{Y}_{k} = \left(\sum_{\ell = 1}^{|\mathcal{H}_{k}|} (\omega_{\ell} R_{k, \ell}+ \omega_{\ell} * \lambda + 0 * \lambda *  \omega_l ) \right) / \left( \sum_{\ell = 1}^{|\mathcal{H}_{k}|} (\omega_{\ell} + \lambda\omega_{\ell} + \lambda\omega_l) \right)$, where $R_{k, l}$ is the $l$-th element of $\mathcal{H}_{k}$ 
   

    }
    }
    \blue{Again, $\overline{Y}_{k}$ should concentrate on $\frac{0 + \lambda + \mu_k}{2\lambda + 1}$, which preserves the order between arms}
   
} 
\caption{Multiplier Bootstrap-based Thompson Sampling for MAB with $[0, 1]$ rewards \blue{[another infeasible version]}}\label{alg:MBTS-MAB}
\end{algorithm}


\begin{theorem}\label{thm}
    Consider a $K$-armed bandit where the reward distribution of arm $k$ is bounded in $[a, b]$ and the multiplier weight $\omega_{\ell} \sim \operatorname{Exp}(1)$. Then, the $T$ round regret of our method with the tuning parameter $\lambda_1 > 0$, $ 0 < \max_{k \in [K] / \{ 1\}} \Delta_k^{-1} \lambda_k < C^*$, and $\lambda_1 + C_* \Delta_k > 0$ for some $C_*, C^* \in \mathbb{R}$ satisfies:
    \begin{equation*}
        \begin{aligned}
            \operatorname{Reg}_T \leq 7 \sum_{k = 2}^K \Delta_k & + \sum_{k = 2}^K \max \left\{ \frac{(b - a)^2}{(\lambda_1 +  C_* \Delta_k)^2}, \, 4 \left( \frac{\lambda_1 + b}{\lambda_1}\right)^2, \, 4 \left( \frac{\Delta_k}{ \lambda_1 }\right)^2 \vee \left( \frac{\Delta_k}{\lambda_1 }\right) \right\} \log T \\
            & + \sum_{k = 2}^K\max \left\{ \frac{(b - a)^2 }{2( C^* \Delta_k -  \lambda_k)^2}, \, 4 \left( \frac{\lambda_k + b}{\lambda_k}\right)^2, \, \frac{8}{(\lambda_k + b)}\right\} \log T
        \end{aligned}
    \end{equation*}
    when the round $T$ is big enough.
\end{theorem}


\begin{remark}
    It is worthy to note that the proper tuning parameter $\lambda_k$ determining on the unknown structure $\Delta_k$.
\end{remark}

{\color{red}
\begin{remark}
    This theorem only give the infeasible algorithm since in practice all the arms share the equivalent standing, which means all $\lambda_k$ should be settled as the same. Hence, Algorithm 1 is an infeasible, while Algorithm 2 trade all the arm the same, it is an adjusted algorithm can give the feasible result. It is worthy to note that the different weights in Algorithm 2 cannot be forced to be the same. 
\end{remark}
}

{\color{red}
\begin{remark}
    In some remarks, we trade $\omega_i$ as the exponential weights since the by fact ${\mathbb{E}} \omega_i = 1$ will uniquely determine the distribution of $\omega_i$. However, the non-symmetric and non-negative property of the exponential distribution will make the analysis complicated. Hence, instead of exponential distribution, we assume $\omega_i \sim N(1, \sigma_{\omega}^2)$. Here $\sigma_{\omega}^2$ can be served as another tuning parameter.
\end{remark}
}

\begin{theorem}\label{thm.MAB}
    Consider a $K$-armed bandit where the reward distribution of arm $k$ is bounded in $[0, 1]$ and the multiplier weight $\omega_{\ell} \sim N(1, \sigma_{\omega}^2)$. Then, the $T$ round regret of our method with the tuning parameter $\lambda > 0$ and $\sigma_{\omega}^2 > 0$  satisfies:
    \begin{equation*}
        \operatorname{Reg}_T \leq 7 \sum_{k = 2}^K \Delta_k + \sum_{k = 2}^K \big[ G(\lambda, \sigma_{\omega}^2, C_1, C_2, \Delta_k) + H(\lambda, \sigma_{\omega}^2, C_1, C_3, \Delta_k) \big] \log T
    \end{equation*}
    with any round $T$.
\end{theorem}





\section{Recall Idea and Framework of the Proof}

\subsection{Setting}

Suppose each arms' rewards $\{ R_{k, \ell}\}$ are sub-Gaussian iid sample such that $\mu_k = {\mathbb{E}} R_{k, \ell} \in [0, 1]$. The quantity what we care is the cumulative regret 
\[
    \operatorname{Reg}_{T} := {\mathbb{E}} \operatorname{Reg}_{T}(Y,\mathcal{A}) = {\mathbb{E}} \sum_{t=1}^{T} [R_1 - R_{A_{t}}]=\sum_{k=2}^{K} \Delta_{k} {\mathbb{E}} \Big[\sum_{t=1}^{T} 1 \left\{A_{t}=k\right\}\Big],
\]
where we assume $\mu_1 = \max_{k \in [K]} \mu_k$. We want to show $\operatorname{Reg}_T$ can be bounded by order $\log T$ under our algorithm, specially under Algorithm 3.

\subsection{Basic Idea}

Here we want to use is the following lemma.


\begin{lemma}\label{lem.a.b}
Recall that we select arms according to the rule 
$A_t = \arg \max_{k \in [K]} \overline{Y}_{k, t}$. 
For any $\{ \tau_k \}_{k = 2}^K \subseteq \mathbb{R}^{K - 1}$, 
we have the bound
\[
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
\]
with
\[
    a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] =: \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \sum_{s = 0}^{T - 1} \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1 =: \sum_{s = 0}^{T - 1} b_{k, s} + 1
\]
where $Q_{k, s} (\tau) = \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_{k, T} \big)$ and $N_{k, s} (\tau) = 1 / Q_{k, s} (\tau) - 1$.
\end{lemma}

From the definition of $a_{k, s}$ and $b_{k, s}$, $a_k$ controls that the tail probability of the optimal arm is not too small, while $b_k$ controls that the tail probabilities of rewards of all arms are not too large in general. It is worthy to note that for here we only need to consider the lower bound of the tail of the optimal arm. The intuition is that we only need to the probability of taking a relatively large value in the optimal arm is not too small. And we do not care about how small the probability of a large value occurred in the other arms. And this Lemma is valid for any algorithm in MAB problem as long as the action step is $A_{t} = \arg \max_{k \in [K]} \overline{Y}_{k, t}$. 

Hence, in the following steps, we will bound $a_k$ and $b_k$ separately up to order $\log T$. 

\section{Bounding $a_k$}

For bounding $a_k$ up to order $\log T$, we first prove a weaker statement: $a_{k, s}$ can be bounded by some constant. For doing this, we note that if we take $\tau_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda}$, then as $Q_{1, s}(\cdot)$ is a decreasing function, we will have
\[
    a_{k, s} = {\mathbb{E}} \left[ \left( \frac{1}{Q_{1, s}(\tau_k)}  - 1\right) \wedge T \right] \leq {\mathbb{E}} \frac{1}{Q_{1, s}(\tau_k)} \leq {\mathbb{E}} Q_{1, s}^{-1} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right).
\]
Hence, we need to lower bound the tail probability $Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right) = \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right)$. For doing this, we just suppose $|\mathcal{H}_{1, T}| = s$ and we write this tail probability as
\[
    \begin{aligned}
        & \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \big( (2 \lambda + 1) R_{1, i} - (\mu_1 + \lambda) \big)\omega_i > \lambda (\mu_1 - 1 - \lambda) \sum_{i = 1}^s \omega_i' + \lambda (\lambda + \mu_1) \sum_{i = 1}^s \omega_i'' \right) \\
        \geq & \exp \bigg\{ - \frac{3s(2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left[s \lambda^2 (\lambda + 1 - \mu_1)^2 + s\lambda^2 (\lambda + \mu_1)^2\right]}\bigg\} \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~ + \big[ 1 - \Phi(\sqrt{2 \pi})\big] \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}.
    \end{aligned}
\]
by the following lemma.
\begin{lemma}
    Suppose $x_i, y_i, z_i \in \mathbb{R}$ and $\{ \omega_i, \omega_i', \omega_i''\}$ are independent $N(1, \sigma_{\omega}^2)$ random variable, then
    \[
        \begin{aligned}
            & \mathbb{P} \left( \sum_{i = 1}^n x_i \omega_i > \sum_{ i = 1}^n y_i \omega_i' + \sum_{i = 1}^n z_i \omega_i'' \right)\\
            \geq & \exp \left\{-\frac{3\left(S_{x}-S_{y}-S_{z}\right)^{2}}{2 \sigma_{\omega}^{2}\left(T_{x}+T_{y}+T_{z}\right)}\right\} \cdot 1\left(S_{x} \geq S_{y}+S_{z}+\sqrt{2 \pi}\right)+(1-\Phi(\sqrt{2 \pi})) \cdot 1\left(S_{x} \leq S_{y}+S_{z}+\sqrt{2 \pi}\right)
        \end{aligned}
    \]
\end{lemma}
And then
\[
    \begin{aligned}
        a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left[ (\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2\right]}\right\} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \\
        & \leq e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}
    \end{aligned}
\]
as long as we have
\begin{equation}
\label{cond.1}
    \frac{3 (2 \lambda + 1)^2}{2 \sigma_{\omega}^2 \left[ (\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2\right]} \leq \frac{s}{8 \sigma_1^2}.
\end{equation}
\begin{lemma}\label{lem_subGtosubE}
    Suppose $\{X_i\}_{i = 1}^n$ is iid  sub-Gaussian sample with mean $\mu$, then 
    \[
        {\mathbb{E}} \exp\left\{ \lambda(\overline{X} - \mu)^2 \right\} \leq \exp \left\{ \frac{64 \lambda^2 \sigma^4}{n^2} + \frac{\lambda \sigma^2}{n}\right\} \leq e^{9 / 8}
    \]
    for any $|\lambda| \leq \frac{n}{8 \sigma^2}$.
\end{lemma}

\begin{proof}
    Note that $X - \mu \sim \operatorname{subG} (\sigma^2)$, then by \cite{zhang2020concentration}, $\overline{X} - \mu \sim \operatorname{subG} (\sigma^2 / n)$ by
    \[
        {\mathbb{E}} \exp \left\{ s (\overline{X} - \mu) \right\} = \prod_{i = 1}^n \exp \left\{ \frac{s}{n} ({X}_i - \mu) \right\} \leq \prod_{i = 1}^n \exp \left\{ \frac{s^2}{n^2} \frac{\sigma^2}{2}\right\} = \exp \left\{ \frac{s^2 (\sigma^2 / n)}{2}\right\}.
    \]
    And hence by Proposition 4.3 in \cite{zhang2020concentration}, $(\overline{X} - \mu)^2 - n^{-1} \operatorname{var} (X) \sim \operatorname{subE} (8\sqrt{2}\sigma^2 / n, 8\sigma^2 / n)$. Therefore,
    \[
        \begin{aligned}
            {\mathbb{E}} \exp\left\{ \lambda(\overline{X} - \mu)^2 \right\} & = {\mathbb{E}} \exp\left\{ \lambda \big( (\overline{X} - \mu)^2 - n^{-1}\operatorname{var}(X) \big) \right\} {\mathbb{E}} \exp \{ \lambda n^{-1}  \operatorname{var}(X)\} \\
            & \leq \exp \left\{ \frac{(8 \sqrt{2} \sigma^2 / n)^2 \lambda^2}{2}\right\} \cdot \exp \left\{ \lambda n^{-1} \sigma^2 \right\} = \exp \left\{ \frac{64 \lambda^2 \sigma^4}{n^2} + \frac{\lambda \sigma^2}{n}\right\} \leq e^{9 / 8}.
        \end{aligned}
    \]
    for any $\lambda \leq \frac{n}{8 \sigma^2}$.
\end{proof}
Next, we denote the shift version of the average of reward, as well as others like previous. 

\subsection{Tackling $s_{a, 2}$}

Note that $\overline{R}_{1, s} - \mu_1 \sim \operatorname{subG} (\sigma^2 /  s)$, we will have
\[
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \left( \mu_1 - \overline{R}_{1, s} \geq (1 + 2\lambda) C_1 \Delta_k \right) \leq \exp \left\{ - \frac{(1 + 2\lambda)^2 C_1^2 \Delta_k^2 s}{2 \sigma_1^2}\right\} \leq \frac{1}{T^2}
    \end{aligned}
\]
as long as 
\[
    s \geq s_{a, 2} = \frac{\sigma_1^2}{C_1^2 (1 + 2 \lambda) \Delta_k^2} \cdot \log T.
\]
Hence, under this setting, we obtain that
\[
    a_{k, s, 2} \leq T {\mathbb{E}} 1(A_{1, s}^c) \leq T^{-1}.
\]
\subsection{Tackling $s_{a, 1, 2}$}

By denote the good events $G_{k', s} := \{ |\overline{Y}_{k', s} - \overline{R}^*_{k', s}| \leq C_3 \Delta_k\}$, then
\[
    \begin{aligned}
        \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, T}) = 2 \mathbb{P}_{\omega, \omega', \omega''} \left\{ (D_1 + D_2 + D_3) - C_3 (1 + 2\lambda) \Delta_k \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] > 0\right\}.
    \end{aligned}
\]
Denote the random function of $\{ R_{1,i}\}_{i = 1}^s$ as 
\[
   f\big(\{ R_{1,i}\}_{i = 1}^s\big) = f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) + f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) - f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big)
\]
with 
\[
    f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) = \sum_{i = 1}^s \big[ (1 + 2 \lambda) R_{1, i} - \lambda - \overline{R}_{1, s} - C_3 (1 + 2 \lambda) \Delta_k \big] \omega_i
\]\[
    f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) = \sum_{i = 1}^s \big[ \lambda (1 + 2 \lambda) - \lambda^2 - \overline{R}_{1, s} \lambda - \lambda C_3 (1 + 2 \lambda) \Delta_k \big] \omega_i'
\] \[
    f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big) =  \sum_{i = 1}^s \big[ (\lambda^2 + \lambda \overline{R}_{1, s}) + \lambda C_3 (1 + 2\lambda) \Delta_k \big] \omega_i''.
\]
By noting that $f_1, f_2, f_3$ are mutually independent with 
\[
    {\mathbb{E}} (f_1 + f_2 - f_3) = - C_3 (1 + 2 \lambda)^2 \Delta_k s
\] \[
    \begin{aligned}
        & \operatorname{var}(f_1 + f_2 - f_3) = \operatorname{var} f_1 + \operatorname{var} f_2 + \operatorname{var} f_3 \\
        = & \bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s})^2 + 4 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + 4 \lambda \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) (R_{1, i} - \mu_1) \\
        & \qquad + 2 \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big] \sum_{i = 1}^s (R_{1,i} - \overline{R}_{1, s}) \\
        & \qquad + 4 \lambda \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big] \sum_{ i =1}^s (R_{1, i} - \mu_1) + s \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big]^2 \bigg\} \\
        & + \lambda^2 \bigg\{ s^2 (\overline{R}_{1, s} - \mu_1)^2 + 2 \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big] s (\overline{R}_{1, s} - \mu_1) +  s \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \bigg\} \\
        & + \lambda^2 \bigg\{ s^2 (\overline{R}_{1, s} - \mu_1)^2 + 2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] s (\overline{R}_{1, s} - \mu_1) + s \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \bigg\} \\
        = & (1 + 4 \lambda^2 + 4 \lambda) \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + (-s - 4 \lambda s + \lambda^2 s^2 + \lambda^2 s^2) (\overline{R}_{1, s} - \mu_1)^2 + \Big\{ 4 \lambda \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big] \\
        & + 2 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] + 2 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big\} \sum_{i = 1}^s (R_{1, s} - \mu_1) \\
        & + \Big\{ s \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big]^2 + s \lambda^2 \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 + s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \Big\} \\
        =: & g_1(\lambda) \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + g_2(\lambda) (\overline{R}_{1, s} - \mu_1)^2 + g_3(\lambda) \sum_{i = 1}^s (R_{i, s} - \mu_1) + g_4(\lambda) =: g_1 + g_2 + g_3 + g_4,
    \end{aligned}
\]
the property of normal distribution implies
\[
    \begin{aligned}
        \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, T}) & = 2 \mathbb{P} \left(\frac{f_1 + f_2 - f_3 - {\mathbb{E}} (f_1 + f_2 - f_3)}{\operatorname{var} (f_1 + f_2 - f_3)} > - \frac{{\mathbb{E}} (f_1 + f_2 - f_3)}{\operatorname{var} (f_1 + f_2 - f_3)} \right) \\
        & \leq \sqrt{\frac{2}{\pi}} \cdot \frac{- {\mathbb{E}} (f_1 + f_2 - f_3) }{\operatorname{var} (f_1 + f_2 - f_3)} \exp \left\{ - \frac{{\mathbb{E}}^2 (f_1 + f_2 - f_3)}{2 \operatorname{var}^2 (f_1 + f_2 - f_3)^2} \right\} \\
        & \leq \sqrt{\frac{2}{\pi}} \frac{1}{8\lambda^2 \Delta_k} \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s^2 }{g_1 + g_2 + g_3 + g_4}\right\}.
    \end{aligned}
\]
{\color{red}{if we assume that $\overline{R}_{1, s} - \mu_1 \geq 0$}}. 
\begin{remark}
   Here the condition was assumed. But as we need ${R}_{1, i} - \mu_1$ is symmetric. We can just use this condition and then double the final upper bound..
\end{remark}
Here we use the fact that
\[
    \begin{aligned}
        \frac{|{\mathbb{E}} (f_1 + f_2 - f_3)}{\operatorname{var} (f_1 + f_2 - f_3)} & = \frac{|{\mathbb{E}} (f_1 + f_2 - f_3)|}{g_1 + g_2 + g_3 + g_4} \\
        (\text{by the inequality later})& \leq \frac{C_3 (1 + 2 \lambda)^2 \Delta_k}{2 \Big[ \sqrt{s} (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1) + \sqrt{2} \lambda \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big]^2} \\
        & \leq \frac{1}{8 \lambda^2 \Delta_k} 
    \end{aligned}
\]
On the other hand, we can reduce $g_1 + g_2 + g_3 + g_4$ to a single sub-Exponential variable. Indeed, 
\[
    \begin{aligned}
        g_1 + g_2 + g_3 + g_4 & = s^2 g_1(\lambda) (\overline{R}_{1, s} - \mu_1)^2 + g_2(\lambda) (\overline{R}_{1, s} - \mu_1)^2  + s g_3(\lambda) (\overline{R}_{1, s} - \mu_1) + g_4 \\
        & = \big[ s^2 g_1(\lambda) + g_2(\lambda)\big](\overline{R}_{1, s} - \mu_1)^2 + s g_3(\lambda) (\overline{R}_{1, s} - \mu_1) + g_4(\lambda) \\
        & \leq \Big\{ s^2(2 \lambda + 1)^2 + (2 \lambda^2 s^2 - 4 \lambda s - 1)\Big\} (\overline{R}_{1, s} - \mu_1)^2 \\
        & \qquad + \Big\{ s \cdot 4 \lambda \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + \lambda) + \lambda (\lambda + \mu_1) + C_3 \lambda (1 + 2 \lambda) \Delta_k \big] \Big\} (\overline{R}_{1, s} - \mu_1) \\
        & \qquad + 3 s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \\
        & \leq 2 s^2 (4 \lambda^2 + 1) (\overline{R}_{1, s} - \mu_1)^2 + 4 s \lambda \big[ (2 C_3 \Delta_k + 1) \lambda^2 + (3 \mu_1 - 1) \lambda - C_3 \big] (\overline{R}_{1, s} - \mu_1) \\
        & \qquad + 4 s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \\
        & \leq 2 s \Big[ \sqrt{s} (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1) + \sqrt{2} \lambda \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big]^2.
    \end{aligned}
\]
If we note that $\sqrt{s} (\overline{R}_{1, s} - \mu_1) \sim \operatorname{subG} (\sigma_1^2)$
\[
    \begin{aligned}
        & \Big[ \sqrt{s} (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1) + \sqrt{2} \lambda \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big]^2 \\
        & \qquad \qquad - 2\lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 - (2 \lambda + 1 )^2 \operatorname{var} (R_{1,s}) \,  \sim \, \operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2),
    \end{aligned}
\]
one can obtain that
\[
    \begin{aligned}
        a_{k, 1, 2} & = T {\mathbb{E}} \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, T}) \\
        & \leq \sqrt{\frac{2}{\pi}} \frac{T}{8 \lambda^2 \Delta_k} {\mathbb{E}} \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s^2 }{g_1 + g_2 + g_3 + g_4}\right\} \\
        & \leq \sqrt{\frac{2}{\pi}} \frac{T}{8 \lambda^2 \Delta_k} {\mathbb{E}} \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s}{2 (\operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2) + 2\lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + (2 \lambda + 1 )^2 \operatorname{var} (R_{1,s})) }\right\} \\
        & \leq \sqrt{\frac{2}{\pi}} \frac{T^2}{16 \lambda^2 \Delta_k} C_3^2 (1 + 2\lambda)^4 \Delta_k^2 \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s}{4 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + 2 (2 \lambda + 1 )^2 \sigma_1^2}\right\} (h_1 + h_2) \leq T^{-1}
    \end{aligned}
\]
as long as we take
\[
    s \geq s_{a, 1, 2} := \frac{12\lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + 6 (2 \lambda + 1 )^2 \sigma_1^2}{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s} \log \frac{\sqrt{2 / \pi}}{16 \lambda^2 \Delta_k (h_1 + h_2)} \log T
\]
by using the following two lemmas, where $h_1$ and $h_2$ are defined in Lemma \ref{lem_subE_ineq}.

\begin{lemma}
    Suppose $X$ be any zero-mean random variable, and $f$ be a differentiable function, then
    \[
        \begin{aligned}
            {\mathbb{E}} f(X) & = f(0) \big[ \mathbb{P} (X  > 0) - \mathbb{P} (X < 0) \big] + \int_0^{\infty} f'(u) \mathbb{P} (X > u) \, du - \int_{-\infty}^0 f'(u) \mathbb{P} (X < u) \, du \\
            & =  f(0) \big[ \mathbb{P} (X  > 0) - \mathbb{P} (X < 0) \big] + \int_0^{\infty} \big[ f'(u) - f'(-u) \big] \mathbb{P} (|X| > u) \, du.
        \end{aligned}
    \]
\end{lemma}

\begin{lemma}\label{lem_subE_ineq}
    Suppose $X \sim \operatorname{subE}(\lambda, \alpha)$ is {\color{red}{symmetric}} about zero, then
    \[
        {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\} \leq c e^{- c / b} \big[h_1(a, b, \alpha) + h_2(a, b, \lambda) \big]
    \]
    for any $a, b, c > 0$, where $h_1$ and $h_2$ are finite positive functions.
\end{lemma}

\begin{proof}
    Note that for any symmetric random variable $X$ and differentiable $f$, we have ${\mathbb{E}} f(X) = \int_0^{\infty} \big[ f'(u) - f'(-u) \big]\mathbb{P} (|X| > u) \, du$, then 
    \[ 
        \begin{aligned}
            & {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\} \\
            = & \int_0^{\infty} \bigg[ \exp \left\{ - \frac{c}{a u + b}\right\}  \frac{ca }{ (a u + b)^2} - \exp \left\{ - \frac{c}{ - a u + b}\right\}  \frac{ca }{ (- a u + b)^2} \bigg] \, \mathbb{P} (|X| > u) \, du \\
            \leq & \int_0^{\infty} \bigg[ \exp \left\{ - \frac{c}{a u + b}\right\}  \frac{ca }{ (a u + b)^2} - \exp \left\{\frac{c}{ a u - b}\right\}  \frac{ca }{ (a u - b)^2} \bigg] \, \exp \left\{ - \frac{1}{2} \left( \frac{u^2}{\lambda^2} \wedge\frac{u}{\alpha} \right) \right\} \, du \\
            = & \int_0^{\infty} \exp \left\{ - \frac{c}{a u + b}\right\} \bigg[   \frac{ca }{ (a u + b)^2} - \exp \left\{\frac{b + au}{ b - au}\right\}  \frac{ca }{ (a u - b)^2} \bigg] \, \exp \left\{ - \frac{1}{2} \left( \frac{u^2}{\lambda^2} \wedge\frac{u}{\alpha} \right) \right\} \, du \\
            \leq & \exp (-c / b) \int_0^{\infty} \Bigg[ \frac{ca}{(a u + b)^2} \exp \left\{ - \frac{u}{2 \alpha}\right\} + \frac{ca}{(a u - b)^2} \exp \left\{ - \frac{u^2}{2 \lambda^2} + \frac{b + au}{b - au}\right\} \Bigg] \, du \\
            =: &  \exp (-c / b) \big[ I_1 + I_2 \big]
        \end{aligned}
    \]
    Denote $\Gamma(\cdot, \cdot)$ as incomplete gamma function, then one can calculate that
    \[
        I_1 = \frac{c}{b} - \frac{c e^{\frac{b}{2 a \alpha}}\Gamma(0, \frac{b}{2 a \alpha})}{2 a \alpha} =: c h_1 (a, b, \alpha).
    \]
    For caculating $I_2$, we first note that 
    \[
        \lim_{u \rightarrow \frac{b}{a}} (a u - b)^{1 / 2}\frac{ca}{(a u - b)^2} \exp \left\{ - \frac{u^2}{2 \lambda^2} + \frac{b + au}{b - au}\right\} = \lim_{u \rightarrow \infty} u^{3 / 2}\exp \left\{ - \frac{u^2}{2 \lambda^2} + \frac{b + au}{b - au}\right\} = 0, \qquad 
    \]
    Hence, $\exists \, I_2 \in \mathbb{R}$, and then
    \[
        \exists \, I_2 = c \int_0^{\infty} \frac{a}{(a u - b)^2} \exp \left\{ - \frac{u^2}{2 \lambda^2} + \frac{b + au}{b - au}\right\} =: c h_2(a, b, \lambda) \in c \times (0, \infty)
    \]
    with $h_1$ and $h_2$ are finite positive functions.
\end{proof}

\begin{remark}
   The symmetirc condition cannot be dropped. A simple conterexample is $X \equiv - b /a $.
\end{remark}

\subsection{Tackling $s_{a, 2, 2}$}

Note that under good event $A_{1, s} \cap G_{1, s}$ it will absolutely bound the difference $\Gamma_1 := \tau_1 - \overline{R}_{1, s}^*$ in $(-4, - C_1 \Delta_k)$, hence the tail probability we care is 
\[
    \begin{aligned}
        & \big[ 1 - Q_{1,s} (\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \left\{ (D_1 + D_2 + D_3) - \Gamma_1 (1 + 2\lambda)  \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] \leq 0\right\} 1(A_{1, s} \cap G_{1, s}).
    \end{aligned}
\]
What we remain is replacing $C_3 \Delta_k$ by $\Gamma_1$ in section 2.2. By using the same technique we get that
\[
    \begin{aligned}
        & \big[ 1 - Q_{1,s} (\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \\
        \leq & \sqrt{\frac{1}{2\pi}} \frac{T}{8 \lambda^2 \Delta_k} {\mathbb{E}} \exp \left\{ - \frac{\Gamma_1^2 (1 + 2\lambda)^4 s}{2 (\operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2) + 2\lambda^2 \big[ (\lambda + \mu_1) +  (1 + 2 \lambda) \Gamma_1 \big]^2 + (2 \lambda + 1 )^2 \sigma_1^2) }\right\} \\
        \leq & \sqrt{\frac{1}{2\pi}} \frac{T}{8 \lambda^2 \Delta_k} {\mathbb{E}} \exp \left\{ - \frac{C_1^2 \Delta_k^2 (1 + 2\lambda)^4 s}{2 (\operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2) + 2\lambda^2 \big[ (\lambda + \mu_1) -  (1 + 2 \lambda) 4 \big]^2 + (2 \lambda + 1 )^2 \sigma_1^2) }\right\} \\
        \leq & \sqrt{\frac{2}{\pi}} \frac{T^2}{16 \lambda^2 \Delta_k} C_1^2 (1 + 2\lambda)^4 \Delta_k^2 \exp \left\{ - \frac{C_1^2 (1 + 2\lambda)^4 \Delta_k^2 s}{4 \lambda^2 (7 \lambda + 4 - \mu_1)^2 + 2 (2 \lambda + 1 )^2 \sigma_1^2}\right\} (h_1^* + h_2^*) \leq T^{-1}
    \end{aligned}
\]
whenever
\[
   s \geq s_{a, 1, 1} := \frac{12\lambda^2 (7 \lambda + 4 - \mu_1)^2 + 6 (2 \lambda + 1 )^2 \sigma_1^2}{C_1^2 (1 + 2\lambda)^4 \Delta_k^2 s} \log \frac{\sqrt{2 / \pi}}{16 \lambda^2 \Delta_k (h_1^* + h_2^*)} \log T.
\]
Combining all results in Section 2, we get that 
\[
    a_{k} \leq \max _{s} a_{k, s} \cdot \max _{j} s_{a, j} + 3 T^{-1} \cdot\big(T-\max _{j} s_{a, j}\big) \lesssim \log T.
\]
\begin{remark}
   Here, we do not give the explicit expresion of the bound as it involves many terms, while its order is exactly $\log T$.
\end{remark}

\section{Bounding $b_k$}

As argued previously, one only need to tackle $b_{k, 1, 1}$. By using the same argument, we have the similar bound $C_2 \Delta_k < \Gamma_k < 4$. Therefore,
\[
    \begin{aligned}
        & Q_{k, s}\left(\tau_{k}\right) 1\left(A_{k, s}^{\prime}\right) 1\left(G_{k, s}\right) = \mathbb{P} \left(\overline{Y}_{k, s}-\overline{R}_{k, s}^{*}>\Gamma_{k} \mid \mathcal{H}_{k, T}\right) 1\left(A_{k, s}^{\prime}\right) 1\left(G_{k, s}\right) \\
        = & \mathbb{P}_{\omega, \omega^{\prime}, \omega^{\prime \prime}}\left\{\left(D_{1}+D_{2}+D_{3}\right)-\Gamma_{k}(1+2 \lambda)\left[\sum_{i=1}^{s} \omega_{i}+\lambda \sum_{i=1}^{s} \omega_{i}^{\prime}+\lambda \sum_{i=1}^{s} \omega_{i}^{\prime \prime}\right]>0\right\} 1\left(A_{k, s}^{\prime} \cap G_{k, s}\right)\\
        \leq & \sqrt{\frac{1}{2\pi}} \frac{1}{8 \lambda^2 \Delta_k} {\mathbb{E}} \exp \left\{ - \frac{\Gamma_k^2 (1 + 2\lambda)^4 s}{2 (\operatorname{subE} (8\sqrt{2}\sigma_k^2, 8\sigma_k^2) + 2\lambda^2 \big[ (\lambda + \mu_k) +  (1 + 2 \lambda) \Gamma_k \big]^2 + (2 \lambda + 1 )^2 \sigma_k^2) }\right\} \\
        \leq & \sqrt{\frac{1}{2\pi}} \frac{1}{8 \lambda^2 \Delta_k} {\mathbb{E}} \exp \left\{ - \frac{C_2^2 \Delta_k^2 (1 + 2\lambda)^4 s}{2 (\operatorname{subE} (8\sqrt{2}\sigma_k^2, 8\sigma_k^2) + 2\lambda^2 \big[ (\lambda + \mu_k) + (1 + 2 \lambda) 4 \big]^2 + (2 \lambda + 1 )^2 \sigma_k^2) }\right\} \\
        \leq & \sqrt{\frac{2}{\pi}} \frac{T}{16 \lambda^2 \Delta_k} C_2^2 (1 + 2\lambda)^4 \Delta_k^2 \exp \left\{ - \frac{C_1^2 (1 + 2\lambda)^4 \Delta_k^2 s}{4 \lambda^2 (9 \lambda + 4 + \mu_k)^2 + 2 (2 \lambda + 1 )^2 \sigma_k^2}\right\} (h_1^{**} + h_2^{**}).
    \end{aligned}
\]
Therefore, one can just take 
\[
    s \geq s_{b, 1, 1} :=  \frac{8\lambda^2 (9 \lambda + 4 + \mu_k)^2 + 4 (2 \lambda + 1 )^2 \sigma_k^2}{C_2^2 (1 + 2\lambda)^4 \Delta_k^2 s} \log \frac{\sqrt{2 / \pi}}{16 \lambda^2 \Delta_k (h_1^{**} + h_2^{**})} \log T
\]
and then $b_k \lesssim \log T$. As all the restrict conditions for $C_1, C_2, C_3$ are exactly the same as previous, we still take $C_1 = C_2 = \frac{1}{8}$ and $C_3 = \frac{1}{16}$. And the final bounds need to be doubled.




\end{document}
\section{Framework of the Proof}


\subsection{An important contribution compared to the existent work}

In \textit{{MOTS: Minimax Optimal Thompson Sampling}}, they purpose an algorithm for $1$-sub-Gaussian rewards in the setting of MAB, and claim they also proof it has the optimal rate. However, their result can ONLY for PROBLEM-INDEPENDENT optimal in ASYMPTOTIC sense, and their method cannot used for  problem-dependent problem: the $I_2$ in their proof can only bounded $\sqrt{T}$ due to CLT. Whereas our method can be applied to PROBLEM-DEPENDENT optimal case and our result is NONASYMPTOTIC.

{More compared to existent theoretical results.}

   
   
   

\subsection{Setting}

We call a \textit{zero-mean} random variable $X$ is sub-Gaussian with parameter $\sigma^2$ if 
\[
    {\mathbb{E}} \exp \{t X \} \leq \exp \left\{ {t^2\sigma^2}/{2}\right\}, \qquad t \in \mathbb{R}.
\]
In this case, we denote $X \sim \operatorname{subG}(\sigma^2)$. A direct result from the definition of sub-Gaussian variable is the concentration inequality from Chernoff inequality that
\[
    \mathbb{P} (X \geq t) \leq \exp \left\{ - \frac{t^2}{2 \sigma^2}\right\}, \qquad t \geq 0.
\]
It can be seen that the sub-Gaussian variable have the similar tail with Gaussian variable. This is the reason we call it sub-Gaussian. Another important fact of sub-Gaussian variable is the sub-Gaussian parameter is not less than the true variance $\sigma^2 \geq \operatorname{var} (X)$.

Similarly, we can define the sub-exponential variable that mimics the tail of the exponential random variable. To specific, we call a zero-mean variable $X$ is sub-exponential with parameter $\lambda$ and $\alpha$ if
\[
    {\mathbb{E}} \exp \{ t X \} \leq \exp \{ t^2 \lambda^2 / 2\}, \qquad |t| \leq \frac{1}{\alpha}
\]
and we denote $X \sim \operatorname{subE}(\lambda, \alpha)$. Again, we have the following concentration 
\[
    \mathbb{P} (X \geq t) \leq \exp \left\{ - \frac{1}{2} \left( \frac{t^2}{\lambda^2} \wedge \frac{t}{\alpha}\right) \right\}, \qquad t \geq 0.
\]
Now suppose each arms' rewards' setting is as follows
\begin{itemize}
    \item $\{ R_{k, \ell}\}$ are sub-Gaussian independent sample such that $\mu_k = {\mathbb{E}} R_{k, \ell} \in [0, 1]$ with finite sub-Gaussian parameter $\sigma_k^2 $.
\end{itemize}

The quantity of what we care about is the cumulative regret 
\[
    \operatorname{Reg}_{T} := {\mathbb{E}} \operatorname{Reg}_{T}(Y,\mathcal{A}) = {\mathbb{E}} \sum_{t=1}^{T} [R_1 - R_{A_{t}}]=\sum_{k=2}^{K} \Delta_{k} {\mathbb{E}} \Big[\sum_{t=1}^{T} 1 \left\{A_{t}=k\right\}\Big],
\]
where we assume $\mu_1 = \max_{k \in [K]} \mu_k$, i.e. $1$-th arm is the "best" arm with the maximal expected reward. We want to show $\operatorname{Reg}_T$ can be bounded by order $\log T$ for any $T \in \mathbb{N}$ under our algorithm, specially under Algorithm 3.

\subsection{Basic Idea}

Here we want to use is the following lemma.


\begin{lemma}\label{lem.a.b}
Recall that we select arms according to the rule 
$A_t = \arg \max_{k \in [K]} \overline{Y}_{k, t}$. 
For any $\{ \tau_k \}_{k = 2}^K \subseteq \mathbb{R}^{K - 1}$, 
we have the bound
\[
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
\]
with
\[
    a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] =: \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \sum_{s = 0}^{T - 1} \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1 =: \sum_{s = 0}^{T - 1} b_{k, s} + 1
\]
where $Q_{k, s} (\tau) = \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_{k, s} \big)$ and $N_{k, s} (\tau) = 1 / Q_{k, s} (\tau) - 1$ with $\mathcal{H}_{k, s}$ is the history $s$ rewards of $k$-th arm.
\end{lemma}
Here $s$ in the summation index can be reviewed as we pull $k$-th arm $s$ times under $T$ rounds. From the definition of $a_{k, s}$ and $b_{k, s}$, $a_k$ controls that the tail probability of the optimal arm is not too small, while $b_k$ controls that the tail probabilities of rewards of all arms are not too large in general. It is worthy to note that for here we only need to consider the lower bound of the tail of the optimal arm. The intuition is that we only need to the probability of taking a relatively large value in the optimal arm is not too small. And we do not care about how small the probability of a large value occurred in the other arms. And this Lemma is valid for any algorithm in MAB problem as long as the action step is $A_{t} = \arg \max_{k \in [K]} \overline{Y}_{k, t}$. 

Hence, in the following steps, we will bound $a_k$ and $b_k$ separately up to order $\log T$. We denote two functions $a(s, T) \lesssim b(s, T)$ if $a(s, T) \leq c \times b(s, T)$ for $c$ free of $s$ and $T$. And we write $a(s, T) \asymp b(s, T)$ if $a(s, T) \lesssim b(s, T)$ and $a(s, T) \gtrsim b(s, T)$.

\section{Bounding $a_k$}

{\color{blue}{State we can only deal with $a_{k, s}$ for $s$ large enough}}

For bounding $a_k$ up to order $\log T$, we first prove a weaker statement: $a_{k, s}$ can be bounded by some constants free of $T$, which will give the weaker bound $a_{k} \lesssim T$.
For doing this, we note that if we take
\begin{equation}\label{requirement_0}
    \tau_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda},
\end{equation}
then we will have
\begin{equation}\label{constant_bound}
    a_{k, s} = {\mathbb{E}} \left[ \left( \frac{1}{Q_{1, s}(\tau_k)}  - 1\right) \wedge T \right] \leq {\mathbb{E}} \frac{1}{Q_{1, s}(\tau_k)} \leq {\mathbb{E}} Q_{1, s}^{-1} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right),
\end{equation}
where the second inequality we use the fact that $Q_{1, s}(\cdot)$ is a decreasing function. Hence, we need to lower bound the tail probability $Q_{1, s} \left( \frac{\mu_1 + \lambda}{1 + 2\lambda}\right) = \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right)$. The tail probability can be written as weighted summation of Gaussian bootstrap weights, i.e.
\[
    \begin{aligned}
        & \mathbb{P} \left( \overline{Y}_{1, s} > \frac{\mu_1 + \lambda}{1 + 2\lambda} \mid \mathcal{H}_{1, s}\right) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \left( \sum_{i = 1}^s \big( (2 \lambda + 1) R_{1, i} - (\mu_1 + \lambda) \big)\omega_i > \lambda (\mu_1 - 1 - \lambda) \sum_{i = 1}^s \omega_i' + \lambda (\lambda + \mu_1) \sum_{i = 1}^s \omega_i'' \right) \\
        \geq & \exp \bigg\{ - \frac{3s(2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left[s \lambda^2 (\lambda + 1 - \mu_1)^2 + s\lambda^2 (\lambda + \mu_1)^2\right]}\bigg\} \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 \geq \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}  \\
        & ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~ + \big[ 1 - \Phi(\sqrt{2 \pi})\big] \cdot 1 \left\{ \overline{R}_{1, s} - \mu_1 < \frac{\sqrt{2 \pi}}{s (2 \lambda + 1)}\right\}.
    \end{aligned}
\]
by the following lemma.
\begin{lemma}
    Suppose $x_i, y_i, z_i \in \mathbb{R}$ and $\{ \omega_i, \omega_i', \omega_i''\}$ are independent $N(1, \sigma_{\omega}^2)$ random variable, then
    \[
        \begin{aligned}
            & \mathbb{P} \left( \sum_{i = 1}^n x_i \omega_i > \sum_{ i = 1}^n y_i \omega_i' + \sum_{i = 1}^n z_i \omega_i'' \right)\\
            \geq & \exp \left\{-\frac{3\left(S_{x}-S_{y}-S_{z}\right)^{2}}{2 \sigma_{\omega}^{2}\left(T_{x}+T_{y}+T_{z}\right)}\right\} \cdot 1\left(S_{x} \geq S_{y}+S_{z}+\sqrt{2 \pi}\right)+(1-\Phi(\sqrt{2 \pi})) \cdot 1\left(S_{x} \leq S_{y}+S_{z}+\sqrt{2 \pi}\right)
        \end{aligned}
    \]
\end{lemma}
\begin{proof}
    Obviously.
\end{proof}
And as $a_{k,s}$ is bounded by the reciprocal of the lower bound of the tail probability in \eqref{constant_bound}, we obtain that 
\begin{equation}\label{a_k_constant}
    \begin{aligned}
        a_{k, s} & \leq {\mathbb{E}} \exp \left\{ \frac{3 (2 \lambda + 1)^2 (\overline{R}_{1, s} - \mu_1)^2}{2 \sigma_{\omega}^2 \left[ (\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2\right]}\right\} + \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1} \\
        & \leq e^{9 / 8} +  \big[ 1 - \Phi(\sqrt{2 \pi})\big]^{-1}
    \end{aligned}
\end{equation}
as long as we have
\begin{equation}
\label{cond.1}
    \frac{3 (2 \lambda + 1)^2}{2 \sigma_{\omega}^2 \left[ (\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2\right]} \leq \frac{s}{8 \sigma_1^2}.
\end{equation}
by the fact that ${R}_{1, i} - \mu_1$ for $i \in [s]$ is sub-Gaussian with the following lemma.
\begin{lemma}\label{lem_subGtosubE}
    Suppose $\{X_i\}_{i = 1}^n$ is iid  sub-Gaussian sample with mean $\mu$, then 
    \[
        {\mathbb{E}} \exp\left\{ \lambda(\overline{X} - \mu)^2 \right\} \leq \exp \left\{ \frac{64 \lambda^2 \sigma^4}{n^2} + \frac{\lambda \sigma^2}{n}\right\} \leq e^{9 / 8}
    \]
    for any $|\lambda| \leq \frac{n}{8 \sigma^2}$.
\end{lemma}
\begin{proof}
    Note that $X - \mu \sim \operatorname{subG} (\sigma^2)$, $\overline{X} - \mu \sim \operatorname{subG} (\sigma^2 / n)$ by the fact that
    \[
        {\mathbb{E}} \exp \left\{ s (\overline{X} - \mu) \right\} = \prod_{i = 1}^n \exp \left\{ \frac{s}{n} ({X}_i - \mu) \right\} \leq \prod_{i = 1}^n \exp \left\{ \frac{s^2}{n^2} \frac{\sigma^2}{2}\right\} = \exp \left\{ \frac{s^2 (\sigma^2 / n)}{2}\right\}.
    \]
    And hence by Proposition 4.3 in \cite{zhang2020concentration}, $(\overline{X} - \mu)^2 - n^{-1} \operatorname{var} (X) \sim \operatorname{subE} (8\sqrt{2}\sigma^2 / n, 8\sigma^2 / n)$. Therefore,
    \[
        \begin{aligned}
            {\mathbb{E}} \exp\left\{ \lambda(\overline{X} - \mu)^2 \right\} & = {\mathbb{E}} \exp\left\{ \lambda \big( (\overline{X} - \mu)^2 - n^{-1}\operatorname{var}(X) \big) \right\} {\mathbb{E}} \exp \{ \lambda n^{-1}  \operatorname{var}(X)\} \\
            & \leq \exp \left\{ \frac{(8 \sqrt{2} \sigma^2 / n)^2 \lambda^2}{2}\right\} \cdot \exp \left\{ \lambda n^{-1} \sigma^2 \right\} = \exp \left\{ \frac{64 \lambda^2 \sigma^4}{n^2} + \frac{\lambda \sigma^2}{n}\right\} \leq e^{9 / 8}.
        \end{aligned}
    \]
    for any $\lambda \leq \frac{n}{8 \sigma^2}$.
\end{proof}

\begin{remark}
   Lemma \ref{lem_subGtosubE} actually is a special case for the fact that the mean of iid sub-Weibull sample with sample size $n$ will have the local sub-Gaussian property in the interval up to $\sqrt{n}$.
\end{remark}
{\color{blue}{More details about the tuning parameters. }}
Hence, for bound $a_{k, s}$ with a constant, one only need to ensure that
\begin{equation}\label{tuning_condition}
    \frac{3 (2 \lambda + 1)^2}{2 \sigma_{\omega}^2 \left[ (\lambda + 1 - \mu_1)^2 + (\lambda + \mu_1)^2\right]} \leq \frac{C}{8 \sigma_1^2}.
\end{equation}
as $s \in [T]$ can be take as a constant free of $T$ large enough to ensure that $s \geq C$. A sufficient condition for this is
\[
    \frac{3 (2 \lambda + 1)^2}{2 \sigma_{\omega}^2 \cdot 2 \lambda^2} \leq \frac{C}{8 \sigma_1^2}.
\]
Hence, the regularity condition for tuning parameter $(\lambda, \sigma_{\omega}^2)$ is $4 (\sigma_{\omega}^2 - 1) \lambda^2 - 4 \lambda -1 \geq 0$ with $\sigma_{\omega}^2 > 1$, i.e. 
\begin{equation}\label{tuning_condition_exact}
    \lambda \geq \frac{1}{2}\frac{\sigma_{\omega}^2 + 1}{ \sigma_{\omega}^2 - 1} \qquad \text{ with } \qquad \sigma_{\omega}^2 > 1.
\end{equation}






\begin{remark}
   Different from the bounded rewards setting, here $\lambda$ together with the bound of $\mu_1$ as well as $\sigma_{\omega}^2$ and $\sigma_k^2$ establish the restriction condition for the tuning parameters. But \eqref{tuning_condition_exact} can be served as a sufficient condition.
\end{remark}

Next, we decompose $a_{k, s}$ as positive several simple expressions $a_{k, s, 1}$, $a_{k, s, 2}$, and $a_{k, s, 3}$ i.e. $a_{k, s} = a_{k, s, 1} + a_{k, s, 2} + a_{k, s, 3}$, and we can focus on these simpler components. Following the idea in \cite{kveton2019garbage}, we will try to bound 
\[
    s_{a, j} = \max \{ s: a_{k, s, j} \geq T^{-1}\}, \qquad j = 1, 2, 3
\]
as order $\log T$. Indeed, if we successfully do so, then 
\[
    a_{k, s, j} \leq a_{k, s} \times s_{a, j} + T^{-1} \times (T - s_{a, j}) \lesssim \log T
\]
gives an order $\log T$. For decomposition, a common approach is using indicators on good events. We define the following good events for $k$-th arm
\[
    A_{k, s} = \left\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2 \lambda}  > - C_1 \Delta_k \right\}, \qquad A_{k, s}^{\prime} = \left\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2 \lambda}  < C_2 \Delta_k \right\}, 
\]
and
\[
    G_{k', s} := \left\{ - C_3 \Delta_k \leq \overline{Y}_{k', s} - \overline{R}_{k', s}^* \leq C_3 \Delta_k \right\}
\]
with the shifted version mean defined as 
\[
    \overline{R}_{k, s}^* = \frac{\sum_{i = 1}^s R_{k, i} + \lambda s}{s(1 + 2 \lambda)} = \frac{\lambda}{1 + 2 \lambda} + \frac{1}{1 + 2 \lambda} \overline{R}_{k, s}.
\]
It is worthy to note that $k'$ in $G_{k', s}$ is different from the fixed index $k$ in $\Delta_k$. And it is also easy to see that under two good events $A_{k', s} \cap G_{k', s}$, our target function $\overline{Y}_{k', s}$ is close to the shifted mean $\frac{\mu_{k'} + \lambda}{1 + 2\lambda}$, which means $\overline{Y}_{k', s}$ does remain the order of expected mean of each arm. Therefore, we decompose $a_{k, s}$ as the following three parts
\[
    a_{k, s, 1} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1(A_{1, s}^c)\Big],
\]
\[
    a_{k, s, 2} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1(A_{1, s}) 1 (G_{1, s}^c)\Big], \qquad a_{k, s, 3} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1(A_{1, s}) 1 (G_{1, s})\Big].
\]
As we can see, $a_{k, s, 1}$ and $a_{k, s, 2}$ defined on "bad" events and hence we will bound the tail probability $\mathbb{P} (A_{1, s}^c)$ and $\mathbb{P} (G_{1, s}^c)$, while $a_{k, s, 3}$ defined on good events will need extra steps to bound (we will specify it later). 

\subsection{Tackling $a_{k, s, 1}$ with $s_{a, 1}$}

Since the summation of independent sub-Gaussian variable is still sub-Gaussian (see Lemma \ref{lem_subGtosubE}), we can easily find that $\overline{R}_{1, s} - \mu_1 \sim \operatorname{subG} (\sigma^2 /  s)$. And hence
\[
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \left( \mu_1 - \overline{R}_{1, s} \geq (1 + 2\lambda) C_1 \Delta_k \right) \leq \exp \left\{ - \frac{(1 + 2\lambda)^2 C_1^2 \Delta_k^2 s}{2 \sigma_1^2}\right\} \leq \frac{1}{T^2}
    \end{aligned}
\]
as long as 
\[
    s \geq s_{a, 2} := \frac{\sigma_1^2}{C_1^2 (1 + 2 \lambda) \Delta_k^2} \cdot \log T.
\]
Hence, under this setting, we obtain that
\[
    a_{k, s, 1} \leq {\mathbb{E}} \Big[ T  1(A_{1, s}^c)\Big] =  T \mathbb{P}(A_{1, s}^c) \leq T^{-1}.
\]

\subsection{Tackling $a_{k, s, 2}$ with $s_{a, 2}$}

As $G_{k', s} = \{ |\overline{Y}_{k', s} - \overline{R}^*_{k', s}| \leq C_3 \Delta_k\}$ is a symmetric event, we have 
\[
    \begin{aligned}
        \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s}) & = 2 \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) \omega_i + \lambda \sum_{i = 1}^s (2 R_{1, i} \omega_i - \overline{R}_{1, s} \omega_i' - \overline{R}_{1, s} \omega_i'') 
        \\
        & + \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s (\omega_i' - \omega''_i) - C_3 (1 + 2\lambda) \Delta_k \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] > 0\Bigg\}.
    \end{aligned}
\]
Denote the random function of $\{ R_{1,i}\}_{i = 1}^s$ as 
\[
   f\big(\{ R_{1,i}\}_{i = 1}^s\big) = f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) + f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) - f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big)
\]
with 
\[
    f_1 \big(\{ R_{1,i}\}_{i = 1}^s \big) = \sum_{i = 1}^s \big[ (1 + 2 \lambda) R_{1, i} - \lambda - \overline{R}_{1, s} - C_3 (1 + 2 \lambda) \Delta_k \big] \omega_i
\]\[
    f_2 \big(\{ R_{1,i}\}_{i = 1}^s \big) = \sum_{i = 1}^s \big[ \lambda (1 + 2 \lambda) - \lambda^2 - \overline{R}_{1, s} \lambda - \lambda C_3 (1 + 2 \lambda) \Delta_k \big] \omega_i'
\] \[
    f_3 \big(\{ R_{1,i}\}_{i = 1}^s \big) =  \sum_{i = 1}^s \big[ (\lambda^2 + \lambda \overline{R}_{1, s}) + \lambda C_3 (1 + 2\lambda) \Delta_k \big] \omega_i'',
\]
then $\mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s}) = \mathbb{P}_{\omega, \omega', \omega'} (f_1 + f_2 - f_3 > 0)$. Since $f_1, f_2, f_3$ are mutually independent, the expectation and variance are
\[
    {\mathbb{E}} (f_1 + f_2 - f_3) = - C_3 (1 + 2 \lambda)^2 \Delta_k s
\] 
and
\[
    \begin{aligned}
        & \operatorname{var}(f_1 + f_2 - f_3) = \operatorname{var} f_1 + \operatorname{var} f_2 + \operatorname{var} f_3 \\
        = & \bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s})^2 + 4 \lambda^2 \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + 4 \lambda \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) (R_{1, i} - \mu_1) \\
        & \qquad + 2 \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big] \sum_{i = 1}^s (R_{1,i} - \overline{R}_{1, s}) \\
        & \qquad + 4 \lambda \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big] \sum_{ i =1}^s (R_{1, i} - \mu_1) + s \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big]^2 \bigg\} \\
        & + \lambda^2 \bigg\{ s^2 (\overline{R}_{1, s} - \mu_1)^2 + 2 \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big] s (\overline{R}_{1, s} - \mu_1) +  s \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \bigg\} \\
        & + \lambda^2 \bigg\{ s^2 (\overline{R}_{1, s} - \mu_1)^2 + 2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] s (\overline{R}_{1, s} - \mu_1) + s \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \bigg\} \\
        = & (1 + 4 \lambda^2 + 4 \lambda) \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + (-s - 4 \lambda s + \lambda^2 s^2 + \lambda^2 s^2) (\overline{R}_{1, s} - \mu_1)^2 + \Big\{ 4 \lambda \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big] \\
        & + 2 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] + 2 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big\} \sum_{i = 1}^s (R_{1, s} - \mu_1) \\
        & + \Big\{ s \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + 2\lambda) \Delta_k \big]^2 + s \lambda^2 \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \\
        & + s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \Big\} \\
        =: & g_1(\lambda) \sum_{i = 1}^s (R_{1, i} - \mu_1)^2 + g_2(s, \lambda) (\overline{R}_{1, s} - \mu_1)^2 + g_3(\lambda) \sum_{i = 1}^s (R_{i, s} - \mu_1) + g_4(s, \lambda) \\
        =: & \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + \mathbb{G}_{3}(s, \lambda) + g_4(s, \lambda)
    \end{aligned}
\]
here the hollow font $\mathbb{G}_{\cdot}$ represents random quantities and $g_{\cdot}$ is non-random.

The $\mathbb{G}_j$ for $j = 1, 2, 3$ and $g_4$ have relatively complex expression, we first want to simply and give some upper bounds of them. First, we note that $\operatorname{var} (f_1 + f_2 - f_3)$ is strictly positive,
\begin{equation}\label{G_function_bound}
    \begin{aligned}
        & \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + \mathbb{G}_{3}(s, \lambda) + g_4(s, \lambda) \\
        = & s^2 g_1(\lambda) (\overline{R}_{1, s} - \mu_1)^2 + g_2(s, \lambda) (\overline{R}_{1, s} - \mu_1)^2  + s g_3(\lambda) (\overline{R}_{1, s} - \mu_1) + g_4(s, \lambda) \\
        = & \big[ s^2 g_1(\lambda) + g_2(s, \lambda)\big](\overline{R}_{1, s} - \mu_1)^2 + s g_3(\lambda) (\overline{R}_{1, s} - \mu_1) + g_4(s, \lambda) \\
        \leq & \Big\{ s^2(2 \lambda + 1)^2 + (2 \lambda^2 s^2 - 4 \lambda s - 1)\Big\} (\overline{R}_{1, s} - \mu_1)^2 \\
        & \qquad + \Big\{ s \cdot 4 \lambda \big[ \lambda(2 \mu_1 - 1) - C_3 (1 + \lambda) + \lambda (\lambda + \mu_1) + C_3 \lambda (1 + 2 \lambda) \Delta_k \big] \Big\} (\overline{R}_{1, s} - \mu_1) \\
        & \qquad + s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \\
        & \qquad + s \lambda^2 \Big\{ \big[ (2 \mu_1 - 1) - C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \Big\} \\
        \leq & 2 s^2 (6 \lambda^2 + 4 \lambda + 1) (\overline{R}_{1, s} - \mu_1)^2 + 4 s \lambda \big[ (2 C_3 \Delta_k + 1) \lambda^2 + (3 \mu_1 - 1 - C_3 + C_3 \Delta_k) \lambda - C_3 \big] (\overline{R}_{1, s} - \mu_1) \\
        & \qquad + s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 \\
        & \qquad + s \lambda^2 \Big\{ \big[ (2 \mu_1 - 1) - C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \Big\} \\
        \leq & 2s^2 (4 \lambda^2 + 4 \lambda + 1) (\overline{R}_{1, s} - \mu_1)^2  + 2 s \lambda (2 \lambda + 1) \cdot \lambda \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \sqrt{s} (\overline{R}_{1, s} - \mu_1) \\
        & \qquad + s \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + g_5(s, \lambda)\\
        \leq & 2 s \Big[ \sqrt{s} (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1) +  \lambda \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big]^2 + g_5(s, \lambda),
    \end{aligned}
\end{equation}
where the last two $\leq$ use the fact that $s \geq 1$ and $\mu_1, C_3 \in (0, 1]$. And furthermore, $g_5(s, \lambda) = s \lambda^2 \Big\{ \big[ (2 \mu_1 - 1) - C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ \mu_1 + C_3 (1 + 2 \lambda) \Delta_k - (1 + \lambda) \big]^2 \Big\}$ is a strict positive function for at least $\Delta_{k_1}, \ldots, \Delta_{k_{K - 3}}$, since
\[
    \frac{C_3 \Delta_k}{2(\mu_1 - C_3 \Delta_k) - 1} = \frac{1 - \mu_1 - C_3 \Delta_k}{2 C_3 \Delta_k - 1}.
\]
is a quadratic equation for $\Delta_k$ and cannot hold for all $k \in [K] / \{ 1\}$, and $g_5(s, \lambda)$ can be furthermore bounded by
\[
    \begin{aligned}
        g_5(s, \lambda) & \leq s \lambda^2 \Big( \big[ 1 + C_3 (\lambda^{-1} + 2) \Delta_k \big]^2 + \big[ 1 + C_3 (1 + 2 \lambda) \Delta_k + (1 + \lambda) \big]^2 \Big) \\
        & \leq 2 s \lambda^2 \left[ 1 + C_3^2 (\lambda^{-1} + 2)^2 \Delta_k^2 + (2 + \lambda)^2 + C_3^2 (1 + 2 \lambda)^2 \Delta_k^2\right] \\
        & \leq 2 s \big[ \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 \big].
    \end{aligned}
\]


   
       
       
       
       
       
   
Note that for standard Gaussian distribution $Z$, we have 
\[
    \mathbb{P} (Z \geq x) \leq \exp \{ - {x^2} / {2}\}, \qquad x > 0,
\]
then
\[
    \begin{aligned}
        \mathbb{P} (G_{1, s}^c \mid \mathcal{H}_{1, s}) & = 2 \mathbb{P} \left(\frac{f_1 + f_2 - f_3 - {\mathbb{E}} (f_1 + f_2 - f_3)}{\operatorname{var} (f_1 + f_2 - f_3)} > - \frac{{\mathbb{E}} (f_1 + f_2 - f_3)}{\operatorname{var} (f_1 + f_2 - f_3)} \right) \\
        & \leq 2 \exp \left\{ - \frac{{\mathbb{E}}^2 (f_1 + f_2 - f_3)}{2 \operatorname{var}^2 (f_1 + f_2 - f_3)} \right\} \\
        & \leq 2 \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s^2 }{2 \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + \mathbb{G}_{3}(s, \lambda) + g_4(s, \lambda) \big]}\right\}.
    \end{aligned}
\]
Therefore, $a_{k, s, 2}$ will be bounded in 
\[
    \begin{aligned}
        a_{k, s, 2} & \leq {\mathbb{E}} \Big[ T  1(G_{1, s}^c)\Big] =  T {\mathbb{E}} \Big[ \mathbb{P}(G_{1, s}^c \mid \mathcal{H}_{1, s}) \Big] \\
        & \leq 2 T {\mathbb{E}} \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s^2 }{2 \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + \mathbb{G}_{3}(s, \lambda) + g_4(s, \lambda) \big]}\right\}
    \end{aligned}
\]
On the other hand, 
if we note that $\sqrt{s} (\overline{R}_{1, s} - \mu_1) \sim \operatorname{subG} (\sigma_1^2)$, then $\sqrt{s} (2\lambda + 1) (\overline{R}_{1, s} - \mu_1) \sim \operatorname{subG} \big((2\lambda + 1)^2\sigma_1^2 \big)$, and
hence
\[
    \begin{aligned}
        & \Big[ \sqrt{s} (2 \lambda + 1) (\overline{R}_{1, s} - \mu_1) + \lambda \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big] \Big]^2 \\
        & \quad - \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 - (2 \lambda + 1 )^2 \operatorname{var} (R_{1,s}) \,  \sim \, \operatorname{subE} \big(8\sqrt{2}(2\lambda + 1)^2\sigma_1^2, 8(2\lambda + 1)^2\sigma_1^2 \big),
    \end{aligned}
\]
with the upper bound in \eqref{G_function_bound}, one can obtain that
\begin{equation}\label{a_2_bound}
    \begin{aligned}
        a_{k, s, 2} & \leq 2 T  {\mathbb{E}} \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s^2 }{2 \big[ \mathbb{G}_{1}(s, \lambda) + \mathbb{G}_{2}(s, \lambda) + \mathbb{G}_{3}(s, \lambda) + g_4(s, \lambda) \big]}\right\} \\
        & \leq 2 T \times \\
        & {\mathbb{E}} \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s}{4 \big[\operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2) + 2\lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + (2 \lambda + 1 )^2 \operatorname{var} (R_{1,s})\big] + 2 s^{-1}g_5(s, \lambda)}\right\} \\
        & \leq 2 T \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s}{8 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + 4 (2 \lambda + 1 )^2 \operatorname{var} (R_{1,s}) + 2 s^{-1} g_5(s, \lambda)}\right\} \\
        & \qquad \times \Big[ 1 + C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s (h_1 + h_2) \Big]\\
        & \leq 2 T^2 \Big[ 1 + C_3^2 (1 + 2\lambda)^4 \Delta_k^2 (h_1 + h_2) \Big] \\
        & \exp \left\{ - \frac{C_3^2 (1 + 2\lambda)^4 \Delta_k^2 s}{8 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + 4 (2 \lambda + 1 )^2 \sigma_1^2 + 4 \big[ \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 \big]}\right\}.
    \end{aligned}
\end{equation}
where $h_1$ and $h_2$ is some finite integral free of $s$ by the following two lemmas by the fact that for $s$ big enough one must have $b \leq \sqrt{2 c a \cdot 8 \sigma_1^2}$ with $a := 4$, $b := 8 \lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + 2 (2 \lambda + 1 )^2 \operatorname{var} (R_{1,s})\big] + 4 s^{-1}g_5(s, \lambda)$ and $c := C_3^2 (1 + 2 \lambda)^4 \Delta_k s$.

\begin{lemma}
    Suppose $X$ be any zero-mean random variable, and $f$ be a differentiable function, then
    \[
        \begin{aligned}
            {\mathbb{E}} f(X) & = f(0) \big[ \mathbb{P} (X  > 0) - \mathbb{P} (X < 0) \big] + \int_0^{\infty} f'(u) \mathbb{P} (X > u) \, du - \int_{-\infty}^0 f'(u) \mathbb{P} (X < u) \, du \\
            & =  f(0) \big[ \mathbb{P} (X  > 0) - \mathbb{P} (X < 0) \big] + \int_0^{\infty} \big[ f'(u) - f'(-u) \big] \mathbb{P} (|X| > u) \, du.
        \end{aligned}
    \]
    Furthermore, if $f(0) \geq 0$ and $X$ is positive, then
    \[
        {\mathbb{E}} f(X) \leq f(0) + \int_0^{\infty} f'(u) \mathbb{P} (X > u) \, du.
    \]
\end{lemma}
\iffalse
\begin{lemma}[Need to be corrected!]\label{lem_subE_ineq}
    Suppose zero-mean random variable $X \sim \operatorname{subE}(\lambda, \alpha)$, and positive constant $a, b, c$ satisfying $a X + b > 0$,
    then
    \[
        {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\}  \leq e^{- c / b} \big[1 + c h_1(a, b, \alpha) + c h_2(a, b, \lambda) \big]
    \]
    where $h_1$ and $h_2$ are finite positive functions.
\end{lemma}

\begin{proof}
    Note that differentiable $f$ is positive, we have $0 \leq {\mathbb{E}} f(X) \leq f(0) + \int_0^{\infty} \big[ f'(u) - f'(-u) \big]\mathbb{P} (|X| > u) \, du$, then 
    \[ 
        \begin{aligned}
            & {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\} \\
            \leq & e^{-c / b} + \int_0^{\infty} \bigg[ \exp \left\{ - \frac{c}{a u + b}\right\}  \frac{ca }{ (a u + b)^2} - \exp \left\{ - \frac{c}{ - a u + b}\right\}  \frac{ca }{ (- a u + b)^2} \bigg] \, \mathbb{P} (|X| > u) \, du \\
            \leq & \int_0^{\infty} \bigg[ \exp \left\{ - \frac{c}{a u + b}\right\}  \frac{ca }{ (a u + b)^2} - \exp \left\{\frac{c}{ a u - b}\right\}  \frac{ca }{ (a u - b)^2} \bigg] \, \exp \left\{ - \frac{1}{2} \left( \frac{u^2}{\lambda^2} \wedge\frac{u}{\alpha} \right) \right\} \, du \\
            = & \int_0^{\infty} \exp \left\{ - \frac{c}{a u + b}\right\} \bigg[   \frac{ca }{ (a u + b)^2} - \exp \left\{\frac{b + au}{ b - au}\right\}  \frac{ca }{ (a u - b)^2} \bigg] \, \exp \left\{ - \frac{1}{2} \left( \frac{u^2}{\lambda^2} \wedge\frac{u}{\alpha} \right) \right\} \, du \\
            \leq & \exp (-c / b) \int_0^{\infty} \Bigg[ \frac{ca}{(a u + b)^2} \exp \left\{ - \frac{u}{2 \alpha}\right\} + \frac{ca}{(a u - b)^2} \exp \left\{ - \frac{1}{2}\frac{u^2}{ \lambda^2}  \wedge \frac{u}{ \alpha} + \frac{b + au}{b - au}\right\} \Bigg] \, du \\
            =: &  \exp (-c / b) \big[ I_1 + I_2 \big].
        \end{aligned}
    \]
    Denote $\Gamma(\cdot, \cdot)$ as incomplete gamma function, then one can calculate that
    \[
        I_1 = \frac{c}{b} - \frac{c e^{\frac{b}{2 a \alpha}}\Gamma(0, \frac{b}{2 a \alpha})}{2 a \alpha} =: c h_1 (a, b, \alpha),
    \]
    which is finite as long as $a, \alpha$ is positive.
    For calculating $I_2$, we write $I_2$ as
    \[
        \begin{aligned}
            I_2 = & c a \int_{0}^{\lambda^2 / \alpha} \frac{1}{(a u - b)^2} \exp \left\{ - \frac{u^2}{2 \lambda^2}  + \frac{b + a u}{b - au}\right\} \, du  + \int_{\lambda^2 / \alpha}^{\infty}  \frac{1}{(a u - b)^2} \exp \left\{ - \frac{u}{2 \alpha} + \frac{b + a u}{b - au}\right\} \, du \\
            =: & ca ( I_{21} + I_{22}).
        \end{aligned}
    \]
    As $I_{21}$ and $I_{22}$ are both free of $c$, we only need to care its irregular point $\frac{b}{a}$ and $\infty$. We separate it as the following two cases:
    \begin{itemize}
        \item[(i)] If $\frac{b}{a} \leq \frac{\lambda^2}{\alpha}$, then 
        \[
            \lim_{u \rightarrow \frac{b}{a}} (a u - b)^{1 / 2}\frac{1}{(a u - b)^2} \exp \left\{ - \frac{u^2}{2 \lambda^2} + \frac{b + au}{b - au}\right\} = 0,
        \]
        and 
        \[
            \lim_{u \rightarrow \infty} u^{3 / 2}\exp \left\{ - \frac{u}{2 \alpha} + \frac{b + au}{b - au}\right\} = 0.
        \]
        \item[(ii)] If $\frac{b}{a} \geq \frac{\lambda^2}{\alpha}$, then $I_{21}$ will be strictly finite. And for $I_{22}$, it will be
        \[
            \lim_{u \rightarrow \frac{b}{a}} (a u - b)^{1 / 2}\frac{1}{(a u - b)^2} \exp \left\{ - \frac{u}{2 \alpha} + \frac{b + au}{b - au}\right\} = \lim_{u \rightarrow \infty},
        \]
        
    \end{itemize}
    Hence, $\exists \, I_2 \in \mathbb{R}$, and then
    \[
        \exists \, I_2 = c \int_0^{\infty} \frac{a}{(a u - b)^2} \exp \left\{ - \frac{u}{2 \alpha} + \frac{b + au}{b - au}\right\} =: c h_2(a, b, \lambda) \in c \times (0, \infty)
    \]
    with $h_1$ and $h_2$ are finite positive functions.
\end{proof}
\fi


\begin{lemma}[sub-Exponential concentration]\label{lem_subE_ineq}
    Suppose zero-mean random variable $X \sim \operatorname{subE}(\lambda, \alpha)$, and positive constant $a, b, c$ satisfying $a X + b > 0$. If $b \leq 2 c a\lambda^2 \alpha^2 \wedge \sqrt{2 c a\alpha}$, then
    \[
        {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\}  \leq e^{- c / b} \big[1 + c h_1(a, b, \alpha, \lambda) + c h_2(a, b, \alpha) \big],
    \]
    where $h_1$ and $h_2$ are finite positive functions free of $c$.
\end{lemma}

\begin{proof}
    Denote $Y := a X + b$ is a strictly positive random variable, then by the above lemma,
    \[
        \begin{aligned}
            {\mathbb{E}} \exp \left\{ - \frac{c}{a X + b}\right\} & = {\mathbb{E}} \exp \left\{ - \frac{c}{Y}\right\} \\
            & \leq e^{- c / b} + c \int_0^{\infty} \frac{1}{u^2}\exp \left\{ - \frac{c}{u} \right\} \mathbb{P} (a X + b > u) \, du \\
            & \leq e^{- c / b} + c \left[ \int_0^{b} + \int_b^{\infty} \right] \frac{1}{u^2}\exp \left\{ - \frac{c}{u} \right\} \mathbb{P} (a X + b > u) \, du \\
            & := e^{- c / b} + c (I_{1} + I_2 ).
        \end{aligned}
    \]
    For $I_1$, we have
    \[
        \begin{aligned}
            I_1 \leq \int_0^b \frac{1}{u^2} \exp \left\{ - \frac{c}{u}\right\} \, du = \frac{1}{c} \exp \left\{ - \frac{c}{b}\right\},
        \end{aligned}
    \]
    while for $I_2$, WLOG, we assume $\lambda^2 / \alpha \geq b$, we have
    \[
        \begin{aligned}
            I_2 & = \int_b^{\infty} \frac{1}{u^2} \exp \left\{ - \frac{c}{u}\right\} \mathbb{P} \left(X > \frac{u - b}{a}\right) \, du \\
            & \leq \int_b^{\infty} \frac{1}{u^2} \exp \left\{ - \frac{c}{u}\right\} \exp \left\{ - \frac{1}{2} \frac{(u - b)^2}{\lambda^2 a^2} \wedge \frac{(u - b)}{\alpha a}  \right\} \, du \\
            & \leq \left[ \int_b^{\lambda^2 / \alpha} + \int_{\lambda^2 / \alpha}^{\infty} \right]  \frac{1}{u^2} \exp \left\{ - \frac{c}{u}\right\} \exp \left\{ - \frac{1}{2} \frac{(u - b)^2}{\lambda^2 a^2} \wedge \frac{(u - b)}{\alpha a}  \right\} \, du =: I_{21} + I_{22},
        \end{aligned}
    \]
    where 
    \[
        \begin{aligned}
            I_{21} & := \int_b^{\lambda^2 / \alpha} \frac{1}{u^2} \exp \left\{ - \frac{c}{u}\right\} \exp \left\{ - \frac{(u - b)^2}{2 \lambda^2 a^2} \right\} \, du \\
            & \leq \exp \left\{ - \frac{c \alpha}{\lambda^2}\right\} \int_b^{\lambda^2 / \alpha} \frac{1}{u^2} \exp \left\{ - \frac{(u - b)^2}{2 \lambda^2 a^2} \right\} \, du =: \exp \left\{ - \frac{c \alpha}{\lambda^2}\right\} h_1(a, b, \lambda, \alpha)
        \end{aligned}
    \]
    and 
    \[
        \begin{aligned}
            I_{22} & := \int_{\lambda^2 / \alpha}^{\infty} \frac{1}{u^2} \exp \left\{ - \frac{c}{u}\right\} \exp \left\{ - \frac{(u - b)}{2 \alpha a}  \right\} \, du \\
            (\text{by } b \leq 2 c a\lambda^2 \alpha^2 )& \leq \exp \left\{ - \frac{c}{b}\right\} \int_{\frac{2 a c \alpha}{b}}^{\infty} \frac{1}{u^2} \exp \left\{ - \frac{c(b - u)}{bu} - \frac{(u - b)}{2 \alpha a}\right\} \, du \\
            & = \exp \left\{ - \frac{c}{b}\right\} \int_{\frac{2 a c \alpha}{b}}^{\infty} \frac{1}{u^2} \exp \left\{ - \frac{(u - b)(u - \frac{2 a c \alpha}{b})}{2 \alpha a u}\right\} \, du \\
            (\text{by } b \leq \sqrt{2 c a\alpha} )& \leq \exp \left\{ - \frac{c}{b}\right\} \int_{\frac{2 a c \alpha}{b}}^{\infty} \frac{1}{u^2} \exp \left\{ - \frac{(u - \frac{2 a c \alpha}{b})^2}{2 \alpha a u}\right\} \, du \\
            & \leq \sqrt{\frac{b}{2 ca \alpha}} \exp \left\{ - \frac{c}{b}\right\}  \int_{\frac{2 a c \alpha}{b}}^{\infty}   \frac{1}{u^{3/2}} \exp \left\{ - \frac{(u - \frac{2 a c \alpha}{b})^2}{2 \alpha a u}\right\} \, du \\
            & = \frac{b}{4 c a \alpha} \sqrt{\frac{\pi b}{c}} \exp \left\{ - \frac{c}{b}\right\} \left[1 - e^{\frac{4 c}{b}} \text{erfc}\left(\frac{2 c}{\sqrt{b c}}\right)\right] \\
            & \leq \frac{b}{4 c a \alpha} \sqrt{\frac{\pi b}{c}} \exp \left\{ - \frac{c}{b}\right\} := \exp \left\{ - \frac{c}{b}\right\} h_2(a, b, \alpha)
        \end{aligned}
    \]
    where $\operatorname{erfc}(x) := \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} \, dt$. Therefore, we get the result in the lemma.
\end{proof}


Finally, let us take
\[
    \begin{aligned}
        s \geq s_{a, 2} := & \frac{12 \big(2\lambda^2 \big[ (\lambda + \mu_1) + C_3 (1 + 2 \lambda) \Delta_k \big]^2 + 6 (2 \lambda + 1 )^2 \sigma_1^2 + \big[ \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 \big] \big)}{C_3^2 (1 + 2\lambda)^4 \Delta_k^2}\\
        & \qquad \times \log \Big[  2 C_3^2 \lambda^2 (1 + 2\lambda)^4 \Delta_k^2 \big[ 1 + C_3^2 (1 + 2\lambda)^4 \Delta_k^2 (h_1 + h_2) \big] \Big] \log T
    \end{aligned}
\]
we will have $a_{k, s, 2} \leq T^{-1}$.

\begin{lemma}
    Suppose $\xi \sim \operatorname{subG}(\sigma^2/n)$, then 
    \[
        {\mathbb{E}} \exp \left\{ - \frac{a}{(\xi - b)^4} \right\} \lesssim \exp \left\{ - c n \right\}
    \]
    with some positive constant $c$ for any $a, b > 0$.
\end{lemma}

\begin{proof}
    Similar to Lemma \ref{lem_subE_ineq}, use Lemma 4, we can write the expectation as
    \[
        \begin{aligned}
            {\mathbb{E}} \exp \left\{ - \frac{a}{(\xi - b)^4} \right\} = \int
        \end{aligned}
    \]
\end{proof}

\subsection{Tackling $a_{k, s, 3}$ with $s_{a, 3}$}

As state previously, $a_{k, s, 3}$ defined on good events, we need special technique to deal with. Write 
\[
    \begin{aligned}
        \{N_{1, s}(\tau_{k})<T^{-1}\} & \supseteq \{N_{1, s}(\tau_{k}) < (T^{2}-1)^{-1}\} = \big\{Q_{1, s}\left(\tau_{k}\right)^{-1}<1+\left(T^{2}-1\right)^{-1}\big\} \\
        &=\big\{Q_{1, s}(\tau_{k})^{-1} < (1-T^{-2})^{-1}\big\} = \big\{\left[1-Q_{1, s}(\tau_{k})\right]<T^{-2}\big\},
    \end{aligned}
\]
and $a_{k, s, 3} \leq {\mathbb{E}} \big[ N_{1, s} (\tau_k) 1(A_{1, s} \cap G_{1, s}) \big]$. Thus, it is sufficient to  find $s \geq s_{a,3}$ such that $\big\{[1-Q_{1,s}(\tau_{k})]<T^{-2}\big\}$ on the event $\{A_{1, s} \cap G_{1, s}\}$. 
Note that
\[
    \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}) = \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Gamma_1 \, | \, \mathcal{H}_{1, s} \big)  1(A_{1, s}) 1(G_{1, s})
\]
with $\Gamma_1 := \Gamma_1(\tau_1) = \tau_1 - \overline{R}_{1, s}^*$ serving as a measurement to balance the difference between $\overline{Y}_{1, s}$ and $\overline{R}_{1,s}^*$. It is not hard to see $\Gamma_1$ should be negative with a moderate value. Fortunately, on event $A_{1, s} \, \cap \, G_{1, s}$, this can be true. To see this, we write 
\[
    A_{1, s} = \bigg\{ \overline{R}_{1, s}^* - \frac{\mu_1 + \lambda}{1 + 2\lambda} > - C_1 \Delta_k \bigg\} = \bigg\{ \Gamma_1 < C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} + \tau_1 \bigg\} 
\]
and
\[
    \begin{aligned}
        G_{1, s} = \big\{ -C_3 \Delta_k \leq \overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq C_3 \Delta_k \big\} & = \big\{ \tau_1 -C_3 \Delta_k \leq \overline{Y}_{1, s} + \Gamma_1 \leq \tau_1 + C_3 \Delta_k \big\} \\
        & = \big\{ \tau_1 - C_3 \Delta_k - \overline{Y}_{1, s} \leq \Gamma_1 \leq \tau_1 + C_3 \Delta_k - \overline{Y}_{1, s}\big\} \\
        & \subseteq \big\{ \tau_1 - 2 C_3 \Delta_k - \overline{R}_{1, s}^* \leq \Gamma_1 \leq \tau_1 + 2 C_3 \Delta_k - \overline{R}_{1, s}^* \big\} \\
        & \subseteq \bigg\{ \tau_1 -  2 C_3 \Delta_k - \frac{1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 \leq \tau_1 + 2 C_3 \Delta_k - \frac{\lambda}{1 + 2 \lambda}\bigg\},
    \end{aligned}
\]
then
\[
    \begin{aligned}
        A_{1, s} \, \cap \, G_{1, s} & = \bigg\{ \tau_1 -  2 C_3 \Delta_k - \frac{1 + \lambda}{1 + 2 \lambda} \leq \Gamma_1 < \tau_1 + C_1 \Delta_k - \frac{\mu_1 + \lambda}{1 + 2\lambda} \bigg\} \\
        & \subseteq \bigg\{ \tau_1 -  2 C_3 \Delta_k - 1 \leq \Gamma_1 < C_1 \Delta_k + \tau_1 - \frac{\mu_1}{2}\bigg\} \\
        (\text{if } C_1 \geq 2 C_3 )& \subseteq \bigg\{ \tau_1 -  C_1 \Delta_k - 1 \leq \Gamma_1 < C_1 \Delta_k + \tau_1 - \frac{\mu_1}{2}\bigg\}
    \end{aligned}
\]
and hence we derive the requirement that
\begin{equation}\label{requirement_1}
     \Delta_k + \tau_1 - \frac{\mu_1}{2} = 0 \leq 0 \qquad \text{ and }  \qquad C_1 \geq 2C_3.
\end{equation}
Therefore, we obtain the range of the positive quantity $-\Gamma_1$ will as follows
\[
    C_1 \Delta_k = \frac{\mu_1 }{2} - \tau_1 - C_1 \Delta_k <  (- \Gamma_1) \leq 1 + C_1 \Delta_k  - \tau_k = 1 - \frac{\mu_1}{2} + 3 C_1 \Delta_k < 4
\]
i.e. $ (- \Gamma_1) \in (C_1 \Delta_k, 4)$. If we take
\begin{equation}\label{requirement_2}
    \tau_1 = \frac{\mu_1}{2} - \Delta_k.
\end{equation}
Besides, as the tail probability we care is 
\[
    \begin{aligned}
        & \big[ 1 - Q_{1,s} (\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \\
        = & \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) \omega_i + \lambda \sum_{i = 1}^s (2 R_{1, i} \omega_i - \overline{R}_{1, s} \omega_i' - \overline{R}_{1, s} \omega_i'') 
        \\
        & \qquad \qquad + \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s (\omega_i' - \omega''_i) - \Gamma_1 (1 + 2\lambda) \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] < 0\Bigg\},
    \end{aligned}
\]
what we need to deal with is by replacing  $C_3 \Delta_k$ by $(-\Gamma_1)$ in the above section. By using the same technique one can get that
\begin{footnotesize}
\[
    \begin{aligned}
        & \big[ 1 - Q_{1,s} (\tau_k)\big] 1(A_{1, s}) 1(G_{1, s}) \\
        \leq & {\mathbb{E}} \exp \left\{ - \frac{\Gamma_1^2 (1 + 2\lambda)^4 s}{4 (\operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2) + 2\lambda^2 \big[ (\lambda + \mu_1) + (1 + 2 \lambda) \Gamma_1 \big]^2 + (2 \lambda + 1 )^2 \sigma_1^2 + 2 \lambda^2(\sqrt{5} + \lambda)^2 + 2 (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 ) }\right\} \\
        \leq & {\mathbb{E}} \exp \left\{ - \frac{C_1^2 \Delta_k^2 (1 + 2\lambda)^4 s}{4 (\operatorname{subE} (8\sqrt{2}\sigma_1^2, 8\sigma_1^2) + 2\lambda^2 \big[ (\lambda + \mu_1) -  (1 + 2 \lambda) 4 \big]^2 + (2 \lambda + 1 )^2 \sigma_1^2 + \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2) }\right\} \\
        \leq &  T  \exp \left\{ - \frac{C_1^2 (1 + 2\lambda)^4 \Delta_k^2 s}{8 \lambda^2 (7 \lambda + 4 - \mu_1)^2 + 4 (2 \lambda + 1 )^2 \sigma_1^2 + 4 [ \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2]}\right\} \\
        & \times \big[1 + C_1^2 \Delta_k^2 (1 + 2\lambda)^4  (h_1^* + h_2^*)\big] \leq T^{-2}
    \end{aligned}
\]
\end{footnotesize}
whenever
\[
   \begin{aligned}
       s \geq s_{a, 3} := & \frac{6 \big[ 4\lambda^2 (7 \lambda + 4 - \mu_1)^2 + 2 (2 \lambda + 1 )^2 \sigma_1^2 + 2 ( \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2) \big] }{C_1^2 (1 + 2\lambda)^4 \Delta_k^2 } \\
       & \qquad \qquad \times \log \Big[ 1 + C_1^2 \Delta_k^2 (1 + 2\lambda)^4  (h_1^* + h_2^*)\Big] \log T,
   \end{aligned}
\]
where $h_1^*$ and $h_2^*$ free of $s$ are the $h$ functions in Lemma \ref{lem_subE_ineq} by replacing the corresponding parameter.

Now Combine all results in Section 2, we get that 
\[
    a_{k} \leq \underbrace{\max_{s = 0, \ldots, T -  1} a_{k, s}}_{\text{bound by a constant in \eqref{a_k_constant}}} \cdot \overbrace{\max _{j = 1, 2, 3} s_{a, j}}^{\text{with order } \log T} + 3 T^{-1} \cdot\big(T - \underbrace{\max_{j = 1, 2, 3} s_{a, j}}_{\text{with order } \log T}\big) \lesssim \log T.
\]
\begin{remark}
   Here, we do not give the explicit expression of the bound as it involves several numeric integral terms, while its order is exactly $\log T$.
\end{remark}

\section{Bounding $b_k$}

For bounding $b_k = \sum_{s = 0}^{T - 1} b_{k, s}$, we will decompose $b_{k, s} = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1}\big) + 1$. Similarly to dealing with $a_k$, we decompose $b_{k, s}$ as $b_k = b_{k, s, 1} + b_{k, s, 2} + b_{k, s, 3} + 1$ with
\[
    b_{k, s, 1} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-1}) 1(A_{k, s}^{\prime c} )\big],
\]
\[
    b_{k, s, 2} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-1}) 1(A'_{k, s}) 1(G_{k, s}^c)\big],
\]
\[
    b_{k, s, 3} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-1}) 1(A'_{k, s}) 1(G_{k, s})\big].
\]
Since $b_{k, s}$ is naturally bounded $2$, it only remains to bound $s_{b, j}: = \max \{ s: b_{k, s, j} \geq T^{-1}\}$ up to $\log T$ for $j = 1, 2, 3$.

It is easy to see that $\mathbb{P} (A^{\prime c}_{k, s}) $ when $s \geq \frac{1}{2}s_{a, 2}'$. Here $s_{\cdot}'$ is the value of $s_{\cdot}$ but in it $C_1$ is replaced by $C_2$. Then we must have $b_{k,s, 1} \vee b_{k, s, 2} \leq T^{-2} < T^{-1}$ as long as $s \geq s'_{a, 1}  / 2 \vee s'_{a, 2}$. one only need to tackle $b_{k, s, 1}$. We will first bound $\Gamma_k = \overline{Y}_{k, s}  - \overline{R}_{k, s}^*$ on $A_{k, s}' \, \cap \, G_{k, s}$
\[
    A'_{k, s} = \bigg\{ \overline{R}_{k, s}^* - \frac{\mu_k + \lambda}{1 + 2\lambda} < C_2 \Delta_k \bigg\} = \bigg\{ \tau_k - \frac{\mu_k + \lambda}{1 + 2\lambda} - C_2 \Delta_k  < \Gamma_k \bigg\},
\]
and
\[
    \begin{aligned}
        G_{k, s} = \big\{ -C_3 \Delta_k \leq \overline{Y}_{k, s} - \overline{R}_{k, s}^* \leq C_3 \Delta_k \big\} & = \big\{ \tau_k -C_3 \Delta_k \leq \overline{Y}_{k, s} + \Gamma_k \leq \tau_k + C_3 \Delta_k \big\} \\
        & = \big\{ \tau_k - 2 C_3 \Delta_k - \overline{R}_{k, s}^* \leq \Gamma_k \leq \tau_k + 2 C_3 \Delta_k - \overline{R}_{k, s}^* \big\} \\
        & \subseteq \bigg\{ \tau_k - 2 C_3 \Delta_k - \frac{1 + \lambda}{1 + 2\lambda} \leq \Gamma_k \leq \tau_k + 2 C_3 \Delta_k - \frac{\lambda}{1 + 2\lambda}\bigg\},
    \end{aligned}
\]
hence, we can conclude that 
\[
    \begin{aligned}
        A'_{k, s} \, \cap \, G_{k, s} & \subseteq  \bigg\{ \tau_k - \frac{\mu_k + \lambda}{1 + 2\lambda} - C_2 \Delta_k  < \Gamma_k \leq\tau_k + C_2 \Delta_k - \frac{\lambda}{1 + 2\lambda} \bigg\} \\
        & \subseteq \bigg\{ \tau_k  - C_2 \Delta_k - \frac{\mu_k}{2}  < \Gamma_k < \tau_k + C_2 \Delta_k \bigg\}
    \end{aligned}
\]
if
\begin{equation}\label{requirement_3}
    2 C_3 \geq C_2.
\end{equation}
Under this establishment, we need $\tau_k - C_2 \Delta_k - \frac{\mu_k }{2} \leq 0$, then we can take
\begin{equation}\label{requirement_4}
    \tau_k = 2 C_2 \Delta_k + \mu_k / 2
\end{equation}
and we will obtain that the bound for $\Gamma_k$ is
\[
    C_2 \Delta_k = \tau_k  - C_2 \Delta_k - \frac{\mu_k}{2} < \Gamma_k < \tau_k + C_2 \Delta_k = 3 C_2 \Delta_k + \frac{\mu_k}{2} \leq \frac{7}{2} < 4,
\]
i.e. $\mathbb{R}_+ \ni \Gamma_k \in (C_2 \Delta_k, 4)$, then we will have
\begin{footnotesize}
\[
    \begin{aligned}
        & Q_{k, s}\left(\tau_{k}\right) 1\left(A_{k, s}^{\prime}\right) 1\left(G_{k, s}\right) = {\mathbb{E}} \mathbb{P} \left(\overline{Y}_{k, s}-\overline{R}_{k, s}^{*}>\Gamma_{k} \mid \mathcal{H}_{k, T}\right) 1\left(A_{k, s}^{\prime}\right) 1\left(G_{k, s}\right) \\
        = & {\mathbb{E}} \mathbb{P}_{\omega, \omega', \omega''} \Bigg\{ \sum_{i = 1}^s (R_{1, i} - \overline{R}_{1, s}) \omega_i + \lambda \sum_{i = 1}^s (2 R_{1, i} \omega_i - \overline{R}_{1, s} \omega_i' - \overline{R}_{1, s} \omega_i'')
        \\
        & \qquad + \lambda \sum_{i = 1}^s (\omega_i' - \omega_i) + \lambda^2 \sum_{i = 1}^s (\omega_i' - \omega''_i) - \Gamma_k (1 + 2\lambda) \bigg[ \sum_{i = 1}^s \omega_i + \lambda \sum_{i = 1}^s \omega_i' + \lambda \sum_{i = 1}^s \omega_i''\bigg] < 0\Bigg\}\\
        \leq & {\mathbb{E}} \exp \left\{ - \frac{\Gamma_k^2 (1 + 2\lambda)^4 s}{4 (\operatorname{subE} (8\sqrt{2}\sigma_k^2, 8\sigma_k^2) + 2\lambda^2 \big[ (\lambda + \mu_k) +  (1 + 2 \lambda) \Gamma_k \big]^2 + (2 \lambda + 1 )^2 \sigma_k^2 + \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2) }\right\} \\
        \leq & {\mathbb{E}} \exp \left\{ - \frac{C_2^2 \Delta_k^2 (1 + 2\lambda)^4 s}{4 (\operatorname{subE} (8\sqrt{2}\sigma_k^2, 8\sigma_k^2) + 2\lambda^2 \big[ (\lambda + \mu_k) + (1 + 2 \lambda) 4 \big]^2 + (2 \lambda + 1 )^2 \sigma_k^2 + \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 ) }\right\} \\
        \leq &  T \exp \left\{ - \frac{C_2^2 (1 + 2\lambda)^4 \Delta_k^2 s}{8 \lambda^2 (9 \lambda + 4 + \mu_k)^2 + 4 (2 \lambda + 1 )^2 \sigma_k^2 + 4 \lambda^2(\sqrt{5} + \lambda)^2 + 4 (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 }\right\} \\
        & \qquad \qquad \times \Big[ 1 + C_2^2 (1 + 2\lambda)^4 \Delta_k^2 (h_1^{**} + h_2^{**} )\Big].
    \end{aligned}
\]
\end{footnotesize}
where $h_1^{**}$ and $h_2^{**}$ are similarly defined as previous. Therefore, one can just take 
\[
    \begin{aligned}
        s \geq s_{b, 3} & :=  \frac{8 \big[ 2 \lambda^2 (9 \lambda + 4 + \mu_k)^2 +  (2 \lambda + 1 )^2 \sigma_k^2 + \lambda^2(\sqrt{5} + \lambda)^2 + (1 + \lambda^2)(1 + 2 \lambda)^2 C_3^2 \Delta_k^2 \big]}{C_2^2 (1 + 2\lambda)^4 \Delta_k^2 s} \\
        & \qquad \qquad \times \log \Big[ 1 + C_2^2 \Delta_k^2 (1 + 2\lambda)^4  (h_1^{**} + h_2^{**})\Big] \log T
    \end{aligned}
\]
and then by using the similar argument, we have 
\[
    b_{k} \leq 2 \cdot \overbrace{\max _{j = 1, 2, 3} s_{b, j}}^{\text{with order } \log T} + 3 T^{-1} \cdot\big(T - \underbrace{\max_{j = 1, 2, 3} s_{b, j}}_{\text{with order } \log T}\big) \lesssim \log T.
\]
Finally, what remains is to show the restrictions (\ref{requirement_0}), (\ref{requirement_1}), (\ref{requirement_2}), (\ref{requirement_3}), and (\ref{requirement_4}) are compatible. Indeed, they will build the following system of equations
\[
    \left\{ \begin{array}{l}
         \frac{\mu_1}{2} - 2 C_1 \Delta_k \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \\
         2 C_2 \Delta_k + \frac{\mu_k}{2} \leq \frac{\mu_1 + \lambda}{1 + 2 \lambda} \\
        \tau_k - \frac{\mu_1}{2} - C_1 \Delta_k  \leq \tau_k + C_2\Delta_k - \frac{\mu_k }{2} \\
        C_1 \geq 2C_3 \\
        2 C_3 \geq C_2 \\
        \frac{\mu_1}{2} - 2 C_1 \Delta_k = 2 C_2 \Delta_k + \frac{\mu_k}{2}
    \end{array}\right..
\]
We can see that 
\[
    C_1 = C_2 = \frac{1}{8}, \qquad C_3 = \frac{1}{16},
\]
is the unique solution for the above system. Up to now, we have finished our proof.






\end{document}
\section{First Step}

The first step is to conclude (or state) the result in \cite{kveton2019garbage}, that is as long as we use
\begin{equation*}
    I_t = \arg \max_{k \in [K]} \overline{Y}_{k, s},
\end{equation*}
then we must have the bound
\begin{equation*}
    \operatorname{Reg}_T \leq \sum_{k = 2}^K \Delta_k \big( a_k + b_k \big)
\end{equation*}
with
\begin{equation*}
    a_k = \sum_{s = 0}^{T - 1} {\mathbb{E}} \big[N_{1, s}(\tau_k) \wedge T\big] =: \sum_{s = 0}^{T - 1} a_{k, s}, \qquad b_k = \sum_{s = 0}^{T - 1} \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-1} \big) + 1 =: \sum_{s = 0}^{T - 1} b_{k, s} + 1
\end{equation*} 
with any $\{ \tau_k \}_{k = 2}^K \subseteq \mathbb{R}^{K - 1}$ where $Q_{k, s} (\tau) = \mathbb{P} \big( \overline{Y}_{k, s} > \tau \, | \mathcal{H}_{k, T} \big)$ and $N_{k, s} (\tau) = 1 / Q_{k, s} (\tau) - 1$ for some $\tau > 0$. So our target switch from $\operatorname{Reg}_T$ to $a_k$ and $b_k$. And the next two main steps will be bounding $a_k$ and $b_k$ separately. 

\section{Second Step}

The second step is bounding $a_k = \sum_{s = 0}^{T - 1} a_{k, s}$. Note that if $\tau_k \leq \mu_1$,
\begin{equation*}
    a_{k, s} \leq {\mathbb{E}} Q_{1, s}(\tau_k)^{-1} \leq  {\mathbb{E}} \big[ \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, T})^{-1} ) \big],
\end{equation*}
we need to first find the lower bound of the tail probability $\mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, T})$ and then find the upper bound of the expectation.

\subsection{The lower bound of tail probability $\mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, T})$.}

{\color{red}{We assume that $\omega_{\ell} \sim \operatorname{Exp}(1)$, $R_{1, \ell} \sim \operatorname{ber} (\mu_1, 1)$ with defining $R_{1, 0} := \lambda_1 + \mu_1$}}.

\subsubsection{A possible construction of $\overline{Y}_{1, s}$.}

If we assume
\begin{equation*}
    \overline{Y}_{1, s} = \left(0 + {\color{red}{\omega_0 \cdot \lambda_1}} + \sum_{\ell = 1}^{s} \omega_{\ell} R_{1, \ell} \right) / \left( {\color{red}{\lambda_2}} + \sum_{\ell = 1}^{s} \omega_{\ell} \right) = \left(\omega _0\lambda s + \sum_{\ell = 1}^{s} \omega_{\ell} R_{1, \ell} \right) / \left( \sum_{\ell = 1}^{s} \omega_{\ell} \right)
\end{equation*}
will be only random on the bootstrap weights $\omega_{\ell}$ when conditions on $\mathcal{H}_{1, s}$ and the lower bound of the tail probability must require the exact distribution of $\omega_{\ell}$ or the harsh restriction for $\omega_{\ell}$ will be involved (see the discussion of \cite{zhang2020non} for details), where $\lambda_1$ and $\lambda_2$ is the tuning parameter.

\begin{equation*}
    \begin{aligned}
        \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, T}) & = \mathbb{P}_{\omega} \left(  \sum_{\ell = 0}^{s} (R_{1, \ell} - \mu_1) \cdot \omega_{\ell} \geq \lambda_2 \cdot \mu_1 \right) \\
        & \geq \mathbb{P}_{\omega} \left( \bigcap_{\ell = 0}^{s} \big\{ (R_{1, \ell} - \mu_1) \omega_{\ell} \geq \big(s + 1 \big)^{-1} \lambda_2 \mu_1 \big\} \right) \\
        & = \exp \left\{ -\frac{\lambda_2 \mu_1}{(s + 1) \lambda_1}\right\} \prod_{\ell = 1}^{s} \mathbb{P}_{\omega} \left( (R_{1, \ell} - \mu_1) \omega_{\ell} \geq \big(s + 1 \big)^{-1} \lambda_2 \mu_1 \right),
    \end{aligned}
\end{equation*}
Since $R_{1, \ell}$ is iid, this bound is actually loose. And then
\begin{equation*}
    \begin{aligned}
        a_{k, s} & \leq  {\mathbb{E}} \big[ \mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, T})^{-1} \big] \\
        & = \exp \left\{ \frac{\lambda_2 \mu_1}{(s + 1) \lambda_1}\right\} \prod_{\ell = 1}^{s} \bigg[\exp \left\{ \frac{\lambda_2 \mu_1}{(1 - \mu_1)(s + 1)}  \right\} \mu_1 \bigg] \\
        & = \mu_1^{s} \exp \left\{ \frac{\lambda_2 \mu_1 }{(s + 1) \lambda_1} + \frac{\lambda_2 \mu_1 s}{(1 - \mu_1)(s + 1)}  \right\}.
    \end{aligned}
\end{equation*}
by the following Lemma.
\begin{lemma}\label{lem1}
    We have the explicit expression for ${\mathbb{E}} \big[ \mathbb{P} (\omega \xi \geq \lambda \, | \, \xi )^{-1}\big]$ with $\xi \sim x_1 * p_1 + x_2 * p_2$ with $x_1 > 0, x_2 \leq 0$ and $\omega \sim \operatorname{Exp}(1)$ as $p_1 \exp \{x_1^{-1} \lambda\}  + p_2 \big[ 1 -  \exp \{ - x_2^{-1} \lambda \} \big]^{-1}$.
\end{lemma}
\begin{proof}
    We have:
    \begin{equation*}
        \begin{aligned}
            \mathbb{P}_{\omega} (\omega \xi \geq \lambda \, | \, \xi ) & = \mathbb{P}_{\omega} (\omega \geq \xi^{-1} \lambda \, | \, \xi > 0) \mathbb{P} (\xi > 0 \, | \, \xi) + \mathbb{P}_{\omega} (\omega \leq \xi^{-1} \lambda \, | \, \xi \leq 0) \mathbb{P} (\xi \leq 0 \, | \, \xi) \\
            & = \exp \{ - \xi^{-1} \lambda \} 1(\xi > 0)
        \end{aligned}
    \end{equation*}
    and then
    \begin{equation*}
        \begin{aligned}
            {\mathbb{E}} \big[ \mathbb{P}_{\omega} (\omega \xi \geq \lambda \, | \, \xi )^{-1}\big] & = {\mathbb{E}} \exp \left\{ \xi^{-1} \lambda \right\} 1(\xi > 0)\\
            & = p_1 \exp \{x_1^{-1} \lambda\}.
        \end{aligned}
    \end{equation*}
\end{proof}
The upper bound of $a_{k, s}$ will immediately give $\lambda_2 \in \mathbb{R}$ should be fixed, and then
\begin{equation*}
    a_{k, s} \leq \mu_1 \exp \left\{ \frac{\lambda_2 \mu_1}{2 \lambda_1} + \frac{\lambda_2 \mu_1}{(1 - \mu_1)}\right\} =: M(\mu_1, \lambda_1, \lambda_2)
\end{equation*}

\begin{remark}
    If the first tuning parameter $\lambda_1$ is associated to a nonrandom weight (i.e. taking $\omega_0 \equiv 1$ for example), the lower bound of $\mathbb{P} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, T})$ will quite different, and need to specify $\lambda_1$ first (since we need to determine whether $\lambda_2 \mu_1 - \lambda_1$ larger than 0).
\end{remark}

\begin{remark}
    From this step, we also know that $\lambda_1$ must be positive and cannot be zero. 
\end{remark}

\iffalse
\begin{lemma}
    For any $\mu \in (0, 1)$, if $\lambda > 0$ is fixed, we have the following inequality
    \begin{equation*}
        \exp \left\{ \frac{\lambda \mu}{n (1 - \mu)}\right\} \mu + \left[ 1 - \exp \left\{ \frac{\lambda}{n}\right\}\right]^{-1} (1 - \mu) < 0
    \end{equation*}
    for enough large $n$.
\end{lemma}
\begin{proof}
    Denote
    \begin{equation*}
        f(u) := \exp \left\{ \frac{\lambda \mu}{n (1 - \mu)}\right\} \mu + \left[ 1 - \exp \left\{ \frac{\lambda}{n}\right\}\right]^{-1} (1 - \mu) 
    \end{equation*}
    then $v = {\lambda / n} \downarrow 0$
    \begin{equation*}
        \begin{aligned}
            f'(u) & = \exp \left\{ \frac{\lambda \mu}{n(1 - \mu)}\right\} \left[ 1 + \frac{\lambda \mu}{n(1 - \mu)^2}\right]- \frac{1}{e^{\lambda / n} - 1} \\
            & \geq 3 e^v - \frac{1}{e^v - 1} \longrightarrow -1
        \end{aligned}
    \end{equation*}
    with $f(\mu) < f(0) = - \infty$.
\end{proof}
\vspace{2ex}

The following graphs show the situation of $\lambda_2$ is fixed and grows with $s$.

\begin{figure}[!ht]
    \setlength{\lineskip}{1ex
    \centering
    \setcounter{subfigure}{0}
    \begin{tabu} to \linewidth {XX}
        \subcaptionbox{$\lambda_2$ is fixed}{\includegraphics[width=.5\textwidth]{sr-220125-fig1.pdf}} &
        \subcaptionbox{$\lambda_2$ grows with $|\mathcal{H}_{1, s}|$}{\includegraphics[width=.5\textwidth]{sr-220125-fig2.pdf}} 
    \end{tabu}
    \caption{$z$ represents the target function, we require it is larger than $0$ or it will be meaningless.}
    \label{Fig1}
\end{figure}

Hence, we naturally take $\lambda_2 = \alpha_2 (|\mathcal{H}_{1, s}| + 1)$ with some $\alpha_2 \geq \log (0.5 + \sqrt{5} / 2)$ which guarantees $a_{k, s} > 0$, then the bound for $a_{k, s}$ will become
\begin{equation*}
    a_{k, s} \leq \exp \{ \alpha_2 \lambda_1^{-1} \mu_1 \} \bigg[ \mu_1 \exp \left\{ \frac{\alpha_2 \mu_1}{(1 - \mu_1)} \right\} - \frac{1 - \mu_1}{e^{\alpha_2} - 1}  \bigg]^{|\mathcal{H}_{1, s}|}
\end{equation*}
which can be further constrained by $\mu_1 > 1 / 2$ and $\mu_1 < 1 - \log (e^{\alpha_2} + 1 / (e^{\alpha_2} - 1))$ to give $a_{k, s}$ is bounded. However, this condition will make the whole thing a little bit strange.

\subsubsection{Another way to construction of $\overline{Y}_{1, s}$.}

We use the nonrandom denominator type of $\overline{Y}_{1, s}$ such that
\begin{equation*}
    \overline{Y}_{1, s} = \left[{\color{red}{\lambda}} \cdot \omega_0 + \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \omega_{\ell} R_{1, \ell}\right] / \Big[\big( 1 + |\mathcal{H}_{1, s}| \big) * |\mathcal{H}_{1, s}| \Big],
\end{equation*}
then similarly, 
\begin{equation*}
    \begin{aligned}
        \mathbb{P}_{\omega} (\overline{Y}_{1, s} > \mu_1 \, | \, \mathcal{H}_{1, s}) \, \geq \, \mathbb{P}_{\omega}\big(\omega_0 > |\mathcal{H}_{1, s}|^{-1} \lambda^{-1} \mu_1 \big) \prod_{\ell = 1}^{|\mathcal{H}_{1, s}|} \mathbb{P}_{\omega} (R_{1, \ell} \omega_{\ell} > |\mathcal{H}_{1, s}|^{-1}\mu_1 ).
    \end{aligned}
\end{equation*}
And Lemma \ref{lem1} will give the bound
\begin{equation*}
    \begin{aligned}
        a_{k, s} & \leq \exp \left\{ |\mathcal{H}_{1, s}|^{-1}\lambda^{-1} \mu_1 \right\} \prod_{\ell = 1}^{|\mathcal{H}_{1, s}|} (\mu_1 / |\mathcal{H}_{1, s}|) e^{\mu_1 / |\mathcal{H}_{1, s}|}\\
        & = \exp \left\{\big(1 + |\mathcal{H}_{1, s}|^{-1}\lambda^{-1} \big) \mu_1\right\} \mu_1^{|\mathcal{H}_{1, s}|} / |\mathcal{H}_{1, s}|^{|\mathcal{H}_{1, s}|} \leq \mu_1.
    \end{aligned}
\end{equation*}
Here we have bounded $a_{k, s}$.
\fi

\subsection{Using the good events to restrict the outliers of $a_{k, s}$.}

Denote the shifted version of average reward as
\begin{equation*}
    \overline{R}_{k, s}^* := \frac{\lambda_1 + \sum_{\ell = 1}^s R_{k, \ell}}{s + \lambda_2}
\end{equation*}
Specify some good events such that $A_{k, s} := \{ \overline{R}_{k, s}^* - \mu_k > - C_1 \Delta_k\}$ (or $A_{k, s} := \{ \overline{R}_{k, s}^* - \mu_k < C_2 \Delta_k\}$) with some $C_1, C_2 > 0$ determined later, and define
\begin{equation*}
    a_{k, s, 1} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) \Big], \qquad a_{k, s, 2} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}^c) \Big],
\end{equation*}
with $s_{a, j} (T) := \min \{ s : a_{k, s, j} \leq T^{-1}\}$ (it is well-defined by the fact $\mu_1 > 0.5$). In this step, our target is to show that $\max_{j = 1, 2} s_{a, j} (T) \lesssim \log T$. Indeed, if we have proved this, then 
\begin{equation*}
    \begin{aligned}
        a_k & \leq M(\mu_1, \lambda_1, \lambda_2) \cdot \max_{j = 1, 2} s_{a, j} (T) + (T - \max_{j = 1, 2} s_{a, j} (T)) \cdot 2 T^{-1} \\
        & \lesssim 2 + M(\mu_1, \lambda_1, \lambda_2) \log T.
    \end{aligned}
\end{equation*}
As a result, it is sufficient to show that $\max_{j = 1, 2} s_{a, j} (T) \lesssim \log T$. 

\subsubsection{Tackling $a_{k, s, 2}$}

The construction of $\overline{R}_{k, s}^*$ indicates the behave of $\overline{R}_{k, s}^*$ is quite similar to the true value $\overline{Y}_{k, s}$, so we study the probability of the bad event $A_{1, s}^c$ (using the first definition) with Hoeffding inequality
\begin{equation*}
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \bigg\{ \sum_{i = 1}^s R_{1, i} \leq (\mu_1 - C_1 \Delta_k) (s + \lambda_2) - \lambda_1 \bigg\} \\
        & = \mathbb{P} \Big( \overline{R}_{1, s} - \mu_1 \leq s^{-1} \big[ \mu_1 \lambda_2 - \lambda_1 - C_1 \Delta_k (s + \lambda_2)\big]\Big) \\
        (\text{letting }\lambda_2 = 0) & = \mathbb{P} \Big( \mu_1 - \overline{R}_{1, s} \geq s^{-1} (\lambda_1 + C_1 \Delta_k s)\Big) \\
        & \leq \exp \bigg\{ - \frac{2 (\lambda_1 + C_1 \Delta_k s)^2}{s}\bigg\} \\
        (\text{letting }\lambda_1 = \lambda s) & = \exp \big\{ - 2 s (\lambda + C_1 \Delta_k)^2\big\} \leq T^{-\eta}
    \end{aligned}
\end{equation*}
for any $\eta > 1$ if we take $s \geq s_{a, 2} := \frac{\eta \log T}{2 (\lambda + C_1 \Delta_k)^2}$, and hence we will get
\begin{equation*}
    a_{k, s, 2} \leq T \cdot T^{-\eta} = T^{-\eta + 1}.
\end{equation*}


\subsubsection{Dealing with $a_{k, s, 1}$}

To study $a_{k, s, 1}$, we note that the fact
\begin{equation*}
    \begin{aligned}
        \overline{Y}_{1, s} - \overline{R}_{1, s}^* \, | \, \mathcal{H}_{1, T} & = \frac{\lambda s (s - \sum_{i = 1}^s \omega_i) + s \sum_{i = 1}^s R_{1, i} \omega_i - \sum_{j = 1}^s R_{1, j} \sum_{i = 1}^s \omega_i }{s \sum_{i = 1}^s \omega_{i}},
    \end{aligned}
\end{equation*}
hence, $\overline{Y}_{1, s} - \overline{R}_{1, s}^* \, | \, \mathcal{H}_{1, T}$ may not be always positive. We need to define a series of good events such that 
\begin{gather*}
    G_{k, s} := \{\overline{Y}_{k, s} - \overline{R}_{k, s}^* \geq 0 \}, 
\end{gather*}
for some fixed $N$ free of $s$, then
\begin{equation*}
    \begin{aligned}
        \mathbb{P} \big(G_{1, s}^{c}\, | \, \mathcal{H}_{1, T}\big) & = \mathbb{P}_{\omega} \Bigg\{ \sum_{i = 1}^s \bigg[ \lambda s + \sum_{j = 1}^s R_{1, j} - s R_{1,i} \bigg] \omega_i  > \lambda s^2 \Bigg\}\\
        & \leq \exp \bigg\{ -\frac{1}{2} \frac{\lambda^2 s^4}{\|\lambda s + \sum_{j = 1}^s R_{1, j} - s R_{1,i} \|_2^2 } \wedge \frac{\lambda s^2}{\max_{i \in [s]}[\lambda s + \sum_{j = 1}^s R_{1, j} - s R_{1,i} ]}\bigg\} \\
        & \leq \exp \bigg\{ -\frac{1}{2} \frac{\lambda^2 s^4}{s(\lambda s + m_{1, s})^2} \wedge \frac{\lambda s^2}{\lambda s + m_{1, s}}\bigg\} \leq \exp \bigg\{ - \frac{s}{2} \frac{\lambda^2}{(\lambda + 1)^2} \wedge \frac{\lambda}{\lambda + 1} \bigg\}
    \end{aligned}
\end{equation*}
where $m_{1, s} = \sum_{i = 1}^s R_{1, i} \sim \operatorname{ber} (s, \mu_1)$. For any $\eta > 1$, by taking 
\begin{equation*}
     s_{a, 1, 2} = 2 \eta \left( \frac{\lambda + 1}{\lambda}\right)^2 \log T 
\end{equation*}
then for any $s \geq s_{a, 1, 2}$
\begin{equation*}
    a_{k, 1, 2} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) 1(G_{1, s}^c)\Big] \leq T^{- \eta + 1}.
\end{equation*}
Now what remains are that 
\begin{equation*}
    a_{k, 1, 1} = a_{k, 1} - a_{k, 1, 2} = {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) 1(G_{1, s})\Big].
\end{equation*}
Note that $\{N_{1, s}\left(\tau_{k}\right)<T^{-1}\} \supseteq \{\left[1-Q_{1, s}\left(\tau_{k}\right)\right]<T^{-2}\}$, what we need to deal with is finding $s \geq s_{a,1,1}$ such that $\big\{[1-Q_{1,s}(\tau_{k})]<T^{-2}\big\}$ on the event $A_{1, s} \cap G_{1, s}$. Denote
\begin{equation*}
    \Delta_1^* := \tau_1 - \overline{R}_{1, s}^*,
\end{equation*}
if $\Delta_1^* \leq 0$, i.e. $\lambda \geq \tau_1$ (recall $\tau_k \leq \mu_1$), then
\begin{equation*}
    \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}) = \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Delta_1^* \, | \, \mathcal{H}_{1, T} \big)  1(A_{1, s}) 1(G_{1, s}) = 0 < T^{-2};
\end{equation*}
if $\Delta_1^* \geq 0$, in this case, $\lambda \leq \tau_1 \leq \mu_1$, then
\begin{equation*}
    \begin{aligned}
        & \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}) = \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Delta_1^* \, | \, \mathcal{H}_{1, T} \big)  1(A_{1, s}) 1(G_{1, s}) \\
        = \, & \mathbb{P}_{\omega} \Bigg\{ \sum_{i = 1}^s \bigg[ (\lambda + \Delta_1^* - R_{1,i}) s + \sum_{j = 1}^s R_{1, j}\bigg] w_i \geq \lambda s^2 \Bigg\} 1 (A_{1, s}) 1(G_{1, s}) \\
        \leq \, & \exp \bigg\{ -\frac{1}{2} \frac{\lambda^2 s^4}{\|(\lambda + \Delta_1^* - R_{1,i}) s + \sum_{j = 1}^s R_{1, j}\|_2^2 } \wedge \frac{\lambda s^2}{\max_{i \in [s]}[(\lambda + \Delta_1^* - R_{1,i}) s + \sum_{j = 1}^s R_{1, j}]}\bigg\} 1 (A_{1, s} \cap G_{1, s}) \\
        \leq \, & \exp \bigg\{ -\frac{s}{2} \frac{\lambda^2}{(\lambda + 1 + \Delta_1^*)^2} \wedge \frac{\lambda}{\lambda + 1 + \Delta_1^*}\bigg\} 1 (A_{1, s} \cap G_{1, s}).
    \end{aligned}
\end{equation*}
Recast that, on $A_{1, s}$, we have
\begin{equation*}
    A_{1, s} = \big\{ \overline{R}_{1, s}^* - \mu_1 > - C_1 \Delta_k \big\} = \big\{ \Delta_1^* < C_1 \Delta_k - \mu_1 + \tau_1 \big\}
\end{equation*}
and hence
\begin{equation*}
    \begin{aligned}
        \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}) & = \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq \Delta_1^* \, | \, \mathcal{H}_{1, T} \big)  1(A_{1, s}) 1(G_{1, s}) \\
        & \leq \exp \bigg\{ -\frac{s}{2} \frac{\lambda^2}{(C_1 \Delta_k - \mu_1 + \tau_1)^2} \wedge \frac{\lambda}{C_1 \Delta_k - \mu_1 + \tau_1}\bigg\} 1 (A_{1, s} \cap G_{1, s}) \\
        (\text{letting } \tau_1 = \mu_1) & \leq  \exp \bigg\{ -\frac{s}{2} \frac{\lambda^2}{(C_1 \Delta_k )^2} \wedge \frac{\lambda}{C_1 \Delta_k}\bigg\} \leq T^{-\eta}
    \end{aligned}
\end{equation*}
with letting $s \geq s_{a, 1, 1} := 2 \eta \log T \left[ ( \lambda^{-1} C_1 \Delta_k)^2 \vee ( \lambda^{-1} C_1 \Delta_k) \right]$.
\vspace{2ex}

Now let $\eta = 2$, conclude from the Section 2.2.1 and 2.2.2, we know that as long as 
\begin{equation*}
    \begin{aligned}
        s \geq \max \{ s_{a, 2}, s_{a, 1, 2}, s_{a, 1, 1} \} & =  \max \left\{ \frac{1}{(\lambda + C_1 \Delta_k)^2}, \, 4 \left( \frac{\lambda + 1}{\lambda}\right)^2, \, 4 ( \lambda^{-1} C_1 \Delta_k)^2 \vee ( \lambda^{-1} C_1 \Delta_k) \right\} \cdot \log T \\
        & =: H(\lambda, C_1, \Delta_k) \cdot \log T
    \end{aligned} 
\end{equation*}
we must have 
\begin{equation*}
    \begin{aligned}
        a_{k} & \leq M (\mu_1, \lambda_1, \lambda_2) H(\lambda, C_1, \Delta_k) \log T + 3 \big[ T - H(\lambda, C_1, \Delta_k) \log T \big] T^{-1} \\
        & \leq 3 + M (\mu_1, \lambda s, 0) H(\lambda, C_1, \Delta_k) \log T = 3 + \mu_1 H(\lambda, C_1, \Delta_k) \log T.
    \end{aligned}
\end{equation*}

\section{Third Step}

The final step is bounding $b_k = \mathbb{P} \big( Q_{k, s} (\tau_k) > T^{-2}\big) + 1$. The technique is quite similar to the second step, we decompose $b_k = b_{k, 1, 1} + b_{k, 1, 2} + b_{k, 2} + 1$ as
\begin{gather*}
    b_{k, 1, 1} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-2}) 1(A'_{k, s}) 1(G_{k, s})\big] \\
    b_{k, 1, 2} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-2}) 1(A'_{k, s}) 1(G_{k, s}^c)\big] \\
    b_{k, 2} := {\mathbb{E}} \big[ 1( Q_{k, s} (\tau_k) > T^{-2}) 1(A_{k, s}^{\prime c} )\big].
\end{gather*}
with $A'_{k, s} = A_{k, s}$ using the second definition, then 
\begin{equation*}
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \bigg\{ \sum_{\ell = 1}^s (R_{1, \ell} - \mu_1) \geq s C_2 \Delta_1 - \lambda s \mu_1\bigg\} \\
        & \leq \exp \big\{ -2 \big[C_2 \Delta_1 - \lambda \big]^2 s \big\}.
    \end{aligned}
\end{equation*}
\begin{remark}
    The above inequality confirms why $\lambda$ cannot be too large since we need that $\lambda < C_2 \Delta_1$.
\end{remark}
Hence, define a new $s_{a, 2}' = \frac{2 \log T}{2 (C_2 \Delta_k - \lambda)^2}$, we must have $b_{k, 1, 2} \vee b_{k, 2} \leq T^{-2} < T^{-1}$ as long as $s \geq s'_{a, 2} \vee s_{a,1, 2}$. It remains to bound $b_{k, 1, 1}$. First, we note that on $A’_{k, s}$, we have
\begin{equation*}
    A'_{k, s} = \big\{ \overline{R}_{k, s}^* - \mu_k < C_2 \Delta_k \big\} = \big\{ \tau_k - C_2 \Delta_k - \mu_k < \Delta_k^* \big\}
\end{equation*}
it is worthy to recall that the construction of $s_{a, 1, 2}$ is free of any $\tau_k$. Under this establishment,
\begin{equation*}
    \begin{aligned}
        Q_{k, s} (\tau_k) 1(A_{k, s}) 1(G_{k, s}) & = \mathbb{P} \big(\overline{Y}_{k, s} - \overline{R}_{k, s}^* > \Delta_k^{*} \, | \, \mathcal{H}_{k, T} \big) 1(A_{k, s}) 1(G_{k, s}) \\
        & = \mathbb{P}_{\omega} \Bigg\{ \sum_{i = 1}^s \bigg[ (\lambda + \Delta_k^{*} - R_{k,i}) s + \sum_{j = 1}^s R_{k, j}\bigg] w_i < \lambda s^2 \Bigg\} 1 (A_{k, s}) 1(G_{k, s}) 
    \end{aligned}
\end{equation*}
{\color{red}{
Note that $ \sum_{i = 1}^s (\lambda + \Delta_k^{*} - R_{k,i}) s$ is
a linear function of $s^2$ with coefficient large than $\lambda$ and $\omega_i$ also has a positive shift $s \sum_{i = 1}^s R_{k, j}$, take $\tau_k$ large enough and for some $s \geq s_{a, 1, 1}'$, we have $Q_{k, s} (\tau_k) 1(A_{k, s}) 1(G_{k, s}) \leq T^{-1}$. Hence we also have $b_k \lesssim \log T$.
}}

\newpage

\begin{equation*}
    \begin{aligned}
        Q_{k, s} (\tau_k) 1(A_{k, s}) 1(G_{k, s}) & = \mathbb{P} \big(\overline{Y}_{k, s} - \overline{R}_{k, s}^* > \Delta_k^{*} \, | \, \mathcal{H}_{k, T} \big) 1(A_{k, s}) 1(G_{k, s}) \\
        & = \mathbb{P}_{\omega} \Bigg\{ \sum_{i = 1}^s \bigg[ (\lambda + \Delta_k^{*} - R_{k,i}) s + \sum_{j = 1}^s R_{k, j}\bigg] w_i < \lambda s^2 \Bigg\} 1 (A_{k, s}) 1(G_{k, s}) \\
        & \leq \sum_{i = 1}^s \mathbb{P}_{\omega}  \Bigg\{\bigg[ (\lambda + \Delta_k^{*} - R_{k,i}) s + \sum_{j = 1}^s R_{k, j}\bigg] w_i < \lambda s \Bigg\} 1 (A_{k, s}) 1(G_{k, s}) \\
        & \leq \sum_{i = 1}^s \bigg[ 1 - \exp \left\{ - \frac{\lambda}{(\lambda + \Delta_k^{*} - R_{k,i}) + \overline{R}_{k, s}}\right\}\bigg] 1 (A_{k, s}) 1(G_{k, s}) \\
        & \leq \sum_{i = 1}^s \bigg[ 1 - \exp \left\{ - \frac{\lambda}{(\lambda + \tau_k - C_2 \Delta_k - \mu_k - R_{k,i}) + \overline{R}_{k, s}}\right\}\bigg]  1 (A_{k, s}) 1(G_{k, s}) \\
        & \leq \sum_{i = 1}^s \frac{\lambda}{(\lambda + \tau_k - C_2 \Delta_k - \mu_k - R_{k,i}) + \overline{R}_{k, s}} \leq \sum_{i = 1}^s .
    \end{aligned}
\end{equation*}
where the last step we take $\tau_k = $

From the result in Section 2, we immediately have $s \geq H(\lambda, C_1, \Delta_k) \log T$, $b_{k, 1, 2} \vee b_{k, 2} \leq T^{-2}$. It remains to bound $b_{k, 1, 1}$, we note that
all the results in Section 1 hold for any $\tau_k$. Hence, we re-define a new 
\begin{equation*}
    \Delta_k^{**} := \tau_k^{\prime} - \overline{R}_{k, s}^*
\end{equation*}
different from the original $\Delta_k^*$ with the different $\tau_k$, then 
\begin{equation*}
    \begin{aligned}
        Q_{k, s} (\tau_k) 1(A_{k, s}) 1(G_{k, s}) & = \mathbb{P} \big(\overline{Y}_{k, s} - \overline{R}_{k, s}^* > \Delta_k^{**} \, | \, \mathcal{H}_{k, T} \big) 1(A_{k, s}) 1(G_{k, s}) \\
        & = \mathbb{P}_{\omega} \Bigg\{ \sum_{i = 1}^s \bigg[ (\lambda + \Delta_k^{**} - R_{k,i}) s + \sum_{j = 1}^s R_{k, j}\bigg] w_i < \lambda s^2 \Bigg\} 1 (A_{k, s}) 1(G_{k, s}) \\
        & \leq \sum_{i = 1}^s \mathbb{P}_{\omega}  \Bigg\{\bigg[ (\lambda + \Delta_k^{**} - R_{k,i}) s + \sum_{j = 1}^s R_{k, j}\bigg] w_i < \lambda s \Bigg\} 1 (A_{k, s}) 1(G_{k, s}) \\
        & \leq \sum_{i = 1}^s \bigg[ 1 - \exp \left\{ - \frac{\lambda s}{(\lambda + \Delta_k^{**} - R_{k,i}) s + \sum_{j = 1}^s R_{k, j}}\right\}\bigg] 1 (A_{k, s}) 1(G_{k, s}) \\
        & \leq  \sum_{i = 1}^s \bigg[ 1 - \exp \left\{ - \frac{\lambda}{\lambda + \Delta_k^{**}}\right\}\bigg] 1 (A_{k, s}) 1(G_{k, s}).
    \end{aligned}
\end{equation*}
Again on $A_{k, s}$, we have
\begin{equation*}
    A_{k, s} = \big\{ \overline{R}_{k, s}^* - \mu_k > -C_1 \Delta_k \big\} = \big\{ \big\}
\end{equation*}


\begin{equation*}
    \begin{aligned}
        \mathbb{P} (A_{1, s}^c) & = \mathbb{P} \bigg\{ \sum_{\ell = 1}^s (R_{1, \ell} - \mu_1) \geq (s + \lambda_2) C_2 \Delta_1 - (\lambda_1 - \lambda_2 \mu_1)\bigg\} \\
        & \leq \exp \Big\{ -2 \big[ (1 + s^{-1} \lambda_2) C_2 \Delta_1 - s^{-1} (\lambda_1 - \lambda_2 \mu_1) \big]^2 s \Big\}.
    \end{aligned}
\end{equation*}
This implies $\lambda_1 = O(s)$ and $\lambda_2 = o(s)$ is a possible choice for a small enough bad event probability. For instance, we take $\lambda_1 = \lambda \cdot s$ and $\lambda_2 \equiv 0$, we have
\begin{equation*}
    \mathbb{P} (A_{1, s}^c) \leq \exp \big\{ -2 (C_2 \Delta_1 - \lambda)^2 s\big\} \leq T^{-\eta}
\end{equation*}
for any $\eta > 1$, which implies taking $s = s_{a, 2} := \frac{\eta \log T}{2 (C_2 \Delta_1 - \lambda)^2}$ can give $\mathbb{P} (A_{1, s}^c) \leq 1 / T^{\eta}$, and then $a_{k, s, 2} \leq T^{-\eta + 1}$. 

\begin{equation*}
    \begin{aligned}
        \mathbb{P} \big(G_{1, s}^{(1)c}\big) & = \mathbb{P}_{\omega} \bigg\{ \lambda (s - \sum_{\ell = 1}^s \omega_{\ell}) + \sum_{\ell = 1}^s \omega_{\ell} R_{1, \ell} - \sum_{\ell = 1}^s \omega_{\ell} \sum_{\ell = 1}^s R_{1, \ell} <  0 \, \Big| \, \mathcal{H}_{1, T}\bigg\} \\
        & = \mathbb{P}_{\omega} \Bigg\{ \sum_{i = 1}^s \bigg[ \sum_{j \neq i} R_{1, j} + \lambda \bigg] \omega_i > \lambda s \, \bigg| \, \mathcal{H}_{1, T}\Bigg\} \\
        & \leq \exp \bigg\{ - \frac{1}{2} \frac{\lambda^2 s^2}{\| \sum_{j \neq i} R_{1, i} + \lambda \|_2^2} \wedge \frac{\lambda s}{\max_{i\in [s]} (\sum_{j \neq i}^s R_{1, j} + \lambda)}\bigg\} \\
        & \leq \exp \bigg\{ - \frac{1}{2} \frac{\lambda^2 s^2}{s(m_1 + \lambda)^2} \wedge \frac{\lambda s}{m_1 + \lambda}\bigg\} \\
        & \leq \exp \bigg\{ - \frac{s}{2} \frac{\lambda^2}{(\log \log T + \lambda)^2} \wedge \frac{\lambda}{\log \log T + \lambda}\bigg\}
    \end{aligned}
\end{equation*}
where $m_k = \sum_{i = 1}^s R_{k, i}$. By taking
\begin{equation*}
    s_{a, 1, 1}^{(1)} = 2 \eta \log T \bigg\{ \frac{(\log \log T + \lambda)^2}{\lambda^2} \vee \frac{\log \log T + \lambda}{ \lambda} \bigg\}
\end{equation*}
we have $\mathbb{P} \big(G_{1, s}^{(1)c} \big) \leq 1 / T^{\eta}$.

Similarly, for any $m \in \{ 2, \ldots, N\}$, 
\begin{equation*}
    \begin{aligned}
        \mathbb{P} \big( G_{1, s}^{(m) c}\big)
    \end{aligned}
\end{equation*}

On the other hand, to study $a_{k, s, 1}$, we note that the fact
\begin{equation*}
    \left\{N_{1, s}\left(\tau_{k}\right)<T^{-1}\right\} \subseteq \left\{\left[1-Q_{1, s}\left(\tau_{k}\right)\right]<T^{-2}\right\},
\end{equation*}
and
\begin{equation*}
    \begin{aligned}
        \overline{Y}_{1, s} - \overline{R}_{1, s} \, | \, \mathcal{H}_{1, T} & = \frac{s \omega_0 \lambda_1 + s \sum_{\ell = 1}^{s} (\omega_{\ell} - 1) R_{1, \ell} - \sum_{\ell = 1}^{s} (\omega_{\ell} - 1) \sum_{\ell = 1}^{s}R_{1, \ell}}{\left(\lambda_2 + \sum_{\ell = 1}^{s} \omega_{\ell}\right) s}\\
        & = 
    \end{aligned}
\end{equation*}
Hence, we have 
\begin{equation*}
    \begin{aligned}
        \left\{\left[1-Q_{1, s}\left(\tau_{k}\right)\right]<T^{-2}\right\} 1(A_{1, s}) & = \big\{ \mathcal{H}_{1, T} : \mathbb{P} (\overline{Y}_{1, s} \leq \tau_1 \, | \, \mathcal{H}_{1, T}) \leq T^{-2} \big\} 1(A_{1, s}) \\
        & \leq \big\{ \mathcal{H}_{1, T} : \mathbb{P} (\overline{R}_{1, s} \leq \tau_1 \, | \, \mathcal{H}_{1, T} ) \leq T^{-2} \big\} 1(A_{1, s})
    \end{aligned}
\end{equation*}

Here, we take advantage that on event $G_{k, s} = \cup_{m = 1}^{N} G_{k, s}^{(m)}$, we have either 
\begin{itemize}
    \item[(i)] $\overline{Y}_{k, s} \geq \overline{R}_{k, s}^* $;
    \item[(ii)] or $|\mathcal{H}_{k, T}| \geq \log \log T$ but $\overline{Y}_{k, s} < \overline{R}_{k, s}^* $.
\end{itemize}

{\color{red}{
For the case (i), denoted by $G_{1,s}^{(1)}$, we have on $G_{1, s}^{(1)}$, there must exist some constants $C_3 > 1$ such that $\overline{Y}_{k, s} \leq C_3 \overline{R}_{k, s}^* $
\begin{equation*}
    \begin{aligned}
        \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}^{(1)}) & = \mathbb{P} (\overline{Y}_{1, s} \leq \tau_1 \, | \, \mathcal{H}_{1, T})  1(A_{1, s}) 1(G_{1,s}^{(1)}) \\
        & \leq \mathbb{P} (3 \overline{R}_{1, s}^* \leq \tau_1 \, | \, \mathcal{H}_{1, T}) 1(A_{1, s}) 1(G_{1,s}^{(1)}) \\
        & \leq \mathbb{P}_{\omega} ( \overline{R}_{1, s} \leq \tau_1) 1 (A_{1, s}) 1(G_{1, s}^{(1)})
    \end{aligned}
\end{equation*}
}}

For the case (ii), define $n_{k, s} = s - m_{k, s} \leq s - \log \log T$,
\begin{equation*}
    \begin{aligned}
        & \big[1-Q_{1, s}(\tau_{k})\big]  1(A_{1, s}) 1(G_{1,s}^{(1)})\\
        = \, & \mathbb{P} \big(\overline{Y}_{1, s} - \overline{R}_{1, s}^* \leq - [\overline{R}_{1, s}^* - \tau_1] \, | \, \mathcal{H}_{1, T} \big)  1(A_{1, s}) 1(G_{1,s}^{(2)}) \\
        = \, & \mathbb{P} \big( \overline{R}_{1, s}^* -\tau_1 \leq 0 \big)  1(A_{1, s}) 1(G_{1,s}^{(2)}) \\
        \leq \, & \mathbb{P} \bigg\{ \sum_{i = 1}^{m_{1, s}} R_{1, i} \leq m_{1, s} (\tau_1 - \lambda)\bigg\} 1(A_{1, s}) < T^{-2}
    \end{aligned}
\end{equation*}
which implies 
\begin{equation*}
    TBW
\end{equation*}

\newpage
{\color{red}{
    For simplify this question, we first assume some specific distributions. 
        
        \begin{itemize}
            \item the Gaussian distribution $R_{1, \ell } \sim N(\mu_1, \sigma^2)$ (used in \cite{wang2020residual}) requires $R_{1, \ell } - \mu_1 > 0$ guaranteeing an upper bound $\lesssim \frac{1}{\sigma \sqrt{\pi}} G^{3, 0}_{0, 0}\left( \left. - \frac{1}{2} \ 0 \ 0\right| \frac{1}{8 \sigma^2}\right)$ with Meijer G-function. However, if we do not require $R_{1, \ell } - \mu_1  > 0$, then ${\mathbb{E}} \exp \left\{ 1 / (R_{1, \ell } - \mu_1) \right\} = \infty$;
            
            \item  another common assumption is $R_{1, \ell} \sim \operatorname{ber} (\mu_1, 1)$. In this case,
            \begin{equation*}
                {\mathbb{E}} \exp \left\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| (R_{1, \ell} - \mu_1)} \right\} = \exp \left\{ - \frac{\lambda_2}{|\mathcal{H}_{1, s}|}\right\} (1 - \mu_1) + \exp \left\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}|(1 - \mu_1)}\right\} \mu_1.
            \end{equation*}
            Note that For any positive number $\mu, \lambda > 0$ and $n \in \mathbb{N}$, we have
                \begin{equation*}
                    \begin{aligned}
                        & \Bigg[ \exp \bigg\{ - \frac{\lambda}{n}\bigg\} (1 - \mu) + \exp \bigg\{ \frac{\lambda \mu}{n (1 - \mu)}\bigg\} \mu \Bigg]^n \\
                        = & e^{-\lambda} (1 - \mu)^n \Bigg[ 1 + \frac{1}{n} \frac{n \mu}{1 - \mu} \exp \bigg\{ \frac{n}{\lambda} + \frac{\lambda \mu}{n(1 - \mu)}\bigg\} \Bigg]^n \\
                        \leq & e^{- \lambda} (1 - \mu)^n \exp \Bigg\{ \frac{n \mu}{1 - \mu} \exp \bigg\{ \frac{n}{\lambda} + \frac{\lambda \mu}{n(1 - \mu)}\bigg\}\Bigg\} \\
                        = & \exp \Bigg\{ n 
                         \log (1 - \mu) + \frac{n \mu}{1 - \mu} \exp \bigg\{ \frac{n}{\lambda} + \frac{\lambda \mu}{n(1 - \mu)}\bigg\} - \lambda \Bigg\}
                    \end{aligned}
                \end{equation*}
                by using the fact $(1 + x / n)^n \leq e^x$. Let the above inequality less than $1$, it sufficiently has $\mu > 1 / 2$, i.e. the optimal arm's mean reward large than $1/2$, this requirement also can be seen in \cite{wang2020residual}.
        \end{itemize}
        
        Up to now, we can conclude that for any $s \in [T]$, the $a_{k, s}$ has the upper bound
        \begin{equation*}
            \begin{aligned}
                a_{k, s} & \leq \exp \left\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| \lambda_1}\right\} \prod_{\ell = 1}^{|\mathcal{H}_{1, s}|} {\mathbb{E}} \exp \left\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| (R_{1, \ell} - \mu_1)} \right\} \\
                & = \exp \Bigg\{ \frac{\lambda_2 \mu_1}{|\mathcal{H}_{1, s}| \lambda_1} + |\mathcal{H}_{1, s}| \log (1 - \mu_1) + \frac{|\mathcal{H}_{1, s}| \mu_1}{1 - \mu_1} \exp \bigg\{ \frac{|\mathcal{H}_{1, s}|}{\lambda_1} + \frac{\lambda_1 \mu_1}{|\mathcal{H}_{1, s}|(1 - \mu_1)}\bigg\} - \lambda_1 \Bigg\}.
            \end{aligned}
        \end{equation*}
        with $\lambda_1$ and $\lambda_2$ still need to verify later. So we have partly finished section 2.1.
        
        \begin{itemize}
            \item[2.2] We denote $\overline{Y}_{1, s}^* := \frac{1}{|\mathcal{H}_{1, s}|} \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \omega_{\ell} R_{1, \ell}$ is the true weighted target function, and then
            \begin{equation*}
                \overline{Y}_{1, s} = \frac{\omega_0 \cdot \lambda_1 + |\mathcal{H}_{1, s}| \overline{Y}_{1, s}^*}{\lambda_2 + \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \omega_{\ell}}.
            \end{equation*}
            
            Specify some good events such that $A_{k, s} := \{ \overline{Y}_{k, s} - \mu_k > C_1 \Delta_k\}$ and $A_{k, s}^* := \{ \overline{Y}_{k, s}^* - \mu_k > C_1 \Delta_k\}$ and define
            \begin{equation*}
                a_{k, s, 1} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}) \Big], \qquad a_{k, s, 2} := {\mathbb{E}} \Big[ \big( N_{1, s} (\tau_k) \wedge T \big) 1 (A_{1, s}^c) \Big],
            \end{equation*}
            with $s_{a, j} (T) := \min \{ s : a_{k, s, j} \leq T^{-1}\}$ (it is well-defined by the fact $\mu_1 > 0.5$). In this step, we will show that $\max_{j = 1, 2} s_{a, j} (T) \lesssim \log T$.
        \end{itemize}
        
        \begin{itemize}
            \item[] \begin{itemize}
                \item[2.2.0] On the event $A_{1, s}^*$ and  $A_{1, s}$ 
            \end{itemize}
        \end{itemize}
        \begin{equation*}
            \begin{aligned}
                \mathbb{P} \big(A_{1, s}^* \, | \, \mathcal{H}_{1, s} \big) & = \mathbb{P} \bigg\{ \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} \frac{R_{1, \ell}}{|\mathcal{H}_{1, s}|} (\omega_{\ell} - 1) \geq C_1 \Delta_k + \mu_1 - \overline{R}_{1, s}\bigg\} \\
                & \leq \exp \bigg\{ - \frac{1}{2} \frac{|\mathcal{H}_{1, s}|(C_1 \Delta_k + \mu_1 - \overline{R}_{1, s})^2}{\sqrt{\sum_{\ell = 1}^{|\mathcal{H}_{1,s}|} R_{1, \ell}^2}} \wedge \frac{|\mathcal{H}_{1, s} | (C_1 \Delta_k + \mu_1 - \overline{R}_{1, s})}{\max_{\ell\in [|\mathcal{H}_{1, s}|]} R_{1, \ell}}\bigg\} \\
                & \lesssim \exp \Big\{  - \frac{1}{2}(C_1 \Delta_k + \mu_1 - \overline{R}_{1, s})^2 \Big\},
            \end{aligned}
        \end{equation*}
        and
        \begin{equation*}
            \begin{aligned}
                & \mathbb{P} \big(A_{1, s} \, | \, \mathcal{H}_{1, s} \big) \\
                = & \, \mathbb{P} \bigg\{ \lambda_1 \cdot (\omega_0 - 1) + \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} (R_{1, \ell} - \mu_1 - C_1 \Delta_k) \cdot (\omega_{\ell} - 1) \\
                & \qquad \qquad \qquad \qquad \qquad \qquad \geq \lambda_2 \cdot (\mu_1 + C_1 \Delta_k) - \lambda_1 + |\mathcal{H}_{1, s}| (\mu_1 + C_1 \Delta_k - \overline{R}_{1, s})\bigg\} \\
                \leq & \exp \Bigg\{ - \frac{1}{2} \frac{\big[ (\lambda_2 + |\mathcal{H}_{1, s}|) \mu_1 + C_1 (\lambda_2 + |\mathcal{H}_{1, s}|) \Delta_k - \lambda_1 - |\mathcal{H}_{1, s}| \overline{R}_{1, s}\big]^2}{\lambda_1^2 + \sum_{\ell = 1}^{|\mathcal{H}_{1, s}|} R_{1, \ell}^2} \\
                & \qquad \qquad \qquad \qquad \qquad \qquad \wedge \frac{(\lambda_2 + |\mathcal{H}_{1, s}|) \mu_1 + C_1 (\lambda_2 + |\mathcal{H}_{1, s}|) \Delta_k - \lambda_1 - |\mathcal{H}_{1, s}| \overline{R}_{1, s}}{\lambda_1 \,  \vee \, 1} \Bigg\} \\
                \overset{\lambda_1 = O(|\mathcal{H}_{1, s}|)}{\leq} &  \exp \Bigg\{ - \frac{1}{2}\frac{\big[ (\lambda_2 + |\mathcal{H}_{1, s}|) \mu_1 + C_1 (\lambda_2 + |\mathcal{H}_{1, s}|) \Delta_k - \lambda_1 - |\mathcal{H}_{1, s}| \overline{R}_{1, s}\big]^2}{\lambda_1^2 + s} \\
                &  \qquad \qquad \qquad \qquad \qquad \qquad \wedge \frac{(\lambda_2 + |\mathcal{H}_{1, s}|) \mu_1 + C_1 (\lambda_2 + |\mathcal{H}_{1, s}|) \Delta_k - \lambda_1 - |\mathcal{H}_{1, s}| \overline{R}_{1, s}}{\lambda_1}\Bigg\}
            \end{aligned}
        \end{equation*}
        by concentration inequality and the fact that $\lambda_1 \geq 1$ (or it will be meaningless). From this, a possible choose is {\color{blue}{$\lambda_1 = C_1 |\mathcal{H}_{1, s}|\Delta_k$ (?????)}} and $\lambda_2 = 0$, then
        \begin{equation*}
            \begin{aligned}
                \mathbb{P} (A_{1, s}) & = {\mathbb{E}} \big[ \mathbb{P} \big(A_{1, s} \, | \, \mathcal{H}_{1, s} \big) \big] \\
                & \leq {\mathbb{E}} \exp \left\{ - \frac{(\overline{R}_{1, s} - \mu_1)^2}{C_1 \Delta_k^2 + s}\right\} = 
            \end{aligned}
        \end{equation*}
        
        \begin{itemize}
            \item[] \begin{itemize}
                \item[2.2.1] Bound $s_{a, 1}(T)$: only need to note that
            \end{itemize}
        \end{itemize}
        
        \begin{equation*}
            \{ N_{1, s} (\tau_k) < T^{-1} \} \supseteq \left\{\left[1-Q_{1, s}(\tau_{k})\right]<T^{-2}\right\}
        \end{equation*}
        with
        \begin{equation*}
            \begin{aligned}
                \big[ 1-Q_{1, s}(\tau_{k}) \big] 1(G_{1, s}) & = \mathbb{P}_{\omega} \left(  \left\{ \sum_{\ell = 0}^{|\mathcal{H}_{1, s}|} (R_{1, \ell} - \mu_1) \cdot \omega_{\ell} \geq \lambda_2 \cdot \mu_1 \right\} \cap G_{1, s} \right) \\
                & = 
            \end{aligned}
        \end{equation*}
        the construction of $G_{1, s}$ is intuitive from the above event.
    }}

~~~~\\


{\color{red}{1. $R_{1, l}$ shifted version?}}



\subsection{Connection and why does MBTS work}







\begin{algorithm}[!h]
\SetAlgoLined
\setcounter{AlgoLine}{1}
\KwData{
number of arms $K$, 
multiplier weight distribution $\rho(\omega)$, 
tuning parameter $\lambda$
}


Set $\mathcal{H}_{k} = \{\}$ be the history of the arm $k$ and $\overline{Y}_{k} = +\infty, \forall k \in [K]$

\For{$t = 1, \dots, T$}{

    Pull $A_t = \argmax_{k \in [K]} \overline{Y}_{k}$ (break tie randomly), 

    Observe reward $R_t$
    
    Set $\mathcal{H}_{k} = \mathcal{H}_{k} \cup \{R_t\}$

    \For{$k = 1, \dots, K$}{
    \If{$|\mathcal{H}_{k}| > 0$}{
    
    Sample the multiplier weights $\{\omega_l, \omega'_l, \omega''_l\}_{l=1}^{|\mathcal{H}_{k}|} \sim \rho(\omega)$. 
    
    Update the mean reward 
    \begin{align*}
        \overline{Y}_{k} = \left(\sum_{\ell = 1}^{|\mathcal{H}_{k}|} (\omega_{\ell} \cdot R_{k, \ell} +  \omega_{\ell}' \cdot 1 \times \lambda + \omega_{\ell}'' \cdot 0 \times \lambda) \right) / \left( \sum_{\ell = 1}^{|\mathcal{H}_{k}|} (\omega_{\ell} + \lambda \omega_{\ell}' + \lambda \omega_{\ell}'') \right), 
    \end{align*}
    where $R_{k, l}$ is the $l$-th element in $\mathcal{H}_{k}$. 
   

    }
    }
} 

\caption{\name{MBE} for MAB with sub-Gaussian rewards with mean bounded in $[0, 1]$}\label{alg:MBTS-MAB-2}
\end{algorithm}















