\documentclass{article}
\usepackage{cse599s14sp}
\usepackage{url, tikz}

% Please use these commands where appropriate:
\newcommand{\Regret}{\operatorname{Regret}}
\newcommand{\Loss}{\operatorname{loss}}
\newcommand{\grad}{\triangledown}
% The following are defined in the style file, please use them as well:
% \newcommand{\field}[1]{\mathbb{#1}}
% \newcommand{\N}{\field{N}} % natural numbers
% \newcommand{\R}{\field{R}} % real numbers
% \newcommand{\Z}{\field{Z}} % integers
% \newcommand{\norm}[1]{\left\| #1\right\|}
% \DeclareMathOperator*{\argmin}{argmin}
% \DeclareMathOperator*{\argmax}{argmax}
% \DeclareMathOperator*{\maximize}{maximize}
% \DeclareMathOperator*{\minimize}{minimize}


%% the format for the lecture environment is
% \begin{lecture}{lecture number}{lecture title}{scribe}{lecture date}
\begin{lecture}{14}{Stochastic Bandits}{Matthias W. Smith}{Ofer Dekel}{05/14/2014}

\section{Stochastic Bandits}
This is a special case of the adversarial bandits covered in a previous lecture. 

\begin{tabular}{| l |}
\hline Stochastic Bandits Game \\ \hline
for rounds $1..T$ \\
\; player pulls a single arm per round $\{1 ... d\}$ \\
\; player receives reward $x_i$ \\ \hline
\end{tabular}

\noindent The game bakes in a few assumptions which distinguish it from the general adversarial bandits case.

\begin{itemize}
\item $\nu_1, \nu_2, ..., \nu_{d}$ are unknown distributions, supported on $\left[0, 1\right]$, over reward
\item pulling arm $i$ for the $s$'th time results in reward $x_{is} \sim \nu_{i}$ and $\left(x_{i1}, x_{i2}, ...\right)$ are independent
\end{itemize}

\noindent Essentially, the adversary generates the following table.

\[
\begin{array}{| l | l  l  l  l |}
\hline
	        & \nu_1  & \nu_2  & \hdots & \nu_d  \\ \hline
	1       & x_{11} & x_{21} & \hdots & x_{d1} \\
	2       & x_{12} & x_{22} & \hdots & x_{d2} \\
	\vdots  & \vdots & \vdots & \ddots & \vdots \\
	T       & x_{1T} & x_{2T} & \hdots & x_{dT} \\ 
\hline	  	  
\end{array}	
\]

\section{Probability Theory}
Before we dive into the stochastic bandits problem, we need to take a bit of a detour through some probability theory.

\subsection{Weak Law of Large Numbers}
We start by requiring that $x_1, ..., x_n$ are iid and ``well-behaved'' random variables. Then we can define the emprical mean as $\hat{\mu} = \frac{1}{n} \sum_{i = 1}^{n} x_i$. If we denote the expected mean $\mu = \E[x]$, then 
$\hat{\mu}$ converges to $\mu$ in probability.

\begin{theorem}
\[
	\forall \epsilon > 0, \lim_{n\rightarrow \infty} \PR\left(|\hat{\mu} 
	- \mu| \geq \epsilon \right) = 0
\]
\end{theorem}

The theorem is readily present in nearly all statistical learning theory.  In order to study \emph{sample complexity}, we need a non-asymptotic version of the law of large numbers (\emph{LLN}).

\subsection{Markov's Inequality}

\begin{theorem}
If $x$ is a non-negative r.v. and $\epsilon > 0$, then
$\PR(x \geq \epsilon) \leq \frac{\E[x]}{\epsilon}$
\end{theorem}

\begin{proof}
Start with the following assertion
\[
\epsilon \cdot \I{x \geq \epsilon} \leq x.
\]

\noindent We have two possible cases, $x \geq \epsilon$ and $x < \epsilon$. 
It is fairly easy to convince yourself that the statement must always be true. Now we proceed by taking the expectation of each side.

\[
	\epsilon \E[\I{x \geq \epsilon}] \leq \E[x]
\]\[
	\epsilon \PR[x \geq \epsilon] \leq \E[x]
\]

\noindent Thus proving Markov's Inequality.
\end{proof}

\subsection{Chebychev's Inequality}

\begin{theorem}
If $x$ is r.v. with $\mu = \E[x] < \infty$ and $\epsilon > 0$, then $\PR(|x - \mu|) > \epsilon) \leq \frac{Var(x)}{\epsilon^2}$.
\end{theorem}

\begin{proof}
Define $z = (x - \mu)^2$. By definition we can write
\[
	\PR(|x - \mu|) \equiv \PR(z \geq \epsilon^2),
\]
and by Markov's Inequality
\[
	\PR(z \geq \epsilon^2) \leq \frac{\E[z]}{\epsilon^2}.
\]

\end{proof}

\noindent Assume without losing generality that $\mu = 0$.  Noting that we could always define a new random variable $y = x - \mu$ where $\E[y] = 0$ now.

\begin{theorem}
Assume $\mu = 0$, then $z = x^2$ and
\[
	\PR(|x| \geq \epsilon) = \PR(z \geq \epsilon^2) \leq \frac{\E[x^2]}{\epsilon^2}.
\]
\end{theorem}

\begin{proof}
Use the Weak \emph{LLN} with $\mu = 0$, assuming $\hat{\mu} = \frac{1}{n}\sum_{i=1}^{n}{x_i}$, $x_1 ... x_n$ are iid and $\E[x_i^2] < \infty$. By definition we get the first relation

\[
	\E[\hat{\mu}^2] = \E[(\frac{1}{n} \sum_{i=1}^{n}{x_i})^2].
\]
Now use the fact that $x_i$ is a generic random variable and recast its squared sum as multiplication of two random variables to get
\[
	\E[\hat{\mu}^2] = \frac{1}{n^2} \E[\sum_{i=1}^{n} \sum_{j=1}^{n} x_i x_j].
\]
Separating terms in which $i = j$ and $i \neq j$,
\[
	\E[\hat{\mu}^2] = \frac{1}{n^2} \sum_{i=1}^{n} \E[x_i^2] 
	+ \sum_{i \neq j} \E[x_i x_j].
\]
Now we can eliminate the second term by using the linearity of expectation and the fact that we have defined $\mu = 0$.
\[
	\E[x_i x_j] = \E[x_i] \E[x_j] = 0 \cdot 0
\]\[
	\E[\hat{\mu}^2] = \frac{1}{n^2} \sum_{i=1}^{n} \E[x_i^2] 
	= \frac{1}{n} \E[x^2]
\]
Plugging back into the bound given by Chebychev's Inequality we get
\[
\Rightarrow \PR(|\hat{\mu} - \mu| \geq \epsilon) \leq 
\frac{\E[x^2]}{n \epsilon^2}.
\]
And as $n \rightarrow \infty$
\[
\frac{\E[x^2]}{n \epsilon^2} \rightarrow 0
\]

\end{proof}

\subsection{Confidence Intervals}
These are all over the place in Machine Learning.  Define $\delta = \frac{\E[x^2]}{n \epsilon^2}$, and similarly $\epsilon = \sqrt{\frac{\E[x^2]}{\delta n}}$. The term is denoted the confidence and it is defined on $\forall \delta \in [0, 1]$.
With probability ($w.p.$) $\geq 1 - \delta$
\[
	|\hat{\mu} - \mu| \leq \sqrt{\frac{\E[x^2]}{\delta n}}
\]

\noindent In diagram form the relation is shown below.

\begin{tikzpicture}
	% a straight line segment
	\draw (0.5,0) -- (10.5,0);
	% the ticks and their labels
	\draw[xshift=1 cm] (0pt,2pt) -- (0pt,-1pt) 
	node[below,fill=white] {0\relax};
	\draw[xshift=10 cm] (0pt,2pt) -- (0pt,-1pt) 
	node[below,fill=white] {1\relax};	
	% the thicker segment
	\draw[ultra thick] (4.5,0) -- (7.5,0);
	% the labels
	\node[fill=white,draw=black,circle,inner sep=2pt,label=above:{$\mu - \bigO(\sqrt{1/n})$}] at (4.5,0) {};
	\node[fill=white,draw=black,circle,inner sep=2pt,label=above:{$\mu + \bigO(\sqrt{1/n})$}] at (7.5,0) {};
	\draw[ultra thick,xshift=6.0 cm] (0pt, 3pt) -- (0pt, -2pt) 
	node at (0.0,-0.8) {$\mu$};
\end{tikzpicture}

\subsection{Hoeffding-Azuma Inequality}

\begin{theorem}
Instead of plugging in the 2nd moment ($\E[x^2]$) into Markov's Inequality use the ``exponential moment'', $z = e^{\lambda \sum_{i=0}{n} x_i}$, to get
\[
	\PR(\hat{\mu} > \epsilon) = \PR(z \geq e^{n \lambda \epsilon})
\]
\end{theorem}

\begin{proof}
Using Markov's Inequality on the righthand side we have
\[
	 \PR(z \geq e^{n \lambda \epsilon}) \leq \E[z] e^{-n \lambda \epsilon}.
\]
We can substitute for $z$ and turn the sum into a $pi$-product since it is in the exponent to get
\[
	\E[z] e^{-n \lambda \epsilon} = \E[\prod_{i=1}^{n} e^{\lambda x_i}] \cdot e^{-n \lambda \epsilon}.
\]\[
	\E[z] e^{-n \lambda \epsilon} = \prod_{i=1}^{n} \E[e^{\lambda x_i}] \cdot e^{-n \lambda \epsilon}
\]

Using convexity and a Taylor Expansion we get derive
\[
	\E[e^{\lambda x}] \leq e^{\lambda^2 / \delta}.
\]

If $x \in [a, b]$ with $b -  a = 1$, then $\E[x] = 0$.

\[
	\E[z] e^{-n \lambda \epsilon} = e^{n \lambda^2 / \delta - n \lambda \epsilon}
\]

\[
	\PR(\hat{\mu} > \epsilon) \leq \min_{\lambda > 0} e^{n(\frac{\lambda^2}{\delta} - \lambda \epsilon)}
\]
Taking the derivative to minimize we find that $\lambda = 4 \epsilon$, so
\[
	\PR(\hat{\mu} \geq \epsilon) \leq e^{-2 n \epsilon^2}
\]
and symmetrically
\[
	\PR(\hat{\mu} \leq -\epsilon) \leq e^{-2 n \epsilon^2}.
\]

\[
	\Rightarrow \PR(|\hat{\mu}| > \epsilon) \leq 2 e^{-2 n \epsilon^2}
\]
\end{proof}

\subsection{Confidence Intervals Revisited}
We can defined a new confidence interval
\[
	\delta = 2 e^{-2 n \epsilon^2}.
\]
and
\[
	\epsilon = \sqrt{\frac{\log{2/\delta}}{2n}}.
\]

We have shown: $\forall \delta \in [0, 1]$, $w.p. \geq 1 - \delta$
\[
	|\hat{\mu} - \mu| \leq \sqrt{\frac{\log{2 / \delta}}{2n}}
\]
\noindent Compare to our previous result we now have $\sqrt{\log{1/\delta}}$ versus $\sqrt{1/\delta}$. This result is heavily predicated by $x \in [a, b]$ where $b - a = 1$.

\section{Stochastic Bandits Revisited}
First we want to define some convenient variables: $\mu_i = \E_{x \sim \nu_i}[x]$ is the expected reward of arm $i$, $T_i(t)$ is the number of times arm $i$ is pulled on rounds $1...t$, and $\Delta_i = \mu^* - \mu_i$.

\noindent \textbf{Comments:}
\begin{itemize}
	\item the exact time that the arm was pulled doesn't matter, only $T_i(t)$ matters
	\item there is a ``best arm'', the one with the largest expetec reward $\mu^{*} = \max_{1 \leq i \leq d} \mu_i$
\end{itemize}

We can now take our regret
\[
	\Regret = T_{\mu^*} - \E[\sum_{t=1}^{T}X_{I_t, T_{I_t}(t)}]
\]
and recast it as
\[
	\Regret = T_{\mu^*} - \E[\sum_{t=1}^{T}\mu_{I_t}]
\]\[
	\Regret = \sum_{i=1}^{d} \Delta_i \E[T_i(t)].
\]

The goal for stochastic bandits is to bound  $\E[T_i(t)]$ for all $i$ with $\Delta_i > 0$.

\subsection{Algorithm}
The simplest feasible algorithm is dubbed ``$\epsilon$-first''. If we know $\Delta$, a lower bound of $\{\Delta_i: \Delta_i > 0\}$.

% \begin{tikzpicture}
% 	% a straight line segment
% 	\foreach \x in {1,...,4}
% 		\draw[xshift=2.0\x cm] (0.5,2.0) -- (0.5,0);
% 	\foreach \x in {1,...,4}
% 		\draw[xshift=2.0\x cm] (0.25,2.0) -- (0.75,2.0) 
% 		node[left, fill=white] {1 \relax};
% 	\foreach \x in {1,...,4}
% 		\draw[xshift=2.0\x cm] (0.25,0.0) -- (0.75,0.0)
% 		node[left, fill=white] {0 \relax};
% \end{tikzpicture}

\noindent We sample each arm $\bigO\left(\frac{\log(1/\delta)}{\Delta^2}\right)$ times, estimate each $\mu_i$ to within $\Delta / 2$, and stick to the emperical best arm henceforth.

\[
	\Regret \leq \sum_{i=1}^{d} \Delta_i \cdot \left(\frac{\log{1/\delta}}{\Delta^2}\right) + \Theta(\delta T) \sim \Theta(\log T)
\]

\end{lecture}
\theend
