\documentclass{article}
\usepackage{cse599s14sp}
\usepackage{url}
\usepackage{graphicx}
\usepackage{amsmath}

% Please use these commands where appropriate:
\newcommand{\Regret}{\operatorname{Regret}}
\newcommand{\Loss}{\operatorname{loss}}
\newcommand{\grad}{\triangledown}
\DeclareMathOperator*{\Expect}{{\rm I\kern-.3em E}}
% The following are defined in the style file, please use them as well:
% \newcommand{\field}[1]{\mathbb{#1}}
% \newcommand{\N}{\field{N}} % natural numbers
% \newcommand{\R}{\field{R}} % real numbers
% \newcommand{\Z}{\field{Z}} % integers
% \newcommand{\norm}[1]{\left\| #1\right\|}
% \DeclareMathOperator*{\argmin}{argmin}
% \DeclareMathOperator*{\argmax}{argmax}
% \DeclareMathOperator*{\maximize}{maximize}
% \DeclareMathOperator*{\minimize}{minimize}


\begin{lecture}{16}{Online Convex Optimization with Bandit Feedback}{Durmus Karatay}{Brendan McMahan}{05/22/2014}

\section{The Game}
We will analyze a general online convex optmimization problem, where we have a convex set $\mathcal{W}$ and a sequence of loss functions $f_t: \mathcal{W} \rightarrow \R$. However, the feedback that the player gets is a bandit feedback. The player only sees $f_t(w_t)$ in each round. 

\vspace{0.1in}

\begin{tabular}{|l|}
\hline
\textbf{Bandit Gradient Descent Game} \\
\hline
choose parameters $\eta, \alpha, \delta$ where $\alpha \in [0, 1]$. \\
$v_1 = 0 \in \mathcal{W}$ \\
for $t = 1, 2,...,T$ \\
\hspace{0.5in} choose unit vector $u_t \in \R^d$ uniformly at random \\
\hspace{0.5in} assign $w_t = v_t +\delta u_t \in \mathcal{W}$ \\
\hspace{0.5in} play $w_t$, observe $f_t(w_t)$ \\
\hspace{0.5in} $v_{t+1 } = \prod_{(1-\alpha)\mathcal{W}} (v_t - \eta \hat{g_t})$, where $\hat{g_t} = \frac{d}{\delta}f_t(w_t)u_t$\\
\hline
\end{tabular}
\vspace{0.1in}

\noindent
Note: Projection to $(1-\alpha)\mathcal{W}$ is required to make sure that $w_t$ stays inside the convex set.

\begin{figure}[h!]
	\centering
	\includegraphics[width=0.5\textwidth]{projection.png}
	\caption{Graphical Interpretation of the Bandit Gradient Descent Game}
\end{figure}

\section{Regret Bound of Bandit Gradient Descent}
The regret bound for online linear optimization is $\mathcal{O}(\sqrt{T})$ and the regret bound for the experts algorithm is  $\mathcal{O}(\sqrt{T\log{d}})$. The expected regret is calculated for the bandit gradient descent game, since it is uniformly randomized: 

\[
\Expect{[\Regret]} \leq \mathcal{O}(T^{\frac{3}{4}})
\]

\section{Analysis}
We will need two tricks to analyze the algorithm: (1) one point gradient estimation and (2) expected gradient-descent.
\subsection{One Point Gradient Estimation}
\begin{lemma}
$|u| = 1$ and choosen uniformly at random, $\delta > 0$,
\[
\nabla\hat{f}(x) = \Expect{[\frac{d}{\delta} f(v + \delta u)u]}.
\]
\end{lemma}

\begin{proof}
When d =1, $u \in [-1, +1]$,
\begin{align}
\Expect{[\frac{1}{\delta} f(v + \delta u)u]} & = \frac{1}{2} \frac{f(v + \delta)}{\delta} - \frac{1}{2} \frac{f(v - \delta)}{\delta}, \\
 & \cong f'(v).
\end{align}
It follows that
\begin{align}
\hat{f}(x) & = \Expect{[f(v + \delta u)]},\\
\nabla\hat{f}(x) & = \Expect{[\frac{d}{\delta} f(v + \delta u)u]} \text{, where } u: ||u||^2 < 1.
\end{align}
$\hat{f}$ is the smoothed version $f$. Note that $\hat{f}$ is differentiable even though $f$ is not.
\end{proof}

\subsection{Expected Gradient Descent}
Regret bound for online gradient descent is:
\begin{align}
\sum_{t=1}^T f_t(w_t) - \min_{u \in W} \sum_{t=1}^T f_t(u) & \leq \frac{B^2}{\eta} + \eta\frac{G^2 T}{2} \text{, where } \eta = BG\sqrt{T}, \\
\sum_{t=1}^T f_t(w_t) - \min_{u \in W} \sum_{t=1}^T f_t(u) & \leq BG\sqrt{T}.
\end{align}

\begin{lemma}
In the randomized version, every round we will get $\hat{g_t} = \frac{d}{\delta}f_t(w_t)u_t$. $\Expect{[g_t]} = \nabla f(v_t)$ and $||g_t|| < G$. For optimum $\eta$:
\[
\Expect{[\sum_{t=1}^T f_t(w_t)]} - \min_{u \in W} \sum_{t=1}^T f_t(u) \leq BG\sqrt{T}.
\]
\end{lemma}

\begin{proof}
\begin{align}
h_t(w) & = f_t(w) + w(\hat{g}_t - \nabla f_t(w_t)), \\
\nabla_w h_t(w)|_{w = w_t} & = \nabla f_t(w_t) + \hat{g}_t - \nabla f_t(w_t), \\
& = \hat{g_t}.
\end{align}

Online gradient descent on the random function $h_t$ is equal to expected gradient descent on the fixed functions $f_t$.
\end{proof}

\end{lecture}
\theend
