\documentclass{article}
\usepackage{cse599s12sp}
\usepackage{amssymb} %various math symbols including \pitchfork
\usepackage{amsmath} %\DeclareMathOperator, \align
\usepackage{amsfonts}
\usepackage[parfill]{parskip}
\usepackage{tikz} %flow charts
\usetikzlibrary{shapes,arrows}

\newcommand{\suchthat}[0]{\stackrel{\rotatebox{270}{$\pitchfork$}}{}}
\newcommand{\comment}[1]{\qquad {\rm #1}} %for proofs

\newcommand{\W}{\mathcal{W}}
\newcommand{\Regret}{\text{Regret}}
% 
% %\DeclareMathOperator*{\argmin}{argmin}
% 
% styles for flow charts
\tikzstyle{decision} = [diamond, draw, text width=4.5em, text badly centered, node distance=3cm, inner sep=0pt]
\tikzstyle{block} = [rectangle, draw, text width=5em, text centered, rounded corners, minimum height=4em]

%EVH 20120404: I can't make the cs599 lecture environment work with tikz, so not using begin{lecture} etc
%
%% the format for the lecture environment is
% \begin{lecture}{lecture number}{lecture title}{scribe}{lecture date}
%\begin{lecture}{3}{The Online Optimization Game}{Evan Herbst}{Brendan McMahan}{04/03/2012}

\begin{document}
\lecturetitle{3}{The Online Optimization Game}{Evan Herbst}{Brendan McMahan}{04/03/2012}

\section{Online Optimization Game}

Again, define the online optimization game:

\begin{itemize}
  \item for $t = 1 .. T$ (eventually we'll have bounds that hold $\forall T$, so that we can let $T = \infty$)
  \begin{itemize}
    \item player chooses a predictor $w_t \in \W \subseteq \mathbb{R}^n$
    \item adversary reveals loss function $f_t: \W \rightarrow \mathbb{R}$
    \item player pays $f_t(w_t)$
  \end{itemize}
\end{itemize}

For real applications, think $T$ maybe O($10^9$), $n$ maybe O($10^7$).

The worst-case sum of losses is arbitrarily high, so instead of
minimizing the sum of losses we'll minimize regret:

\begin{displaymath}
\Regret = \sum_{t=1}^T f_t(w_t) - \min_{w \in \W} \sum_{t=1}^T f_t(w).
\end{displaymath}

We also can't minimize regret wrt strategies that can change $w$ at
each iteration, because we'd still do arbitrarily badly. For now we'll
write $w$ as though it's constant over time, but later we will develop
algorithms that have low regret against the set of strategies that
switch $w$ a known finite number of times (switching regret) and to
the set of strategies in which $|w_t - w_{t - 1}|$ is bounded
(drifting regret).

\section{Example Online Optimizers}

An example algorithm: \textbf{Follow-The-Leader (FTL)}:

\begin{displaymath}
w_{t+1} = \argmin_{w \in W} \sum_{s=1}^t f_s(w) \equiv \argmin_w f_{1:t}(w)
\end{displaymath}

Against linear functions, FTL can have the worst possible regret,
O($T$).

Another example algorithm (not realizable because it sees the future):
\textbf{Be-The-Leader (BTL)}: as FTL, but play $w_{t+1}$ on round $t$
instead of playing $w_t$ on round $t$.

Suppose $W \subset \mathbb{R}$ and the loss is required to be linear ($f_t(w) = g_t w$). Table for the same example we did for FTL last week:

\begin{tabular}{ccccc}
{\bf t} & {\bf $w_t$} & {\bf $g_t$} & {\bf loss($t$)} & {\bf $g_{1:t}$} \\ \hline
1 & -1 & .5 & -.5 & .5 \\
2 & 1 & -1 & -1 & -.5 \\
3 & -1 & 1 & -1 & .5 \\
\end{tabular}

Etc. Regret(BTL) will be about $-T$, not about $T$ as it was for FTL. This is because now each $g_t$ entry gets filled in {\it before} the $w_t$ entry on the same line. \newline

\begin{theorem}[BTL Theorem]
  For arbitrary bounded $f_t$, $\Regret(BTL) \le 0$. Equivalently,
  $\sum_{t=1}^T f_t(w_{t+1}) \le \sum_{t=1}^T f_t(w_{T+1})$, which
  will be our IH.
\end{theorem}

\begin{proof}
By induction.

Base case: $T = 1$. $f_1(w_2) \le f_1(w_2)$.

For the induction step, suppose the IH holds for $T$.  Then,

\begin{align*}
\sum_{t=1}^{T+1} f_t(w_{t+1}) &= \sum_{t=1}^T f_t(w_{t+1}) + f_{T+1}(w_{T+2}) \\
                              &\le \sum_{t=1}^T f_t(w_{T+1}) + f_{T+1}(w_{T+2}) \comment{IH} \\
                              &\le \sum_{t=1}^T f_t(w_{T+2}) + f_{T+1}(w_{T+2}) \comment{def.\ \ensuremath{w_{T+1}}} \\
                              &= \sum_{t=1}^T f_t(w_{T+2}).
\end{align*}
\end{proof}

Since the difference between FTL and BTL is whether we play $w_t$ or $w_{t+1}$, let's try to bound the regret of FTL using that of BTL. \newline

\begin{theorem}[FTL Theorem]
For $\forall w^* \in W$, 
\[ \Regret(\text{FTL vs $w^*$}) \le \sum_{t=1}^T (f_t(w_t) - f_t(w_{t+1})),\]
again assuming the $f_t$ are bounded.
\end{theorem}

\begin{proof}

\begin{align*}
\text{Regret(FTL)} &= \sum_{t=1}^T f_t(w_t) - \sum_{t=1}^T f_t(w^*) \\
                   &\le \sum_{t=1}^T f_t(w_t) - \sum_{t=1}^T f_t(w_{T+1}) 
                   &&\text{def.\ \ensuremath{w_{T+1}}} \\
                   &= \underbrace{\sum_{t=1}^T f_t(w_{t+1}) - \sum_{t=1}^T f_t(w_{T+1})}_
                   {\textrm{=\ regret(BTL)\ } \ensuremath{\le 0}} 
                       + \sum_{t=1}^T (f_t(w_t) - f_t(w_{t+1})) \\
                   &\le \sum_{t=1}^T (f_t(w_t) - f_t(w_{t+1})).
\end{align*}
\end{proof}

\section{Transformations in Online Optimization}

\begin{figure}[h!]
 \begin{center}
  \begin{tikzpicture}[node distance=3cm, auto, >=stealth]
   % nodes
   \node[block] (a)               {adversary};
   \node[block] (b)  [right of=a] {transform-ation};
   \node[block] (d)  [right of=b] {inner online alg (eg FTL)};
   
   % edges
   \draw[->] (a.north east) -- node[above]{$f_t$} (b.north west);
   \draw[->] (b.north east) -- node[above]{$f_t'$} (d.north west);
   \draw[->] (d.south west) -- node[below]{$w_{t+1}' \in W'$} (b.south east);
   \draw[->] (b.south west) -- node[below]{$w_{t+1} \in W$} (a.south east);
  \end{tikzpicture}
  \label{flowchart}
 \end{center}
\caption{transformations in online optimization.}
\label{fig:transformations}
\end{figure}

Sometimes we can transform the adversary's loss and parameter space
into something that it's easier to prove things about. See
fig.~\ref{fig:transformations} for an overview. Information flows from
upper left to lower left by the arrows.

An algorithm using a transformation:
\textbf{Follow-The-Regularized-Leader (FTRL)}, which runs FTL on a
regularized loss function.  The udpate is defined by 
\[
w_{t+1} = \argmin_w (f_{1:t}(w) + r(w))
\]
where the regularization function $r: \R^n \rightarrow \R$ satisfies
$r(w) \ge 0$, and typically also $r(0) = 0$.

We view this as running FTL together with the following
transformation: Given functions $f_t(w)$ chosen by the adversary, we
let $f_1'(w) = g_t \cdot w + r(w)$ (that is, we add a regularization
component to the first function we see), and take $f_t' = f_t$ for $t
> 1$.  
\vspace{0.1in}

Again, we can prove a strong result that holds for arbitrary
(potentially even non-convex $f_t$):
\begin{theorem}[FTRL theorem]
The FTRL algorithm has
\[\Regret(FTRL) \le \sum_{t=1}^T (f_t(w_t) - f_t(w_{t+1})) + r(w^*).\]
\end{theorem}

\begin{proof}
\begin{align*}
\sum_t f_t'(w_t) - \sum_t f_t'(w^*) &\le \sum_t f_t'(w_t) - f_t'(w_{t+1}) 
\quad \text{Applying the FTL theorem to $f'$} \\
\iff \underbrace{\sum_t f_t(w_t) - \sum_t f_t(w^*)}_{regret(FTRL)} + r(w_1) - r(w^*) &\le \sum_t (f_t(w_t) - f_t(w_{t+1})) + r(w_1) - r(w_2) \\
\iff \text{Regret(FTRL)} &\le \sum_t(f_t(w_t) - f_t(w_{t+1})) + r(w^*) - \underbrace{r(w_2)}_{\le 0}
\end{align*}
\end{proof}

FTRL for linear $f_t$:
\begin{itemize}
 \item $f_t(w) = g_t \cdot w$
 \item $|g_t| \le G$ (bounded loss)
 \item $w_{t+1} = \argmin_w \left( \sum_t g_t \cdot w + \frac{\sigma}{2} |w|^2 \right), \sigma \in \R^+$ (quadratic regularizer)
\end{itemize}

Then,
\begin{align*}
  \text{Regret(FTRL)} &\le \sum_{t=1}^T (f_t(w_t) - f_t(w_{t+1})) + r(w^*) 
                           && \text{FTRL theorem} \\
                      &= \sum_t g_t \cdot (w_t - w_{t+1}) + r(w^*) \\
                      &\le \sum_t G |w_t - w_{t+1}| + \frac{\sigma}{2} |w^*|^2 
                           && \text{Cauchy-Schwarz} \\
                      &\le \frac{T}{\sigma} G^2 + \frac{\sigma}{2} |w^*|^2. \\
\end{align*}

This is just gradient descent with a fixed learning rate
$\frac{1}{\sigma}$, because
\begin{displaymath}
w_t - w_{t+1} = \frac{-g_{1:t-1} + g_{1:t}}{\sigma} = \frac{g_t}{\sigma}.
\end{displaymath}

To choose $\sigma$ optimally you need to know $G$ and $T$. But in
practice it doesn't matter that you don't know $T$.  In general, you
can always apply the ``doubling trick'' (not explained in class).  Or
even better, you can analyze a version of the algorithm that changes
the amount of regularization adaptively, which we may analyze later.
\newline

\begin{theorem}
  $\forall w^*$ with $|w^*| \le R$, against linear loss functions
  $f_t(w) = g_t \cdot w$ with $|g_t| \le G$, FTRL with $r(w) =
  \frac{\sigma}{2} |w|^2$, where $\sigma = \frac{G \sqrt{2T}}{R}$, has
  regret $\le GR \sqrt{2T}$.
\end{theorem}

\begin{proof}
Plug this value for $\sigma$ into the regret bound above.
\end{proof}

%\end{lecture}
\end{document}
%\end{lecture}
%\theend
