\documentclass{article}
\usepackage{cse599s14sp}

\newcommand{\Regret}{\operatorname{Regret}}
\newcommand{\grad}{\triangledown}

\begin{lecture}{3}{Follow-The-Regularized-Leader}{Alexandre Bykov}{Brendan McMahan}{04/08/2014}

\section{Review of Follow-The-Leader}
Last time we analyzed the Follow-The-Leader (FTL) algorithm. As a reminder, below is the rule the player uses at round $t+1$ for picking $w_{t+1}$:
\begin{equation*}
w_{t+1}=\argmin_{w \in W} f_{1:t}(w).
\end{equation*}
We proved that FTL has the following regret bound:
\begin{lemma}[FTL Regret Bound]
Let $w_1,...,w_T$ be the points played by the player during FTL. Then, $\forall u \in W$
\begin{equation*}
\Regret \leq \sum_{t=1}^{T} f_t(w_t)-f_t(w_{t+1}).
\end{equation*}
\end{lemma}
There are two key problems with FTL. First, solving for the $w$ that minimizes $f_{1:t}(w)$ could be a hard optimization problem. The bigger issue is that, in general, the regret bound is $O(T)$. This is especially true for linear functions. The poor regret bound comes from the fact that $f_t(w_t)-f_t(w_{t+1})$ will be large if $w_{t+1}$ is far from $w_t$. This issue can be mitigated by adding regularization to improve the stability of the solution.

\section{Follow-The-Regularized-Leader}
The Follow-The-Regularized-Leader (FTRL) algorithm is the FTL algorithm with a regularizer term added. Let $R:W \tends \R$. At round $t+1$ the player will pick $w_{t+1}$ according to
\begin{equation}
w_{t+1}=\argmin_{w \in W} (f_{1:t}(w)+R(w)).
\end{equation}
Without loss of generality we can say that $R(0)=0$, which minimizes the regularization function. Any regularization function can be translated so that it is minimized at $0$. From this fact it follows that
\begin{equation}
w_1=\argmin_{w \in W} R(w)=0.
\end{equation}
\begin{claim}
By inserting the regularization term we can make the solution more stable.
\end{claim}
This claim will be proven later but first we will consider a simple example.

\subsection{Example: Linear Loss FTRL}
Consider a linear loss function of the form
\begin{equation*}
f_t(w)=g_t \cdot w
\end{equation*}
with $g_t \in \R^n$. Let the regularization term be of the form
\begin{equation*}
R(w)=\frac{1}{2 \eta}\|w\|_2^2
\end{equation*}
where $\eta \geq 0$. For this example we can let $W=\R^n$ since the regularization term will ensure a nice solution. Substituting the above definitions into (1) we get
\begin{equation*}
w_{t+1}=\argmin_{w \in \R^n} (g_{1:t} \cdot w + \frac{1}{2 \eta}\|w\|_2^2).
\end{equation*}
To minimize the inside expression we can take the gradient with respect to $w$ and set it to $0$:
\begin{equation*}
\grad_w(g_{1:t} \cdot w + \frac{1}{2 \eta}\|w\|_2^2)=g_{1:t}+\frac{1}{\eta}w=0 \implies w = -\eta g_{1:t}.
\end{equation*}
Therefore we have that at step $t+1$ the player must pick
\begin{equation*}
w_{t+1}=-\eta g_{1:t}.
\end{equation*}
Instead of recalculating this quantity at each time step, we can reformulate this in terms of $w_t$. From the equation above we know that
\begin{equation*}
w_t=-\eta g_{1:t-1} \implies g_{1:t-1}=\frac{-w_t}{\eta}.
\end{equation*}
We can therefore rewrite $w_{t+1}$ as
\begin{equation*}
w_{t+1}=-\eta g_{1:t}=-\eta (\frac{-w_t}{\eta}+g_t)=w_t-\eta g_t.
\end{equation*}
This is exactly the same as the formula for gradient descent with a constant learning rate. To check for the stability of these solution vectors we can compute
\begin{equation}
f_t(w_t)-f_t(w_{t+1})=g_t \cdot (w_t-w_{t+1})=g_t \cdot (-\eta g_{1:t-1} + \eta g_{1:t})=g_t \cdot (\eta g_t)=\eta \|g_t\|_2^2.
\end{equation}
By correctly picking a value for $\eta$ we can ensure that this value is low.
\begin{observation}
For certain loss functions, FTL will also acheive the above regret bound. For example, loss functions of the form
\begin{equation*}
f_t(w)=\frac{1}{2}\|w-z_t\|_2^2
\end{equation*}
where $z_t$ is picked by the adversary, will also create a stable solution. Our quadratic regularizer produces the same effect as a quadratic loss function.
\end{observation}

\subsection{FTRL Regret Bound}
\begin{lemma}[FTRL Regret Bound]
Let $w_1,...,w_T$ be the points played by the player during FTRL. Then, $\forall u \in W$
\begin{equation*}
\Regret(u) \leq R(u) + \sum_{t=1}^T f_t(w_t)-f_t(w_{t+1}).
\end{equation*}
\end{lemma}
\begin{proof}
Play FTL with the following sequence of loss functions: $f_0=R, f_1, f_2,...,f_T$. By the way we defined $w_t$ in (1), we can guarantee that FTL will play the exact same sequence $w_1, w_2, ..., w_T$ as FTRL in this situation. From Lemma 1 we know that
\begin{equation*}
\sum_{t=0}^T f_t(w_t)-f_t(u) \leq \sum_{t=0}^T f_t(w_t)-f_t(w_{t+1}).
\end{equation*}
Substituing $f_0=R$ we get:
\begin{equation*}
R(w_0)-R(u)+\sum_{t=1}^T f_t(w_t)-f_t(u) \leq R(w_0) - R(w_1) + \sum_{t=1}^T f_t(w_t)-f_t(w_{t+1}).	
\end{equation*}
By (2) we know that $R(w_1)=0$. Substituting that in and simplifying we get:
\begin{equation*}
\begin{split}
\sum_{t=1}^T f_t(w_t)-f_t(u) \leq R(u) + \sum_{t=1}^T f_t(w_t)-f_t(w_{t+1}) \implies \\
\Regret(u) \leq R(u) + \sum_{t=1}^T f_t(w_t)-f_t(w_{t+1}).
\end{split}
\end{equation*}
\end{proof}

\subsection{Linear Loss Function Analysis}
Linear loss functions caused some of the worst behavior for FTL. Here we analyze the performance of FTRL on linear loss functions.
\begin{corollary}
Consider FTRL with regularizer
\begin{equation*}
R(w)=\frac{1}{2\eta}\|w\|_2^2.
\end{equation*}
Then $\forall u \in \R^n$:
\begin{equation*}
\Regret(u) \leq \frac{1}{2\eta}\|u\|_2^2 + \sum_{t=1}^T \eta\|g_t\|_2^2.
\end{equation*}
\end{corollary}
\begin{proof}
As shown in Lemma 4, we know that
\begin{equation*}
\Regret(u) \leq R(u) + \sum_{t=1}^T f_t(w_t)-f_t(w_{t+1}).
\end{equation*}
By (3) we know that
\begin{equation*}
f_t(w_t)-f_t(w_{t+1})=\eta \|g_t\|_2^2
\end{equation*}
for the specified $R$. Substituting that and the equation for $R$ into the regret bound completes the proof.
\end{proof}
An important question is whether this regret is sublinear. Clearly this depends on the learning rate that we choose. If the learning rate is too high then the adversary can pick points such that we constantly oscillate around the optimal solution. If the learning rate is too small the adversary can pick an optimal value far away from the initial value and we will never get there. To avoid these issues we will need to pick an optimal step size.
\begin{assumption}
$\forall t, \|g_t\|_2 \leq G$ and $W=\{u| \|u\|_2 \leq B\}$
\end{assumption}
By the above assumptions and Corollary 5 we get:
\begin{equation*}
\Regret(u) \leq \frac{1}{2\eta}\|u\|_2^2 + \sum_{t=1}^T \eta\|g_t\|_2^2 \leq \frac{B^2}{2\eta}+\eta T G^2.
\end{equation*}
We can solve for the $\eta$ that minimizes this regret bound. Taking the derivate of the above expression in terms of $\eta$ and setting it equal to $0$ we get:
\begin{equation*}
\frac{d}{d\eta}(\frac{B^2}{2\eta}+\eta T G^2)=-\frac{B^2}{2\eta^2}+T G^2=0 \implies \frac{B^2}{2\eta^2}=T G^2 \implies \eta=\frac{B}{G \sqrt{2T}}.
\end{equation*}
Substituting this value for $\eta$ back into the regret bound we get:
\begin{equation*}
\Regret(u) \leq \frac{GB^2 \sqrt{2T}}{2B}+\frac{BTG^2}{G\sqrt{2T}}=\frac{BG \sqrt{T}}{\sqrt{2}}+\frac{BG \sqrt{T}}{\sqrt{2}}=BG \sqrt{2T}.
\end{equation*}
Therefore the regret is sublinear and is in fact $O(\sqrt{T})$. This analysis highlights that picking a correct learning reate is crucial for online gradient descent. Estimating the constants $D$, $G$ and $T$ can be difficult in practice and there has been a lot of work to solve that issue. By picking an $\eta$ that varies with $t$ we can achieve a regret that is almost as good without knowing $T$ in advance. Furthermore, $T$ and $G$ can be eliminated by rephrasing the problem in terms of $g_t$. These topics will be covered in future lectures.

\section{Generalizing Beyond Linear Loss Functions}
We can envision the online learning task as the following diagram: 
\begin{figure}[h]
\setlength{\unitlength}{0.92cm}
\begin{picture}(17.2,3.4)(0,-3.4)
% Adversarial World
\put(1,-0.2){\line(0,-1){3}}
\put(1,-3.2){\line(1,0){4}}
\put(5,-3.2){\line(0,1){3}}
\put(1,-0.2){\line(1,0){4}}
%Transformation
\put(7,-0.2){\line(0,-1){3}}
\put(7,-3.2){\line(1,0){4}}
\put(11,-3.2){\line(0,1){3}}
\put(7,-0.2){\line(1,0){4}}
% Online Algorithm
\put(13,-0.2){\line(0,-1){3}}
\put(13,-3.2){\line(1,0){4}}
\put(17,-3.2){\line(0,1){3}}
\put(13,-0.2){\line(1,0){4}}
%Dashed box
\multiput(6.8, 0)(0.42, 0){25}{\line(1,0){0.21}}
\multiput(17.2, 0)(0, -0.42){8}{\line(0,-1){0.21}}
\multiput(17.2, -3.4)(-0.42, 0){25}{\line(-1,0){0.21}}
\multiput(6.8, -3.4)(0, 0.42){8}{\line(0,1){0.21}}
%Arrows
\put(5, -1.2){\vector(1,0){2}}
\put(7, -2.2){\vector(-1,0){2}}
\put(11, -1.2){\vector(1,0){2}}
\put(13, -2.2){\vector(-1,0){2}}
% Text
\put(1.5, -1.8){Adversarial World}
\put(7.7, -1.8){Transformation}
\put(13.55, -1.8){Online Algorithm}
\put(5.8, -1){$f_t$}
\put(5.6, -2){$w_{t+1}$}
\put(11.8, -1){$\hat{f}_t$}
\put(11.6, -2){$\hat{w}_{t+1}$}
\end{picture}
\figcaption{Block diagram for online learning game}
\end{figure} \\
The arrows illustrate one round of the online learning game. First, the adversary sends a loss function $f_t$. Next it gets transformed in the transformation block and the transformed $\hat{f}_t$ is sent to the online algorithm. The online algorithm responds with a point $\hat{w}_{t+1}$. This point may then have to be transformed again into the $w_{t+1}$ that is sent back to the adversary. The dashed box represents the parts that we have control over. Through our previous analysis we have shown that the FTRL algorithm can be envisioned as a transformation of the FTL algorithm. Below is the diagram for FTRL: 
\begin{figure}[h]
\setlength{\unitlength}{0.92cm}
\begin{picture}(17.2,3.4)(0,-3.4)
% Adversarial World
\put(1,-0.2){\line(0,-1){3}}
\put(1,-3.2){\line(1,0){4}}
\put(5,-3.2){\line(0,1){3}}
\put(1,-0.2){\line(1,0){4}}
%Transformation
\put(7,-0.2){\line(0,-1){3}}
\put(7,-3.2){\line(1,0){4}}
\put(11,-3.2){\line(0,1){3}}
\put(7,-0.2){\line(1,0){4}}
% Online Algorithm
\put(13,-0.2){\line(0,-1){3}}
\put(13,-3.2){\line(1,0){4}}
\put(17,-3.2){\line(0,1){3}}
\put(13,-0.2){\line(1,0){4}}
%Dashed box
\multiput(6.8, 0)(0.42, 0){25}{\line(1,0){0.21}}
\multiput(17.2, 0)(0, -0.42){8}{\line(0,-1){0.21}}
\multiput(17.2, -3.4)(-0.42, 0){25}{\line(-1,0){0.21}}
\multiput(6.8, -3.4)(0, 0.42){8}{\line(0,1){0.21}}
%Arrows
\put(5, -1.2){\vector(1,0){2}}
\put(11, -1.2){\vector(1,0){2}}
\put(13, -2.2){\vector(-1,0){8}}
% Text
\put(1.5, -1.8){Adversarial World}
\put(7.7, -1.05){Transformation}
\put(8.15, -1.65){Add $R(w)$}
\put(13.55, -1.5){Online Algorithm}
\put(14.6, -2.1){FTL}
\put(5.8, -1){$f_t$}
\put(5.6, -2){$w_{t+1}$}
\put(11.8, -1){$\hat{f}_t$}
\end{picture}
\figcaption{Block diagram for FTRL}
\end{figure} \\
The trasnformation reflects adding a regularizer term to the standard FTL algorithm. As can be seen in the diagram, $w_{t+1}$ does not need to be transformed on the way back to the adversary.\\
Similar to the FTL to FTRL transformation, we can think of generalizing the FTRL algorithm by transforming convex loss functions into linear loss functions that we already know how to analyze.\\\\\\\\\\
\begin{figure}[h]
\setlength{\unitlength}{0.92cm}
\begin{picture}(17.2,2.6)(0,-3.4)
% Adversarial World
\put(1,-0.2){\line(0,-1){3}}
\put(1,-3.2){\line(1,0){4}}
\put(5,-3.2){\line(0,1){3}}
\put(1,-0.2){\line(1,0){4}}
%Transformation
\put(7,-0.2){\line(0,-1){3}}
\put(7,-3.2){\line(1,0){4}}
\put(11,-3.2){\line(0,1){3}}
\put(7,-0.2){\line(1,0){4}}
% Online Algorithm
\put(13,-0.2){\line(0,-1){3}}
\put(13,-3.2){\line(1,0){4}}
\put(17,-3.2){\line(0,1){3}}
\put(13,-0.2){\line(1,0){4}}
% Dashed box
\multiput(6.8, 0)(0.42, 0){25}{\line(1,0){0.21}}
\multiput(17.2, 0)(0, -0.42){8}{\line(0,-1){0.21}}
\multiput(17.2, -3.4)(-0.42, 0){25}{\line(-1,0){0.21}}
\multiput(6.8, -3.4)(0, 0.42){8}{\line(0,1){0.21}}
% Arrows
\put(5, -1.2){\vector(1,0){2}}
\put(11, -1.2){\vector(1,0){2}}
\put(13, -2.2){\vector(-1,0){8}}
% Text
\put(1.5, -1.8){Adversarial World}
\put(7.7, -1.05){Transformation}
\put(7.6, -1.65){Convex $\implies$ Linear}
\put(13.55, -1.5){Online Algorithm}
\put(14.5, -2.1){FTRL}
\put(5.8, -1){$f_t$}
\put(5.6, -2){$w_{t+1}$}
\put(11.8, -1){$\hat{f}_t$}
\end{picture}
\figcaption{Block diagram for FTRL with convex loss functions}
\end{figure} 
As in Figure 2, we don't need to transform $w_{t+1}$ on the way back to the adversary. To determine the correct transformation we first need to review convexity.

\section{Convexity}
\begin{definition}
A function $f:\R^n \rightarrow \R$ is convex if $\forall \alpha \in [0,1]$ and $\forall u,w \in \R^n$:
\begin{equation*}
f(\alpha w+(1-\alpha)u) \leq \alpha f(w)+(1-\alpha)f(u).
\end{equation*}
\end{definition}
\begin{figure}[h]
\setlength{\unitlength}{1cm}
\begin{picture}(0,3)(-8.25,0)
% Axes
\put(-3,0){\vector(1,0){6}}
\put(0,0){\vector(0,1){3}}
\put(3.1,-0.1){$x$}
\put(0.1,2.9){$y$}
% Curve
\qbezier(-2.8, 2.5)(0, -2.5)(2.8, 2.5)
\put(2.3, 2.4){$f$}
% Ticks
\put(-2.5, -0.1){\line(0,1){0.2}}
\put(2, -0.1){\line(0,1){0.2}}
\put(-2.6, -0.3){$u$}
\put(1.9, -0.3){$w$}
% Points
\put(-2.5, 2){\circle*{0.1}}
\put(2, 1.26){\circle*{0.1}}
\put(-2.5, 2){\line(6, -1){4.5}}
\put(-2.5, 2.1){$\alpha = 0$}
\put(2.2, 1.2){$\alpha = 1$}
\end{picture}
\figcaption{Illustration of convexity definition}
\end{figure}
The curve represents a convex function $f$ and the line above it is $y=\alpha f(w)+(1-\alpha)f(u)$ for varying values of $\alpha$. The above definition states that a convex function must be below that line for any choices of $u$ and $w$.
A key property of convex functions (that will turn out to be very useful for transforming them into linear functions) is that a tangent line drawn at any point will always be below the function and will approximate it well in a small neighborhood around the tangent point. Differentiability is not necessary for this property to hold. For example, the absolute value function is not differentiable at the origin, however any tangent line through the origin satisfies this property.
\begin{figure}[h]
\setlength{\unitlength}{1cm}
\begin{picture}(0,3)(-8.25,0)
% Axes
\put(-3,0){\vector(1,0){6}}
\put(0,0){\vector(0,1){3}}
\put(3.1,-0.1){$x$}
\put(0.1,2.9){$y$}
% Curve
\qbezier(-2.8, 2.5)(0, -2.5)(2.8, 2.5)
\put(2.3, 2.4){$f$}
% Tangent line
\put(-2.3, 1.5){\line(1,-1){1.4}}
\end{picture}
\figcaption{Illustration of tagent line property}
\end{figure}

\end{lecture}
\theend