\documentclass{article}
\usepackage{cse599s14sp}
\usepackage{multirow}
\usepackage{epstopdf}

\newcommand{\grad}{\triangledown}


%% the format for the lecture environment is
% \begin{lecture}{lecture number}{lecture title}{scribe}{lecture date}
\begin{lecture}{1}{Introduction to Online Learning}{De Meng}{Ofer Dekel}{04/01/2014}

\section{Course Info}
\begin{itemize}
  \item Instructors: Ofer Dekel, {\tt oferd@microsoft.com}, Brendan McMahan, {\tt mcmahan@google.com}
  \item Course website: {\tt http://courses.cs.washington.edu/courses/cse599s/14sp/index.html}  
\end{itemize}

\section{Introduction}
Online prediction can be considered as a repetitive game between one player (a.k.a. predictor, learning algorithm) and the environment (a.k.a. adversary). Denote $T$ the number of rounds of the game. On each round $t$ ($t = 1, \ldots, T$), the game is played by the player and environment in the following intuitive form
\begin{itemize}
  \item \textbf{Environment}: Choose an \emph{instance} of problem.
  \item \textbf{Player}: Make a \emph{prediction} for this instance.
  \item \textbf{Environment}: Incur a \emph{loss} $\in \R$ for this prediction. (Smaller loss is preferred by the player.). Send the \emph{feedback} on the accuracy of the prediction to the player.
  \item \textbf{Player}: \emph{Learn} and record the feedback.
\end{itemize}

\subsection{Example: Online Binary Prediction Game}
Online binary prediction has many applications, such as email spam classification, which may not fit into stochastic models. In this application, the player observes some features of an email and makes a binary prediction, either spam or not spam. Here are details: for each round $t = 1, \ldots, T$
\begin{itemize}
  \item Player observes a feature vector $x_t \in \R^n$ of an instance generated by the environment.
  \item Player makes a \emph{binary} prediction $\hat{y}_t \in \{+1, -1\}$. $+1, -1$ represent ``spam'' and ``not spam'' respectively.
  \item Player observes feedback $y_t \in \{+1, -1\}$.
  \item A loss is incurred $\ell_t = \1_{\hat{y}_t \neq y_t}$.
\end{itemize}
After $T$ rounds, the cumulative loss is $\sum_{t=1}^T \ell_t$. Intuitively, a ``small'' cumulative loss is desired at the end. This ``small'' will later be defined precisely in the term of regret.

Here we present an equivalent definition of the binary prediction game. For $t= 1, \ldots, T$,
\begin{itemize}
  \item Player chooses a \emph{hypothesis} (a.k.a. binary classifier) $h_t$ from hypothesis class $\mathcal{H}$
      \[h_t: \R^n \rightarrow \{+1, -1\}.\]
      The hypothesis is a function that maps from a feature vector $x \in \R^n$ to a binary prediction set, e.g. $\{+1, -1\}$. At round $t$, the choice of $h_t$ could base on all information recorded before $t$.
  \item Environment chooses $(x_t, y_t)$.
  \item Player incurs a loss $\ell_t = \1_{h(x_t) \neq y_t}$.
\end{itemize}

\subsection{Example: Online Binary Linear Predictor with Hinge Loss}
     Let's first define binary linear predictor and hinge loss function.

     The hypothesis $h_w: \R^n \rightarrow \{+1, -1\}$
     \[h_w(x) = \mathrm{sign}(w \cdot x) = \left\{\begin{array}{ll} +1, & \mbox{if}~w \cdot x > 0 \\ -1,  & \mbox{if}~w \cdot x < 0 \end{array}\right.\]
     is called \emph{binary linear predictor}. The only parameter for this hypothesis is the vector $w \in \R^n$. Geometrically, all vectors that are perpendicular to $w$ (i.e. zero inner product) forms a hyperplane $\{x: w \cdot x = 0\}$, shown in Figure  \ref{fig:hyperplane}. The data may fall into one of halfspaces $\{x: w \cdot x < 0\}$ and $\{x: w \cdot x > 0\}$. $|w\cdot x|$ can be interpreted as
     the prediction \emph{confidence}.
    \begin{figure}
    \begin{center}
    \includegraphics[width = 0.6\textwidth]{hyperplane}
    \end{center}
    \caption{Hyperplane and halfspaces}
    \label{fig:hyperplane}
    \end{figure}

     The hypothesis class $\mathcal{H}$
     \[\mathcal{H} = \{h_w(x) : w \in \R^n, \|w\|_2 \leq 1\},\]
     is the class of binary linear predictors.

     A number of loss functions have been proposed to incur penalty for prediction error. $\1_{\hat{y}_t \neq y_t}$ is simply the \emph{error indicator}, or called \emph{zero-one loss}. The \emph{hinge loss} is defined as
     \[l(w; (x_t, y_t)) = \max\{0, 1-y_tw \cdot x_t\}.\]
     As shown in Figure \ref{fig:hingeloss}, hinge loss function imposes penalty for wrong prediction ($y_tw \cdot x_t <0$) and right prediction with small confidence ($ 0 \leq y_tw \cdot x_t < 1$).
     \begin{figure}
    \begin{center}
    \includegraphics[width = 0.6\textwidth]{hingeloss}
    \end{center}
    \caption{Hinge loss function}
    \label{fig:hingeloss}
    \end{figure}

     The binary linear prediction game with hinge loss can be presented as follows. For $t = 1, \ldots, T$,
     \begin{itemize}
      \item Player chooses $w_t \in \mathcal{W}$, where $\mathcal{W} = \{w\in \R^n:\|w\|_2 \leq 1\}$, a unit ball in $\R^n$.
      \item Environment chooses $(x_t, y_t)$.
      \item Player incurs a loss $\ell_t(w_t; (x_t, y_t)) = \max\{0, 1-y_tw\cdot x_t\}$. ($x_t, y_t$ are regarded as parameters of $\ell_t$.)
      \item Player receives feedback $(x_t, y_t)$.
    \end{itemize}

This game setting can be generalized as \emph{online convex optimization (OCO)} which is discussed in next section.

\section{Online Convex Optimization (OCO)}
For $t= 1, \ldots, T$
    \begin{itemize}
      \item Player chooses $w_t \in \mathcal{W}$, where $\mathcal{W}$ is a \emph{convex} set in $\R^n$.
      \item Environment chooses a \emph{convex} loss function $f_t: \mathcal{W} \rightarrow \R$.
      \item Player incurs a loss $\ell_t = f_t(w_t) = f_t(w_t; (x_t, y_t))$.
      \item Player receives feedback $f_t$.
    \end{itemize}
\textbf{Assumption:}
The environment is oblivious to $w_1, \ldots, w_T$, but can define $f_1, \ldots, f_T$ arbitrarily (perhaps maliciously, even with full knowledge of the player's algorithm, with infinity computation power).
    
Then, at the end, the cumulative loss $\sum_{t=1}^T l_t$ can be arbitrarily large. How to evaluate the prediction performance? What is a good benchmark for the performance? 

A good choice is the cumulative loss of the \emph{best fixed (or say static)} hypothesis in \emph{hindsight},
\[\min_{w \in \mathcal{W}} \sum_{t=1}^T f_t(w). \]
To choose this best fixed hypothesis, we need to know future, that is to collect all $f_1, \ldots, f_T$, then run an \emph{off-line} algorithm.


The difference between the real cumulative loss and this minimum cumulative loss for fixed hypothesis in hindsight is defined as \emph{regret},
\[R(T) = \sum_{t=1}^T f_t(w_t) - \min_{w \in \mathcal{W}} \sum_{t=1}^T f_t(w).\]

\begin{itemize}
  \item If regret grows linearly, $R(T) = \Omega(T)$, the player is not learning.
  \item If regret grows sub-linearly, $R(T) = o(T)$, the player is learning and its prediction accuracy is improving. The regret per round goes to zeros as $T$ goes to infinity,
      \[\frac{1}{T}\left(\min_{w \in \mathcal{W}} \sum_{t=1}^T f_t(w)\right) \rightarrow 0, \quad T \rightarrow \infty.\]
  \item $R(T) = O(\sqrt{T})$.
\end{itemize}

\section{Comparison Between Online Learning and Statistical Learning}

\vspace{0.1in}

\begin{tabular}{|p{0.15\linewidth}|p{0.35\linewidth}|p{0.35\linewidth}|}
\hline
 & \textbf{Online learning (OL)} & \textbf{Statistical learning (SL)} \\
\hline
\multirow{3}{*}{\textbf{Similarities}} & \multicolumn{2}{p{0.7\linewidth}|}{Both define hypothesis space/class of predictors (in each round of a game in OL while in training procedure of SL). } \\ \cline{2-3}
& \multicolumn{2}{p{0.7\linewidth}|}{Both define a loss function to evaluate the prediction performance, and small loss is preferred.} \\ \cline{2-3}
& \multicolumn{2}{p{0.7\linewidth}|}{Instances and labels} \\ \hline

\multirow{2}{*}{\textbf{Differences}} & learning in each round of game, no distinction between training and testing & first train a model, then test it\\ \cline{2-3}
& adversary case & statistical assumption \\ \hline
\end{tabular}



\end{lecture}
\theend
