\documentclass{beamer}
% \documentclass[handout]{beamer}

\usetheme{dcc}

\usepackage[utf8]{inputenc}
\usepackage[american]{babel}

\newcommand{\ie}{i.\,e.}
\newcommand{\Ie}{I.\,e.}
\newcommand{\eg}{e.\,g.}
\newcommand{\Eg}{E.\,g.}

\title{Mineração de Dados Aplicada}
\subtitle{Supervised Classification}

\author{Loïc Cerf}

\date{June 2nd 2025}

\institute
{
  \scriptsize
  DCC -- ICEx -- UFMG
  \bigskip\\
  \includegraphics[width=.334113288\linewidth]{images/dcc}\hfill\includegraphics[width=.5\linewidth]{images/ufmg}
}

\subject{Supervised Classification}

\keywords{classification, supervised learning, k nearest neighbors,
  decision trees, C4.5, naive Bayes, associative classification, CBA,
  neural networks, support vector machines, ROC curve,
  cross-validation}

\hypersetup{pdfpagemode=FullScreen}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\begin{frame}[plain]
  \titlepage
\end{frame}

\begin{frame}
  \frametitle{Outline for the next sessions}
  \begin{description}
  \item[Today] Supervised classification;
  \item[04/06] Assessing classifiers;
  \item[09--23/06] Project work.
  \end{description}

  \vfill
  \pause

  \emph{All} final reports are due for June 23rd at 23:59 (Brasília
  time). A $5k$-point malus will be applied for a report delivered in
  the subsequent $24k$ hours ($k \in \mathbb{N}$).

  \vfill
  \pause

  Oral defenses on June 30th and July 2nd.
\end{frame}

\begin{frame}
  \frametitle{Outline for this session}
  \tableofcontents
\end{frame}

\section{Classifying}

\begin{frame}
  \frametitle{Learning a classifier}
  \begin{definition}
    Learning from a set of objects how to predict the value of
    a nominal attribute.
  \end{definition}

  \vfill

  \only<1>
  {
    Input:\\
    \begin{columns}
      \begin{column}{.5\linewidth}
        \begin{displaymath}
          \begin{array}{c|cccc}
            & a_1 & \dots & a_n & \alert{\text{class}}\\
            \hline
            o_1 & d_{1, 1} & \dots & d_{1, n} & \alert{c_1}\\
            o_2 & d_{2, 1} & \dots & d_{2, n} & \alert{c_2}\\
            \vdots & \vdots & \ddots & \vdots & \alert{\vdots}\\
            o_m & d_{m, 1} & \dots & d_{m, n} & \alert{c_m}
          \end{array}
        \end{displaymath}
      \end{column}
      \begin{column}{.5\linewidth}
        A parametric \emph{classification model}, that takes the
        values of the attributes $a_1$, \dots, $a_n$ at input and
        returns a value for the \emph{class}.
      \end{column}
    \end{columns}
  }\only<2->
  {
    Output:\\

    \vfill

    Estimates of the parameters of the classification model.  The
    classification model with its parameters substituted by their
    estimates is the \emph{classification function} aka
    \emph{classifier}.
  }
\end{frame}

\begin{frame}
  \frametitle{Interpreting a classifier}
  If the model is \emph{interpretable}, the classifier provides an
  understanding of the relationships between the class and the other
  attributes, \ie, of how the latter explain the given class
  structure.

  \vfill
  \pause

  Interpretable classifiers may be significantly worse than
  non-interpretable ones.

  \vfill
  \pause

  Interpretable classification can help the interpretation of a
  clustering, considered as a given class structure.  However, simpler
  tools should be tried first: conditional box plots, scatter plots
  with different colors or shapes for the different clusters, etc.
\end{frame}

\begin{frame}
  \frametitle{Clustering vs. classification}
  Estimating the parameters of a clustering or a classification model
  is \emph{globally} modeling the data.  Since every object influences
  the output, the optimal estimates are usually unreachable within a
  reasonable time and those problems are \emph{inexactly} solved.

  \vfill
  \pause

  Clustering aims to discover a \emph{hidden} organization of the
  objects: it is \emph{unsupervised}.

  \vfill
  \pause

  In contrast, learning a classifier is a \emph{supervised} task: it
  aims to explain a \emph{given} organization of the objects (the
  class structure) from the other attributes.
\end{frame}

\begin{frame}
  \frametitle{Supervised vs. unsupervised}
  \emph{\only<1>{Uns}\only<2>{S}\only<3>{Semi-s}upervised} approaches
  learn how to classify from:

  \vfill

  \begin{center}
    \only<1>{\includegraphics[width=.66\linewidth]{img/clustering}}\only<2>{\includegraphics[width=.66\linewidth]{img/classified_data}}\only<3>{\includegraphics[width=.66\linewidth]{img/all_data}}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Semi-supervised classification}
  Semi-supervised classification deals with learning how to classify
  from both classified and unclassified objects. It is particularly
  useful when only a small proportion of the objects is classified.
\end{frame}

\begin{frame}
  \frametitle{Making predictions}
  Applying a classifier to an object, with the value of the class
  missing, predicts it.

  \vfill

  \only<1>{Input: a classification function of $a_1, \dots, a_n$
    and}\only<2>{Output:}\\
  \begin{displaymath}
    \begin{array}{c|cccc}
      & a_1 & \dots & a_n & \alert<2>{\text{class}}\\
      \hline
      o_{m + 1} & d_{m + 1, 1} & \dots & d_{m + 1, n} &
      \uncover<2>{\alert{c_{m + 1}}}\\
      o_{m + 2} & d_{m + 2, 1} & \dots & d_{m + 2, n} &
      \uncover<2>{\alert{c_{m + 2}}}\\
      \vdots & \vdots & \ddots & \vdots &
      \uncover<2>{\alert{\vdots}}\\
      o_p & d_{p, 1} & \dots & d_{p, n} & \uncover<2>{\alert{c_p}}
    \end{array}
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Eager classification}
  \emph{Eager} classification proceeds in two steps:
  \begin{enumerate}
  \item Learn a classifier from all the classified objects (using a
    ``Learner'' node in KNIME);
  \item Apply the classifier to every unclassified object (using a
    ``Predictor'' node in KNIME).
  \end{enumerate}

  \vfill
  \pause

  The computational cost of learning a classifier that aims to
  correctly classify any object may be high.  Applying the classifier
  to every unclassified object is computationally cheap, in
  comparison.

  \vfill
  \pause

  If the model is \emph{interpretable}, the classifier provides a
  \emph{global understanding} of the relationships between the class
  and the other attributes, \ie, of how these attributes explain the
  class structure.
\end{frame}

\begin{frame}
  \frametitle{Eager classification: illustration}
  \only<-3>{Explain}\only<4->{Predict}ing the class of objects
  described with two attributes of the interval type.
  \begin{columns}
    \begin{column}{.33\linewidth}
      \only<-3>
      {
        \begin{displaymath}
          \begin{array}{c|cc}
            & x & y\\
            \hline
            \color{yellow} o_1 & 147 & 155\\
            \color{yellow} o_2 & 176 & 145\\
            \color{red} o_3 & 301 & 326\\
            \color{red} o_4 & 372 & 270\\
            \color{yellow} o_5 & 129 & 274\\
            \color{blue} o_6 & 455 & 320\\
            \color{blue} o_7 & 475 & 135\\
            \color{blue} o_8 & 395 & 95\\
            \color{red} o_9 & 343 & 65\\
            \vdots & \vdots & \vdots
          \end{array}
        \end{displaymath}
      }\only<4->
      {
        \begin{displaymath}
          \begin{array}{c|cc}
            & x & y\\
            \hline
            \only<5>{\color{yellow}} o_{10} & 150 & 66\\
            \only<5>{\color{yellow}} o_{11} & 154 & 105\\
            \only<5>{\color{blue}} o_{12} & 466 & 89\\
            \only<5>{\color{red}} o_{13} & 271 & 143\\
            \only<5>{\color{yellow}} o_{14} & 186 & 155\\
            \only<5>{\color{red}} o_{15} & 334 & 329\\
            \only<5>{\color{blue}} o_{16} & 464 & 113\\
            \only<5>{\color{blue}} o_{17} & 417 & 123\\
            \only<5>{\color{blue}} o_{18} & 439 & 67\\
            \vdots & \vdots & \vdots
          \end{array}
        \end{displaymath}
      }
    \end{column}
    \begin{column}{.66\linewidth}
      \only<1>{\includegraphics[width=\linewidth]{img/learning_data}}\only<2>{\includegraphics[width=\linewidth]{img/C45-2}}\only<3>{\includegraphics[width=\linewidth]{img/C45_classifier}}\only<4>{\includegraphics[width=\linewidth]{img/C45_data_to_classify}}\only<5>{\includegraphics[width=\linewidth]{img/C45_classified_data}}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{Lazy classification}
  \emph{Lazy} classification independently considers every single
  object to classify.  For each, a (different) classifier is
  specifically learned.

  \vfill
  \pause

  The computational cost of learning a classifier that aims to
  correctly classify one single object is rather low.  However, that
  cost is multiplied by the number of objects to classify.

  \vfill
  \pause

  If the model is \emph{interpretable}, every classifier provides a
  \emph{justification} for its single prediction.  Lazy classification
  does not globally model the relationships between the class and the
  other attributes.
\end{frame}

\begin{frame}
  \frametitle{Lazy classification: illustration}
  Predicting the class of one single object described with two
  attributes of the interval type.
  \begin{columns}
    \begin{column}{.33\linewidth}
      \begin{displaymath}
        \begin{array}{c|cc}
          & x & y\\
          \hline
          \color{yellow} o_1 & 147 & 155\\
          \color{yellow} o_2 & 176 & 145\\
          \only<1>{\color{red} o_3 & 301 & 326}\\
          \only<1>{\color{red} o_4 & 372 & 270}\\
          \only<1>{\color{yellow} o_5 & 129 & 274}\\
          \only<1>{\color{blue} o_6 & 455 & 320}\\
          \only<1>{\color{blue} o_7 & 475 & 135}\\
          \only<1>{\color{blue} o_8 & 395 & 95}\\
          \only<1>{\color{red} o_9 & 343 & 65}\\
          \vdots & \vdots & \vdots\\
          \only<3>{\color{yellow}} o & 186 & 155
        \end{array}
      \end{displaymath}
    \end{column}
    \begin{column}{.66\linewidth}
      \only<1>{\includegraphics[width=\linewidth]{img/lazy1}}\only<2>{\includegraphics[width=\linewidth]{img/lazy2}}\only<3>{\includegraphics[width=\linewidth]{img/lazy3}}
    \end{column}
  \end{columns}
\end{frame}

\section{Interpretable classification}

\begin{frame}
  \frametitle{$k$-nearest neighbors: illustration}
  $3$-nearest neighbors using the Euclidean distance.

  \vfill

  \begin{center}
    \only<1>{\includegraphics[width=.67\linewidth]{img/all_data}}\only<2>{\includegraphics[width=.67\linewidth]{img/3-NN_classified_data}}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{$k$-nearest neighbors: algorithm}
  For each object to classify:
  \begin{enumerate}[<+->]
  \item List the $k$ most \alert<3-5>{similar} classified objects;
  \item Predict the majority class among those objects (weighted or
    not with their similarities to the object to classify).
  \end{enumerate}

  \vfill

  \uncover<3->
  {
    \begin{itemize}
    \item \uncover<3->{Choice of a similarity measure;}
    \item \uncover<4->{Selection/Extraction of a small number $n$ of
        relevant and non-redundant attributes (curse of
        dimensionality);}
    \item \uncover<5->{Scaling and weighting (w.r.t. the relevance) of
        the attributes;}
    \item \uncover<6->{Space and time complexities are
        $O(|\mathcal{O}|n)$ and $O(k\log{|\mathcal{O}|})$, to classify
        one object;}
    \item \uncover<7>{The proportion of misclassified objects is at
        most $R\left(2 - \frac{CR}{C - 1}\right)$, with $C$ the number
        of classes and $R$ the minimal possible proportion.}
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Naive Bayes: illustration}
  \begin{columns}
    \begin{column}{.5\linewidth}
      \only<1,4>{Estimated class distribution: \color{red} 0.333
        \hfill \color{yellow} 0.333 \hfill \color{blue}
        0.333}\only<2,5,6>{Estimated Gaussian distributions of $x$
        given the
        class:\\}\only<2>{\includegraphics[width=\linewidth]{img/x_gaussian}}\only<5,6>{\includegraphics[width=\linewidth]{img/x_gaussian2}}\only<3,7->{Estimated
        Gaussian distributions of $y$ given the
        class:\\}\only<3>{\includegraphics[width=\linewidth]{img/y_gaussian}}\only<7->{\includegraphics[width=\linewidth]{img/y_gaussian2}}
    \end{column}
    \begin{column}{.5\linewidth}
      \only<-3>{\includegraphics[width=\linewidth]{img/learning_data}}\only<4-8>{\includegraphics[width=\linewidth]{img/one_unclassified_object}}\only<9>{{\includegraphics[width=\linewidth]{img/one_classified_object}}}
    \end{column}
  \end{columns}

  \vfill

  \uncover<4->{Products\uncover<9>{ scaled to become probabilities
      summing to 1}:\\
    \color{red} \only<4,5>{0.333}\only<5>{ $\times$
      1.09e-11}\only<6,7>{3.64e-12}\only<7>{ $\times$
      0.00138}\only<8>{5.02e-15}\only<9>{0} \hfill \color{yellow}
    \only<4,5>{0.333}\only<5>{ $\times$
      0.0206}\only<6,7>{0.00686}\only<7>{ $\times$
      0.000691}\only<8>{4.74e-6}\only<9>{1} \hfill \color{blue}
    \only<4,5>{0.333}\only<5>{ $\times$
      1.21e-18}\only<6,7>{4.04e-19}}\only<7>{ $\times$
    0.00199}\only<8>{8.03e-22}\only<9>{0}
\end{frame}

\begin{frame}
  \frametitle{Naive Bayes}
  Estimate from the classified objects the probability of every class
  and, in every class (\alert<3>{taken independently}), \alert<5>{the
    parameters of a distribution for every attribute} (\alert<3>{taken
    independently}).

  \vfill
  \pause

  Given an object to classify, predict the class providing the
  greatest \alert<3>{product of its probability with the probabilities
    of every value of every attribute given this class}.

  \vfill

  \uncover<3->
  {
    \begin{itemize}
    \item The attributes are assumed (and \emph{should} be)
      independent given the class, the opposite of what decision trees
      assume;
    \item \uncover<4->{No curse of dimensionality thanks to the assumed
        independence;}
    \item \uncover<5>{Choice of distributions (usually Gaussian if the
        attribute is of the interval type, Bernoulli otherwise).}
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Decision tree: illustration}
  \only<-4,6->{
    \begin{center}
      \only<1>{\includegraphics[width=.67\linewidth]{img/learning_data}}\only<2>{\includegraphics[width=.67\linewidth]{img/C45-1}}\only<3,4>{\includegraphics[width=.67\linewidth]{img/C45-2}}\only<6>{\includegraphics[width=.67\linewidth]{img/C45_classifier}}\only<7>{\includegraphics[width=.67\linewidth]{img/C45_data_to_classify}}\only<8>{\includegraphics[width=.67\linewidth]{img/C45_classified_data}}
    \end{center}

    \vfill

    \uncover<4>{Splitting more would lead to too few objects on one
      side of the separation: the recursive partitioning stops.}
  }\only<5>{
    \begin{columns}
      \begin{column}{.5\linewidth}
        \includegraphics[width=\linewidth]{img/C45-2}
      \end{column}
      \begin{column}{.5\linewidth}
        \includegraphics[width=.75\linewidth]{img/decision_tree}
      \end{column}
    \end{columns}}
\end{frame}

\begin{frame}
  \frametitle{Decision tree: algorithm}
  Starting from all classified objects, recursively split them:
  \begin{enumerate}[<+->]
  \item w.r.t. the \alert<5,6>{decision on \emph{one single}
      attribute} (``being in a category'' if nominal, ``being above a
    threshold'' if ordinal) that is greedily the \alert<3>{best at
      separating the classes};
  \item unless \alert<4>{two few classified objects} are on one side
    of the separation.
  \end{enumerate}

  \vfill

  \begin{itemize}
  \item \uncover<3->{Choice of a measure of the quality of the
      separation (\eg, the Gini index or the normalized information
      gain);}
  \item \uncover<4->{Choice of a minimal number of classified objects
      per node and/or the tree can be pruned in a post-processing
      step;}
  \item \uncover<5->{Easy to interpret: \only<-5>{successive decisions
        (the most general decisions are taken first) on single
        attributes until the prediction of the majority class at the
        leaf, the opposite of what Naive Bayes assumes}\only<6->{the
        branches of a decision tree are (\alert<7>{greedily} built)
        classification rules that \alert<7>{must form a tree}}.}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Associative classifiers}
  Given classified objects described with \alert<3>{Boolean
    attributes}, \alert<4>{select} \alert<6->{association rules}
  concluding on $\{$``in class $c$''$\}$.

  \vfill
  \pause

  Find those whose left-hand sides would have the object to classify
  in their supports and predict the (\alert<4>{weighted})
  \alert<5>{majority} right-hand side (or simply that of the
  \alert<4>{best} such rule).

  \vfill

  \uncover<3->
  {
    \begin{itemize}
    \item All relevant attributes must be turned Boolean \emph{a
        priori};
    \item \uncover<4->{Choice of a measure (\eg, the confidence) of
        how discriminant a rule and of thresholds on this measure and
        on the frequency;}
    \item \uncover<5->{To avoid biases, every classified object should
        be in the supports of about the same number of selected
        association rules.}
    \item \uncover<6->{The time to learn \only<6>{an eager}\only<7>{a
          lazy} classifier may exponentially grow with the
        \only<6>{total} number of attributes \only<7>{that are true
          for the object to classify}.}
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Single-layer perceptron: illustration}
  \begin{center}
    \only<1>{\includegraphics[width=.67\linewidth]{img/learning_data}}\only<2>{\includegraphics[width=.67\linewidth]{img/yellow-red}}\only<3>{\includegraphics[width=.67\linewidth]{img/lin-sep_yellow-red}}\only<4>{\includegraphics[width=.67\linewidth]{img/yellow-blue}}\only<5>{\includegraphics[width=.67\linewidth]{img/lin-sep_yellow-blue}}\only<6>{\includegraphics[width=.67\linewidth]{img/red-blue}}\only<7>{\includegraphics[width=.67\linewidth]{img/lin-sep_red-blue}}\only<8>{\includegraphics[width=.67\linewidth]{img/lin-sep}}\only<9>{\includegraphics[width=.67\linewidth]{img/lin-sep_data_to_classify}}\only<10>{\includegraphics[width=.67\linewidth]{img/lin-sep_classified_data}}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Single-layer perceptron}
  The \emph{single-layer perceptron} computes, for each pair of
  classes, \alert<3,4>{a weighted sum of the attributes plus an
    intercept}.  The sign indicates the class.  The most indicated
  class is predicted.

  \vfill
  \pause

  A \alert<4->{gradient descent} tunes the weights and intercepts.

  \vfill
  \pause

  \uncover<3->{
    \begin{itemize}
    \item Hyperplanes, in the attribute space, separate the objects;
    \item \uncover<4->{If the classes are linearly separable, such a
        separation is found;}
    \item \uncover<5>{Otherwise, the single-layer perceptron is only
        locally optimal.}
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Support vector machines (linear kernel)}
  With a linear kernel, the support vector machine is the most stable
  single-layer perceptron, \ie, it maximizes the distances between the
  linear separations and the closest classified objects on both sides.

  \vfill

  \begin{center}
    \only<1>{\includegraphics[width=.67\linewidth]{img/learning_data}}\only<2>{\includegraphics[width=.67\linewidth]{img/yellow-red}}\only<3>{\includegraphics[width=.67\linewidth]{img/SVM_yellow-red}}\only<4>{\includegraphics[width=.67\linewidth]{img/yellow-blue}}\only<5>{\includegraphics[width=.67\linewidth]{img/SVM_yellow-blue}}\only<6>{\includegraphics[width=.67\linewidth]{img/red-blue}}\only<7>{\includegraphics[width=.67\linewidth]{img/SVM_red-blue}}\only<8>{\includegraphics[width=.67\linewidth]{img/SVM}}\only<9>{\includegraphics[width=.67\linewidth]{img/SVM_data_to_classify}}\only<10>{\includegraphics[width=.67\linewidth]{img/SVM_classified_data}}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{One-Versus-One/All}
  Given $C > 2$ classes, learning a classifier can be turned into
  learning several binary classifiers discriminating:
  \begin{description}[<+->]
  \item[One-versus-one] one class from another class, for all
    $\frac{C(C - 1)}{2}$ pairs;
  \item[One-versus-all] one class from all other classes, for all $C$
    classes.
  \end{description}

  \vfill

  \uncover<3->{Given an object to classify, predict:
    \begin{description}
    \item[One-versus-one] the most predicted class;
    \item[One-versus-all] \uncover<4>{the class of the most certain
        classsifier (greatest distance to the separation for SVM).}
    \end{description}
  }
\end{frame}

\section{Less interpretable classification}

\begin{frame}
  \frametitle{Artificial neural networks}
  Single-layer perceptrons are graphs that fully connect an input
  layer (one vertex per attribute) to an output layer (one vertex per
  class or pair of classes), with one weight per edge and one
  intercept per output vertex.

  \vfill
  \pause

  Adding interconnected layers of vertices (aka \emph{hidden
    neurons}), which non-linearly map the sums of their weighted
  inputs, allows arbitrary continuous separations of the classes.

  \vfill
  \pause

  \emph{Deep learning} is about having many layers. It works
  remarkably well on unstructured data such as texts/speeches
  (recurrent neural networks), images (convolutional neural networks),
  etc.
\end{frame}

\begin{frame}
  \frametitle{Support vector machines}
  The support vector machine is the most stable single-layer
  perceptron after employing the kernel trick:
  \begin{enumerate}[<+->]
  \item Non-linearly map the classified objects to a
    higher-dimensional space where the classes hopefully become
    linearly separable;
  \item Compute the linear separations maximizing the distances
    between the separations and the closest classified objects on both
    sides.
  \end{enumerate}

  \vfill

  \uncover<3>{The kernel and its parameter(s) are hard to choose.}
\end{frame}

\begin{frame}
  \frametitle{Ensemble learning}
  Learn and combine many classifiers (wisdom of the crowd).

  \vfill
  \pause

  \begin{description}[<+->]
  \item[Bagging] Randomly sample classified objects and/or attributes,
    learn a classifier from each sample and predict the class that is
    predicted the most often;
  \item[Stacking] Learn many classifiers, append their predictions as
    new attributes and learn a final classifier;
  \item[Boosting] Iteratively learn classifiers that are more severely
    penalized for misclassifying objects that are less correctly
    classified so far.
  \item[etc.]
  \end{description}

  \vfill

  \uncover<6>{KNIME has nodes for random forests and gradient boosted
    trees.}
\end{frame}

\begin{frame}
  \frametitle{Classification vs. regression}
  Both regression and supervised classification aim to \emph{explain}
  one attribute from other attributes.

  \vfill
  \pause

  The explained attribute is of the interval type in the case of
  regression, nominal in the case of classification.

  \vfill
  \pause

  Most algorithms to learn classifiers are easily modified to learn
  regression functions.  \Eg, substitute in $k$-nearest neighbors the
  (weighted) majority vote with the (weighted) mean of the values of
  the dependent attribute, idem at a leaf of a decision tree, etc.

  \vfill
  \pause

  KNIME has nodes for regression using trees or forests.

  \vfill
  \pause

  Techniques to validate classifiers can validate regression
  functions.
\end{frame}

\section{Validation}

\begin{frame}
  \frametitle{Tuning hyperparameters}
  \emph{Hyperparameters} tune the pre-processes and the algorithm that
  estimates the parameters of the classification model.  That model
  and the pre-processes themselves can be seen as hyperparameters.

  \vfill
  \pause

  Hyperparameters are to be tuned to maximize the quality of the
  resulting classifier.  The tested values can be:
  \begin{itemize}
  \item randomly drawn in reasonable ranges;
  \item exhaustively taken on a ``grid'' (Cartesian product of finite
    sets of reasonable values for every hyperparameter);
  \item set by black-box optimization.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Over-fitting or over-generalizing}
  Some hyperparameters tune the minimal number of classified objects
  that justify the predictions:
  \begin{description}[<+->]
  \item[$k$-NN] $k$;
  \item[Decision tree] the minimal number of classified objects per
    node;
  \item[Associative classification] the minimal frequency threshold.
  \end{description}

  \vfill

  \begin{description}
  \item[Over-fitting] \uncover<4->{Fitting too much the classified
      objects is learning the noise; the classifier poorly generalizes
      to new objects;}
  \item[Over-generalizing] \uncover<5>{Not fitting enough the
      classified objects is missing significant details to explain the
      class.}
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Over-fitting: illustration}
  \begin{center}
    \only<1>{\includegraphics[width=.5\linewidth]{img/overfitting1}}\only<2>{\includegraphics[width=.5\linewidth]{img/overfitting2}}\only<3>{\includegraphics[width=.5\linewidth]{img/overfitting3}}
  \end{center}

  \vfill

  \begin{block}{\tiny \copyright 2008 Chabacano (from
      Wikimedia Commons)}
    \tiny \includegraphics[scale=.15]{img/licenses/cc-by-sa}\hskip1pt
    This diagram is licensed under the Creative Commons
    Attribution-Share Alike 4.0 International License.
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Assessing a classifier}
  Because classifying is a supervised task, assessing it is rather
  easy.

  \vfill

  \begin{block}{Basic idea}
    Learn a classifier from a \emph{subset} of the classified objects,
    apply it to the \emph{remaining} classified objects, and compute,
    for these objects, how correlated the given class and the
    predicted one.
  \end{block}

  \vfill
  \pause

  Using the same objects to learn and test leads to over-fitting.
\end{frame}

\begin{frame}
  \frametitle{Conventional validation}
  \emph{Conventional} validation \alert<3>{uses every classified
    object either to learn a classifier or to test it}:
  \begin{enumerate}
  \item \alert<2>{Randomly partition} the set of classified objects
    into two;
  \item Learn a classifier from one of the two subsets;
  \item Apply the learned classifier to every object of the other
    subset;
  \item Compute how correlated the given class and the predicted one.
  \end{enumerate}

  \vfill
  \pause

  For a more trustworthy assessment, each of the two subsets should
  well represent all objects.  \emph{Stratified} sampling helps.  It
  forces a same class distribution in each subset.

  \vfill
  \pause

  However, representativeness may still be an issue, especially if
  there are not many classified objects.
\end{frame}

\begin{frame}
  \frametitle{$k$-fold cross-validation}
  The \emph{$k$-fold cross-validation} \alert<6>{uses every classified
    object $k - 1$ times to learn classifiers and once to test a
    classifier}:
  \begin{enumerate}[<+->]
  \item Randomly partition the classified objects (in a stratified way
    for increased trustworthiness) into \alert<7>{$k$} equally-sized
    subsets;
  \item Learn a classifier from all subsets but one;
  \item Apply the learned classifier to every object of the remaining
    subset;
  \item Repeat steps 2 to 3, setting apart a different subset every
    time;
  \item Compute how correlated the given class and the predicted one.
  \end{enumerate}

  \vfill

  \uncover<6->
  {
    \begin{itemize}
    \item The cross-validation is more trustworthy than the
      conventional one;
    \item \uncover<7>{The larger $k$, the more trustworthy the
        validation.  The more time-consuming too.  $k$ should be at
        least $5$.}
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Leave-one-out cross-validation}
  The \emph{leave-one-out cross-validation} is the $k$-fold
  cross-validation where $k$ is the number of classified objects.

  \vfill
  \pause

  \begin{itemize}
  \item No randomness, hence a more trustworthy validation;
  \item \uncover<3>{High computational cost.}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Cross-validation in KNIME}
  \includegraphics[width=\linewidth]{img/cross-validation}
\end{frame}

\begin{frame}
  \frametitle{Random sampling and bootstrapping}
  The \emph{repeated random sampling} is a \emph{repeated}
  conventional validation. It returns the mean, over the repetitions,
  of the correlation between the given class and the predicted one.

  \vfill
  \pause

  \emph{Bootstrapping} is repeated random sampling \emph{with
    replacement} of the classified objects that are drawn for
  learning, $\approx 63.2\%$ of the classified objects if as many
  objects are sampled.

  \vfill
  \pause

  Bootstrapping is advisable when the classified objects are too few
  to properly represent all objects.
\end{frame}

\begin{frame}
  \frametitle{Learning the final classifier}
  \begin{block}{A baseline}
    If the class is correlated with the other attributes, always
    predicting the majority class must be significantly worse than the
    assessed classification.  A p-value can be computed.
  \end{block}

  \vfill
  \pause

  For validation, classifiers are learned from subsets of the
  classified objects.  The final classifier is learned from \emph{all}
  the classified objects, using the same configuration of the
  hyperparameters.
\end{frame}

\begin{frame}
  \frametitle{Quality of a classifier}
  Any measure of how correlated the given class and the predicted one
  is computed from the \emph{confusion matrix}:

  \vfill

  \begin{center}
    \includegraphics[width=.5\linewidth]{img/confusion_matrix}
  \end{center}

  \vfill

  KNIME's ``Scorer'' node shows the confusion matrix.
\end{frame}

\begin{frame}
  \frametitle{Accuracy}
  The \emph{accuracy} is the proportion of correctly classified
  objects.

  \vfill

  \begin{columns}
    \begin{column}{.5\linewidth}
      \includegraphics[width=\linewidth]{img/confusion_matrix}
    \end{column}
    \begin{column}{.5\linewidth}
      Accuracy:
      \begin{displaymath}
        \frac{18 + 12}{18 + 1 + 0 + 12} \approx 0.96774
      \end{displaymath}
    \end{column}
  \end{columns}

  \vfill
  \pause

  The accuracy mostly depends on correctly predicting the majority
  class(es). It is inadequate if correctly predicting the minority
  class(es) matters, \eg, in alarm applications.
\end{frame}

\begin{frame}
  \frametitle{Matthews correlation coefficient}
  The \emph{Matthews correlation} (aka \emph{phi}) \emph{coefficient}
  is a better global quality measure, for taking into account the
  sizes of the classes.  It is Pearson's $r$ for nominal attributes.
  KNIME does not compute it.

  \vfill

  \begin{center}
    \includegraphics[width=.5\linewidth]{img/confusion_matrix}

    \vfill

    MCC: $\frac{18 \times 12 - 1 \times 0}{\sqrt{(18 + 1)(18 + 0)(12 +
        1)(12 + 0)}} \approx 0.93514$
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Precision and recall}
  Focusing on one particular class $c$:
  \begin{description}[<+->]
  \item[Precision] Among the objects classified in $c$, the proportion
    that truly belongs to $c$;
  \item[Recall] Among the objects in $c$, the proportion that is
    correctly classified in $c$.
  \end{description}

  \vfill

  \begin{columns}
    \begin{column}{.5\linewidth}
      \includegraphics[width=\linewidth]{img/confusion_matrix}
    \end{column}
    \begin{column}{.5\linewidth}
      \only<1>{Precision}\only<2>{Recall} in the ``male'' class:
      \begin{displaymath}
        \only<1>{\frac{18}{18 + 0} = 1}\only<2>{\frac{18}{18 + 1}
          \approx 0.94737}
      \end{displaymath}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{F-scores}
  Focusing on one particular class $c$:
  \begin{description}[<+->]
  \item[F$_1$-score] harmonic mean of the precision and the recall in
    $c$;
  \item[F$_\beta$-score] generalization of the F$_1$-score to attach
    $\beta > 0$ times more importance to the recall than to the
    precision.
  \end{description}

  \vfill

  \begin{columns}
    \begin{column}{.5\linewidth}
      \includegraphics[width=\linewidth]{img/confusion_matrix}
    \end{column}
    \begin{column}{.5\linewidth}
      F$_{\only<1>{1}\only<2>{2}\only<3>{0.5}}$-score in the ``male''
      class:
      \begin{displaymath}
        \only<1>{\frac{2}{\frac{1}{1} + \frac{1}{0.94737}} \approx
          0.97297}\only<2>{0.95745}\only<3>{0.98901}
      \end{displaymath}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{ROC curve}
  Some classifiers estimate the probabilities for an object to be in
  each class.  In that context, the ROC curve depicts the performance
  of the classifier for one particular class $c$.

  \vfill
  \pause

  Considering a varying minimal probability threshold to predict $c$,
  the ROC curve plots the recall in $c$ (aka \emph{true positive
    rate}) in function the proportion of objects not in $c$ that are
  incorrectly classified in $c$ (aka \emph{false positive rate}).  An
  application-dependent minimal probability threshold can be chosen
  from the curve.

  \vfill
  \pause

  The \emph{area under the curve} is a popular quality measure.  It is
  the probability that the classifier gives to a random object in $c$
  a higher probability to be in $c$ than to a random object out of
  $c$.
\end{frame}

\begin{frame}
  \frametitle{ROC curve: illustration}
  \begin{center}
    \includegraphics[width=.5\linewidth]{img/ROC_curves}
  \end{center}

  \vfill

  \begin{block}{\tiny \copyright 2011 cboettig (from
      flickr$^{\text{\textregistered}}$)}
    \tiny \includegraphics[scale=.15]{img/licenses/cc-by}\hskip1pt
    This figure is licensed under the Creative Commons Attribution 2.0
    License.
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Cost-sensitive classification}
  For each pair $(c_i, c_j)$ of classes, the cost of predicting $c_j$
  for an object in class $c_i$ is given.  For instance:

  \begin{center}
    \begin{tabular}{c|cc}
      actual \textbackslash{} predicted & Explosion & No explosion\\
      \hline
      Explosion & 0 & 10000\\
      No explosion & 1 & 0
    \end{tabular}
  \end{center}

  \vfill
  \pause

  \emph{Cost-sensitive classification} aims to minimize the overall
  cost of mis-classification.  Given any classifier estimating the
  probabilities to be in each class, the class minimizing the expected
  cost can be predicted.  For any algorithm learning a classifier, the
  classified objects can be under/over-sampled to match the total
  costs per actual class.  A KNIME extension provides both.
\end{frame}

\section{Coping with imbalanced classes}

\begin{frame}
  \frametitle{Bias towards the majority class(es)}
  Maximizing the accuracy at the detriment of the recall(s) in the
  minority class(es), many algorithms tend to learn classifiers that
  over-predict the majority class(es).
\end{frame}

\begin{frame}
  \frametitle{Over and under sampling}
  To still use algorithms biased towards the majority class(es), to
  correct a bias in the dataset (the distribution among the classes is
  known; the classified objects do not match it) or to take
  mis-classification costs into account, a pre-process can do:
  \begin{description}[<+->]
  \item[Over-sampling] add objects to the under-represented class(es);
  \item[Under-sampling] replace sets of objects in the
    over-represented class(es) with single objects.
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Two techniques to over/under-sample}
  SMOTE over-samples $\ell$ objects in an under-represented class $c$:
  \begin{enumerate}
  \item randomly draw, in a uniform way, an object $o_1$ in $c$, an
    object $o_2$ among $o_1$'s nearest neighbors in $c$, and
    $\alpha \in [0, 1]$;
  \item compute $\alpha o_1 + (1 - \alpha) o_2$;
  \item repeat $\ell$ times the steps 1 to 2;
  \item add all objects computed at step 2 to the class $c$.
  \end{enumerate}

  \vfill
  \pause

  $k$-means under-samples $k$ objects in an over-represented class
  $c$:
  \begin{enumerate}
  \item Cluster with $k$-means the object in $c$;
  \item Replace the objects in $c$ with the $k$ centroids.
  \end{enumerate}
\end{frame}

\section{Case study}

\section*{}

\begin{frame}
  \frametitle{License}
  \begin{block}{\copyright 2012--2025 Loïc Cerf}
    \includegraphics[height=1.5ex]{img/licenses/cc-by-sa}\hskip1pt
    These slides are licensed under the Creative Commons
    Attribution-ShareAlike 4.0 International License.
  \end{block}
\end{frame}

\end{document}