\documentclass{beamer}
% \documentclass[handout]{beamer}

\usetheme{dcc}

\usepackage[utf8]{inputenc}
\usepackage[american]{babel}
\usepackage{gensymb}

\newcommand{\ie}{i.\,e.}
\newcommand{\Ie}{I.\,e.}
\newcommand{\eg}{e.\,g.}
\newcommand{\Eg}{E.\,g.}

\title{Mineração de Dados Aplicada}
\subtitle{Similarities}

\author{Loïc Cerf}

\date{May 5th 2025}

\institute
{
  \scriptsize
  DCC -- ICEx -- UFMG
  \bigskip\\
  \includegraphics[width=.334113288\linewidth]{images/dcc}\hfill\includegraphics[width=.5\linewidth]{images/ufmg}
}

\subject{Similarities}

\keywords{Similarity, distance, scaling, curse of dimensionality, PCA}

\hypersetup{pdfpagemode=FullScreen}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\begin{frame}[plain]
  \titlepage
\end{frame}

\begin{frame}
  \frametitle{Outline for the next sessions}
  \begin{description}
  \item[today] Similarities and PCA;
  \item[07/05] Clustering: algorithms;
  \item[12/05] Clustering: assessment and case study;
  \item[14/05] On Technical Reports and Talks + Project work.
  \end{description}

  \vfill

  Partial report for May 19th (4 pages).
\end{frame}

\begin{frame}
  \frametitle{Outline for this session}
  \tableofcontents
\end{frame}

% \section{Feedback on the exam}

% \begin{frame}
%   \frametitle{Statistics on the exam}
%   \begin{center}
%     \begin{tabular}{r|cc|c}
%       & Turma TG2 & Turma TL & All\\
%       \hline
%       arithmetic mean & 13.8 & 8.4 & 10.1\\
%       median & 13 & 10 & 12\\
%       $\min$ & 10 & 1 & 1\\
%       $\max$ & 17 & 15 & 17
%     \end{tabular}
%   \end{center}
% \end{frame}

% \begin{frame}
%   \frametitle{Observations}
%   Although specific, every command can do a lot.  \Eg, \texttt{tr} can
%   substitute more than one character.  To write one reference per line
%   (the first line, empty, still has to be removed):\\
%   \texttt{tr -s '@\textbackslash n[:blank:]' '\textbackslash n ' <
%     loic.bib}

%   \vfill
%   \pause

%   Regexps, such as \texttt{'author *= *\{[\^{}\}]*\{'}, are very
%   useful.  Not only in the shell.

%   \vfill
%   \pause

%   The \emph{specified} format must be guide the solution.  Ad hoc
%   solutions are not reusable and it is hard to know if they actually
%   work when the input is large.
% \end{frame}

\section{How similar two values}

\begin{frame}
  \frametitle{Example of use: clustering}
  \begin{definition}
    Partitioning the objects into clusters so that each cluster
    contains \emph{similar} objects and objects in different clusters
    are \emph{dissimilar}.
  \end{definition}
  \vfill

  \only<1>
  {
    Input:\\
    \begin{displaymath}
      \begin{array}{c|cccc}
        & a_1 & a_2 & \dots & a_n\\
        \hline
        o_1 & d_{1, 1} & d_{1, 2} & \dots & d_{1, n}\\
        o_2 & d_{2, 1} & d_{2, 2} & \dots & d_{2, n}\\
        \vdots & \vdots & \vdots & \ddots & \vdots\\
        o_m & d_{m, 1} & d_{m, 2} & \dots & d_{m, n}
      \end{array}
    \end{displaymath}
  }\only<2>
  {
    Output:\\
    \begin{displaymath}
      \begin{array}{c|ccccc}
        & a_1 & a_2 & \dots & a_n & \alert{\text{cluster}}\\
        \hline
        o_1 & d_{1, 1} & d_{1, 2} & \dots & d_{1, n} & \alert{c_1}\\
        o_2 & d_{2, 1} & d_{2, 2} & \dots & d_{2, n} & \alert{c_2}\\
        \vdots & \vdots & \vdots & \ddots & \vdots & \vdots\\
        o_m & d_{m, 1} & d_{m, 2} & \dots & d_{m, n} & \alert{c_m}
      \end{array}
    \end{displaymath}
  }
\end{frame}

\begin{frame}
  \frametitle{Similarity}
  The similarity between pairs of objects is defined w.r.t. a
  \emph{small} subset of the attributes.

  \vfill
  \pause

  \begin{exampleblock}{Is $o_1$ more similar to $o_2$ or to $o_m$
      w.r.t. $a_1$\only<-3>{ and $a_n$}?}
    \begin{displaymath}
      \begin{array}{c|cccc}
        & \alert{a_1} & a_2 & \dots & \alert<-3>{a_n}\\
        \hline
        \alert{o_1} & \alert{d_{1, 1}} & d_{1, 2} & \dots &
        \alert<-3>{d_{1, n}}\\
        \alert<2,4>{o_2} & \alert<2,4>{d_{2, 1}} & d_{2, 2} & \dots &
        \alert<2>{d_{2, n}}\\
        \vdots & \vdots & \vdots & \ddots & \vdots\\
        \alert<3->{o_m} & \alert<3->{d_{m, 1}} & d_{m, 2} & \dots &
        \alert<3>{d_{m, n}}
      \end{array}
    \end{displaymath}
    Choose a similarity $s$ over the objects projected onto two
    attributes and compare $s(o_1, o_2)$ \uncover<3->{to $s(o_1,
      o_m)$}.
  \end{exampleblock}
\end{frame}

\begin{frame}
  \frametitle{Difference or ratio?}
  \begin{block}{Quizz}
    What is the average surface temperature of the sun?
    \begin{itemize}
    \item Alice answers 5\degree C;
    \item Bob answers 15,000\degree C.
    \end{itemize}
  \end{block}

  \vfill

  The correct answer is 5,500\degree C.  Who gave the closest answer?

  \pause

  \begin{description}
  \item[Using a difference] Alice;
  \item[Using a ratio] Bob.
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Which square contains more dots?}
  \begin{center}
    \uncover<1,3,4,6>{\begin{tabular}{cc}
      \only<-3>{\includegraphics[width=.33\linewidth]{img/10}
      & \includegraphics[width=.33\linewidth]{img/20}}\only<4->{\includegraphics[width=.33\linewidth]{img/110}
      & \includegraphics[width=.33\linewidth]{img/120}}\\
      \only<-3>{\uncover<3>{10 & 20}}\only<4->{\uncover<6>{110 & 120}}
    \end{tabular}}
  \end{center}

  \vfill

  \begin{block}{\tiny \copyright 2017 MrPomidor (in
      Wikimedia)}
    \tiny \includegraphics[scale=.15]{img/licenses/cc-by-sa}\hskip1pt
    This illustration is licensed under the Creative Commons
    Attribution-ShareAlike 4.0 International License.
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Psychology experiments}
  \begin{block}{Stanislas Dehaene et al. (2008)}
    Placing integer on a segment whose extremities are labeled 1 and
    10, adults with a mathematical education let an approximately
    constant gap between every two consecutive integers.  Young
    children and Amazonian indigenes without a mathematical education
    tend to let smaller gaps between larger integers.
  \end{block}

  \vfill
  \pause

  \begin{block}{Weber-Fechner law (1860)}
    \begin{quotation}
      Simple differential sensitivity is inversely proportional to the
      size of the components of the difference; relative differential
      sensitivity remains the same regardless of size.
    \end{quotation}
  \end{block}

  % \vfill

  % \pub Stanislas Dehaene et al. Log or Linear? Distinct Intuitions of
  % the Number Scale in Western and Amazonian Indigene Cultures.
  % \emph{Science}, 320:1217--1220, 2008.
\end{frame}

\begin{frame}
  \frametitle{Compressing/dilating differences}
  The difference between values of one attribute may not intuitively
  measure their similarities.
  % \eg, when the values follow a power-law distribution, which can
  % result from preferential attachment.


  \vfill
  \pause

  In those cases, differences between smaller/larger values ought to
  be compressed/dilated.

  \vfill
  \pause

  When comparing differences between values of one attribute, adding
  or multiplying the values by a constant is useless.
\end{frame}

\begin{frame}
  \frametitle{Bringing larger values closer}
  Logarithmic functions compress (respectively, dilate) differences
  between larger (respectively, smaller) values of an attribute $x$:
  \begin{displaymath}
    x \mapsto ln(x + k) \enspace .
  \end{displaymath}

  \vfill
  \pause

  Multiplying $x$ by a parameter is useless.
\end{frame}

\begin{frame}
  \frametitle{Logarithm}
  \begin{center}
    \includegraphics[width=.67\linewidth]{img/ln}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Bringing smaller values closer}
  Exponential functions compress (respectively, dilate) differences
  between smaller (respectively, larger) values of an attribute $x$:
  \begin{displaymath}
    x \mapsto e^{kx} \enspace .
  \end{displaymath}

  \vfill
  \pause

  Adding a paramater to $x$ is useless.
\end{frame}

\begin{frame}
  \frametitle{Exponential function}
  \begin{center}
    \includegraphics[width=.67\linewidth]{img/exp}
  \end{center}
\end{frame}

\section{How similar two objects}

\begin{frame}
  \frametitle{Similarity}
  The similarity between pairs of objects is defined w.r.t. a
  \emph{small} subset of the attributes.

  \vfill

  \begin{exampleblock}{Is $o_1$ more similar to $o_2$ or to $o_m$
      w.r.t. $a_1$ and $a_n$?}
    \begin{displaymath}
      \begin{array}{c|cccc}
        & \alert{a_1} & a_2 & \dots & \alert{a_n}\\
        \hline
        \alert{o_1} & \alert{d_{1, 1}} & d_{1, 2} & \dots &
        \alert{d_{1, n}}\\
        \alert<1>{o_2} & \alert<1>{d_{2, 1}} & d_{2, 2} & \dots &
        \alert<1>{d_{2, n}}\\
        \vdots & \vdots & \vdots & \ddots & \vdots\\
        \alert<2>{o_m} & \alert<2>{d_{m, 1}} & d_{m, 2} & \dots &
        \alert<2>{d_{m, n}}
      \end{array}
    \end{displaymath}
    Choose a similarity $s$ over the objects projected onto the two
    attributes and compare $s(o_1, o_2)$ \uncover<2>{to
      $s(o_1, o_m)$}.
  \end{exampleblock}
\end{frame}

\begin{frame}
  \frametitle{Similarity between objects}
  The explicit definition of a similarity between the objects in
  $\mathcal{O}$ is a square matrix with \emph{real} elements:
  \begin{displaymath}
    \begin{array}{c|cccc}
      & o_1 & o_2 & \dots & o_m\\
      \hline
      o_1 & s(o_1, o_1) & s(o_1, o_2) & \dots & s(o_1, o_m)\\
      o_2 & s(o_2, o_1) & s(o_2, o_2) & \dots & s(o_2, o_m)\\
      \vdots & \vdots & \vdots & \ddots & \vdots\\
      o_m & s(o_m, o_1) & s(o_m, o_2) & \dots & s(o_m, o_m)
    \end{array}
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Distance}
  The explicit definition of a distance between the objects in
  $\mathcal{O}$ is a square matrix with \emph{real} elements:
  \begin{displaymath}
    \begin{array}{c|cccc}
      & o_1 & o_2 & \dots & o_m\\
      \hline
      o_1 & \only<-2>{D(o_1, o_1)}\only<3->{\alert<3>{0}} & D(o_1,
      o_2) & \dots & D(o_1, o_m)\\
      o_2 & D(\only<-3>{o_2, o_1}\only<4->{\alert<4>{o_1, o_2}}) &
      \only<-2>{D(o_2, o_2)}\only<3->{\alert<3>{0}} & \dots & D(o_2,
      o_m)\\
      \vdots & \alert<4>{\vdots} & \alert<4>{\vdots} &
      \alert<3>{\ddots} & \vdots\\
      o_m & D(\only<-3>{o_m, o_1}\only<4->{\alert<4>{o_1, o_m}}) &
      D(\only<-3>{o_m, o_2}\only<4->{\alert<4>{o_2, o_m}}) &
      \alert<4>{\dots} & \only<-2>{D(o_m, o_m)}\only<3->{\alert<3>{0}}
    \end{array}
  \end{displaymath}

  \vfill

  such that:
  \begin{displaymath}
    \begin{cases}
      \uncover<2->{\forall (o_i, o_j) \in \mathcal{O}^2$, $D(o_i, o_j)
        \geq 0}\\
      \uncover<3->{\forall (o_i, o_j) \in \mathcal{O}^2$, $D(o_i, o_j)
        = 0 \Leftrightarrow o_i = o_j}\\
      \uncover<4->{\forall (o_i, o_j) \in \mathcal{O}^2$, $D(o_i, o_j)
        = D(o_j, o_i)}\\
      \uncover<5>{\forall (o_i, o_j, o_k) \in \mathcal{O}^3$, $D(o_i,
        o_j) + D(o_j, o_k) \geq D(o_i, o_k)}
    \end{cases}.
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{A distance?}
  \begin{alertblock}{Question}
    What is the shortest path between two points on earth?
  \end{alertblock}

  \vfill
  \pause

  \begin{exampleblock}{Answer}
    It is that of a ``segment'' of a great circle, on a spherical
    earth.
  \end{exampleblock}

  \vfill
  \pause

  \begin{block}{A distance}
    It is a distance.
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Another distance?}
  \begin{alertblock}{Question}
    What is the distance to travel from the earth to the moon?
  \end{alertblock}

  \vfill
  \pause

  \begin{exampleblock}{Answer}
    Considering an instantaneous rectilinear travel, it is the
    distance between their centers minus their radius. If that
    assumption cannot be made (spaceship), ask a rocket scientist.
  \end{exampleblock}

  \vfill
  \pause

  \begin{block}{Not a distance}
    The function above is not a distance: the distance between an
    astronomical object and itself is negative.
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Combining absolute differences}
  The most usual distances over attributes of the interval type are
  functions of the absolute differences for each attribute:
  \begin{displaymath}
    D(o_i, o_j) = f(|d_{j, 1} - d_{i, 1}|, |d_{j, 2} - d_{i, 2}|,
    \dots, |d_{j, n} - d_{i, n}|) \enspace .
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Minkowski distance of order $p$}
  Let $o_i$ and $o_j$ two objects:
  \begin{displaymath}
    \begin{array}{c|cccc}
      & a_1 & a_2 & \dots & a_n\\
      \hline
      o_i & d_{i, 1} & d_{i, 2} & \dots & d_{i, n}\\
      o_j & d_{j, 1} & d_{j, 2} & \dots & d_{j, n}
    \end{array}
  \end{displaymath}

  \vfill

  \begin{definition}
    The Minkowski distance of order $p$ between $o_i$ and $o_j$,
    described with attributes of the interval type, is:
    \begin{displaymath}
      \Big(\sum_{k = 1}^n |d_{i, k} - d_{j, k}|^p\Big)^{\frac{1}{p}}
      \enspace .
    \end{displaymath}
  \end{definition}
\end{frame}

\begin{frame}
  \frametitle{Euclidean distance: definition}
  \begin{definition}
    The Euclidean distance is the Minkowski distance of order $2$.
  \end{definition}

  \vfill

  \begin{columns}
    \begin{column}{.33\linewidth}
      \begin{displaymath}
        \begin{array}{c|cc}
          & x & y\\
          \hline
          o_1 & 91 & 70\\
          o_3 & 359 & 243
        \end{array}
      \end{displaymath}
    \end{column}
    \begin{column}{.66\linewidth}
      \only<1>{\includegraphics[width=\linewidth]{img/two_points}}\only<2>{\includegraphics[width=\linewidth]{img/euclidian_distance}}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{Euclidean distance: use}
  The ``default'' (most natural) distance between objects described
  with attributes of the interval type.

  \vfill
  \pause

  Squared Euclidean distances are ordered like Euclidean
  distances. Because they are cheaper to compute, they are preferred
  when the distances must only be compared.
\end{frame}

\begin{frame}
  \frametitle{Manhattan distance}
  \begin{definition}
    The Manhattan distance is the Minkowski distance of order $1$.
  \end{definition}

  \vfill

  \begin{columns}
    \begin{column}{.33\linewidth}
      \begin{displaymath}
        \begin{array}{c|cc}
          & x & y\\
          \hline
          o_1 & 91 & 70\\
          o_3 & 359 & 243
        \end{array}
      \end{displaymath}
    \end{column}
    \begin{column}{.66\linewidth}
      \only<1>{\includegraphics[width=\linewidth]{img/two_points}}\only<2>{\includegraphics[width=\linewidth]{img/manhattan_distance}}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{Manhattan distance: use}
  The Manhattan distance makes more sense than the Euclidean distance
  in some applications.  \Eg, it is the length of a taxicab ride in
  Manhattan.

  \vfill

  \begin{center}
    \includegraphics[width=.33\linewidth]{img/taxicab}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Chebyshev distance}
  \begin{definition}
    The Chebyshev distance is the Minkowski distance when its order
    goes to infinity.
  \end{definition}

  \vfill

  \begin{columns}
    \begin{column}{.33\linewidth}
      \begin{displaymath}
        \begin{array}{c|cc}
          & x & y\\
          \hline
          o_1 & 91 & 70\\
          o_3 & 359 & 243
        \end{array}
      \end{displaymath}
    \end{column}
    \begin{column}{.66\linewidth}
      \only<1>{\includegraphics[width=\linewidth]{img/two_points}}\only<2>{\includegraphics[width=\linewidth]{img/Chebyshev_distance}}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{Chebyshev distance: use}
  The Chebyshev distance makes more sense than the Euclidean distance
  in some applications.  \Eg, playing chess, it is the minimal number
  of moves a king requires to move between two squares.

  \vfill

  \begin{center}
    \includegraphics[width=.33\linewidth]{img/chessboard_distance}
  \end{center}

  \vfill

  \begin{block}{\tiny \copyright 2012 Colin M.L. Burnett (in Wikimedia
      Commons)}
    \tiny \includegraphics[scale=.15]{img/licenses/cc-by-sa}\hskip1pt
    This chessboard is licensed under the Creative Commons
    Attribution-ShareAlike 3.0 Unported License.
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Hamming distance}
  Two nominal values are distant of 0 is they are equal, 1 otherwise.

  \vfill
  \pause

  \begin{definition}
    The Hamming distance is \only<-3>{the number of values to
      substitute to transform one object into the other
      one}\only<4>{half the Manhattan distance after splitting all
      attributes into 0/1 attributes ``being in the category''}.
  \end{definition}

  \vfill

  \begin{columns}
    \only<-3>
    {
      \begin{column}{.33\linewidth}
        \begin{displaymath}
          \begin{array}{c|cc}
            & \mathtt{sex} & \mathtt{job}\\
            \hline
            o_1 & \text{male} & \text{teacher}\\
            o_3 & \text{female} & \text{teacher}
          \end{array}
        \end{displaymath}
      \end{column}
      \begin{column}{.66\linewidth}
        \only<3>
        {
          \begin{displaymath}
            (1 - \delta_{\text{male}, \text{female}}) + (1 - \delta_{\text{teacher},
              \text{teacher}}) = 1
          \end{displaymath}
        }
      \end{column}
    }\only<4>{
      \begin{column}{.5\linewidth}
        \begin{displaymath}
          \begin{array}{c|cccc}
            & \mathtt{male} & \mathtt{female} & \mathtt{teacher}\\
            \hline
            o_1 & 1 & 0 & 1\\
            o_3 & 0 & 1 & 1
          \end{array}
        \end{displaymath}
      \end{column}
      \begin{column}{.5\linewidth}
        \begin{displaymath}
          \frac{1}{2}(|1 - 0| + |0 - 1| + |1 - 1|) = 1
        \end{displaymath}
      \end{column}
    }
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{Hamming distance: use}
  The ``default'' (most natural) distance between objects described
  with nominal attributes.

  % \vfill
  % \pause

  % The Lee distance over a $q$-ary alphabet generalizes the Hamming
  % distance (cases $q = 2$) but requires a metric on each categorical
  % attribute:
  % \begin{displaymath}
  %   d(o_i, o_j) = \sum_{k = 1}^n \min(|o_{i, k} - o_{j, k}|, q -
  %   |o_{i, k} - o_{j, k}|) \enspace .
  % \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Edit distances}
  Other distances (\eg, the Levenshtein and Damerau-Levenshtein
  distances) are defined between strings of different sizes. They are
  the minimal number of edit operations to transform one string into
  the other one.  Insertions, deletions, substitutions and (adjacent
  or not) transpositions can be counted. Other distances are based on
  aligning the two words, \eg, the BLAST method.

  \vfill
  \pause

  They are computationally costly.
\end{frame}

\begin{frame}
  \frametitle{Distances between vertices in a graph}
  In a (respectively, weighted) graph, the distance between two
  vertices can be the number of edges on the shortest path between
  them (respectively, the path with the smallest sum of weights).

  \vfill

  \begin{columns}
    \begin{column}{.5\linewidth}
      \includegraphics[width=\linewidth]{img/graph_distances}
    \end{column}
    \begin{column}{.5\linewidth}
      \begin{block}{\tiny \copyright 2009 HB (in Wikimedia Commons)}
        \tiny \includegraphics[scale=.15]{img/licenses/cc-by-sa}\hskip1pt
        This graph is licensed under the Creative Commons
        Attribution-ShareAlike 3.0 License.
      \end{block}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{When similar is enough}
  A \only<1>{distance}\only<2>{similarity} is a real function on pairs
  of objects. It satisfies:
  \begin{displaymath}
    \begin{cases}
      \uncover<1>{\forall (o_i, o_j) \in \mathcal{O}^2$, $D(o_i, o_j)
        \geq 0}\\
      \uncover<1>{\forall (o_i, o_j) \in \mathcal{O}^2$, $D(o_i, o_j)
        = 0 \Leftrightarrow o_i = o_j}\\
      \forall (o_i, o_j) \in \mathcal{O}^2$, $D(o_i, o_j) = D(o_j,
      o_i)\\
      \uncover<1>{\forall (o_i, o_j, o_k) \in \mathcal{O}^3$, $D(o_i,
        o_j) + D(o_j, o_k) \geq D(o_i, o_k)}
    \end{cases}.
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Cosine similarity}
  \begin{definition}
    The cosine similarity \only<-2>{is the cosine of the angle between
      the objects}\only<3>{between two objects $o_i$ and $o_j$} seen
    as vectors\only<3>{ is $\frac{o_i \cdot
        o_j}{\|o_i\|_2\|o_j\|_2}$}.
  \end{definition}

  \vfill

  \begin{columns}
    \begin{column}{.33\linewidth}
      \begin{displaymath}
        \begin{array}{c|cc}
          & x & y\\
          \hline
          o_1 & 91 & 70\\
          o_3 & 359 & 243
        \end{array}
      \end{displaymath}
    \end{column}
    \begin{column}{.66\linewidth}
      \only<1>{\includegraphics[width=\linewidth]{img/two_points_and_origin}}\only<2->{\includegraphics[width=\linewidth]{img/cosine_similarity}}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{Cosine similarity: use}
  The objects are seen as vectors whose norms are irrelevant
  (sort of a per-object scaling).

  \vfill
  \pause

  The cosine similarity is \emph{not} related to a distance: the
  similarity can be maximal even if the objects are different and the
  traditional triangle inequality does not hold.
\end{frame}

\section{Normalization}

\begin{frame}
  \frametitle{Normalization}
  Usually, when a similarity aggregates differences over attributes of
  the interval type, these attributes are first \emph{normalized} so
  that they equally contribute to the measure.

  \vfill
  \pause

  In KNIME, the ``Normalizer'' node normalizes. The normalization
  function is output as well. ``Normalizer (Apply)'' applies
  it. ``Denormalizer'' applies its inverse.

  \vfill
  \pause

  Multiplying a normalized attribute by a constant greater
  (resp. smaller) than 1 gives it more (resp. less) weight.
\end{frame}

\begin{frame}
  \frametitle{Decimal normalization}
  \begin{definition}
    Multiplication by a power of 10 chosen so that the maximal output
    value is ``just under'' 1.
  \end{definition}

  \vfill
  \pause

  That normalization ``preserves'' the decimal
  digits... \uncover<3>{but what is the point?}
\end{frame}

\begin{frame}
  \frametitle{Min-max normalization}
  \begin{definition}
    An affine transformation of the values so that the extremal ones
    are chosen.
  \end{definition}

  \vfill
  \pause

  To be used when ``theoretical'' extrema are known.  Using the
  observed minimal and maximal values is not robust.
\end{frame}

\begin{frame}
  \frametitle{Z-score normalization}
  \begin{definition}
    The number of standard deviations above (positive Z-score) or
    below (negative Z-score) the mean.
  \end{definition}

  \vfill
  \pause

  The ``default'' normalization.
\end{frame}

\begin{frame}
  \frametitle{Application-dependent normalization}
  \Eg, you normalized the \texttt{retweets.txt} dataset using several
  linear transformations whose coefficients depend on the total number
  of retweets per team.
\end{frame}

\section{Curse of dimensionality}

\begin{frame}
  \frametitle{Hypermarkets}
  \begin{definition}
    A \emph{hypermarket} is a market with more than 3 spatial
    dimensions.  Its fruit aisle sells hyperapples, hyperbananas,
    hypergrapes, hyperlemons, etc.
  \end{definition}

  \vfill
  \pause

  \begin{alertblock}{Problem}
    In a 500-dimensional hypermarket, the hyperoranges with extra-thin
    rind (1\% of the radius) are sold for 50 cents/kg.

    Is it a good offer?
  \end{alertblock}
\end{frame}

\begin{frame}
  \frametitle{Buying in Flatland's market}
  In Flatland's markets, oranges of radius $r$ are disks whose areas
  are $\frac{\tau}{2} r^2$, with $\tau \approx 6.28318$.

  \vfill
  \pause

  With the rind occupying 1\% of $r$, the proportion of the area that
  can be eaten is
  $\frac{2\tau (0.99r)^2}{2\tau r^2} = 0.99^2 \approx 0.98$.
\end{frame}

\begin{frame}
  \frametitle{Buying in $3$-dimensional markets}
  In $3$-dimensional markets, oranges of radius $r$ are balls whose
  volumes are $\frac{2\tau r^3}{3}$, with $\tau \approx 6.28318$.

  \vfill
  \pause

  With the rind occupying 1\% of $r$, the proportion of the volume
  that can be eaten is
  $\frac{6\tau (0.99r)^3}{6\tau r^3} = 0.99^3 \approx 0.97$.
\end{frame}

\begin{frame}
  \frametitle{Buying in $n$-dimensional markets}
  More generally, with $n$-dimensional hyperoranges whose rind occupy
  1\% of their radius, the proportion of the hypervolume that can be
  eaten is $0.99^n$:
  \only<-3>{
    \begin{displaymath}
      \begin{array}{cc}
        n & \text{eatable proportion}\\
        \hline
        \only<1>{
        4 & 0.961\\
        5 & 0.951\\
        6 & 0.941\\
        7 & 0.932\\
        8 & 0.923\\
        \vdots & \vdots}\only<2>{
        68 & 0.505\\
        69 & 0.500\\
        70 & 0.495\\
        71 & 0.490\\
        72 & 0.485\\
        \vdots & \vdots}\only<3>{
        100 & 0.366\\
        200 & 0.134\\
        300 & 0.049\\
        400 & 0.018\\
        500 & 0.007\\
        \vdots & \vdots}
      \end{array}
    \end{displaymath}
    }\only<4>{\begin{center}\includegraphics[width=.67\linewidth]{img/eatable_proportion}\end{center}}
\end{frame}

\begin{frame}
  \frametitle{Conclusions}
  \begin{block}{For shoppers}
    Don't buy hyperoranges of high dimensionality: it is a scam!  Even
    with extra-thin rind, you almost only get rind!
  \end{block}

  \vfill
  \pause

  \begin{block}{For statisticians}
    No matter how thin the rind, a point of the hyperorange sampled
    uniformly is almost surely in the rind when
    $n \rightarrow +\infty$.
  \end{block}

  \vfill
  \pause

  \begin{block}{For data-scientists}
    \only<-3>{Picture the hyperorange centered on an $n$-dimensional
      object and whose radius is the distance to the farthest object:
      if $n$ is large, the other objects, including the closest one,
      are almost as far.}\only<4>{All pairs of objects in a
      high-dimensional space are almost equidistant and any algorithm
      relying on distances (clustering, k-NN, etc.) fails: it has
      nothing to eat.}
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Curse of dimensionality}
  \begin{block}{Curse of dimensionality}
    When the dimensionality goes to infinity, the similarity between
    any two objects becomes the same.
  \end{block}

  % \vfill

  % \begin{center}
  %   \includegraphics[width=.67\linewidth]{img/curse_of_dimensionality}
  % \end{center}

  \vfill
  \pause

  \begin{alertblock}{Remember}
    Similarities between objects described with more than $\approx 10$
    attributes are usually meaningless.
  \end{alertblock}

  \vfill
  \pause

  Selecting highly correlated attributes is taking into account
  several times the same information.
\end{frame}

\begin{frame}
  \frametitle{Dimensionality reduction}
  Dimensionality reduction techniques rewrite objects using fewer
  attributes and losing as little information (whose definition varies
  from a technique to another) as possible.  Those technique alleviate
  the curse of dimensionality.

  \vfill
  \pause

  If they \emph{extract} new attributes (\ie, unlike feature
  \emph{selection}), the subsequent analyses are harder to interpret.
  Many techniques can be reversed to describe objects in the reduced
  space as objects in the original space.

  \vfill
  \pause

  Remember however that dimensionality reduction loses information:
  many potential objects in the original space are reduced to a same
  object in the reduced space.

  % \vfill
  % \pause

  % The \emph{principle component analysis} rewrites objects described
  % with interval-scaled attributes.  The extracted attributes are
  % linear combinations of the original attributes and are uncorrelated.
  % Those with the smallest variances can be removed to avoid the curse
  % of dimensionality.
\end{frame}

\section{Principal Component Analysis}

\begin{frame}
  \frametitle{PCA: definition}
  \begin{definition}
    Given objects described with attributes of the interval type
    repeatedly extract an attribute, called \emph{component} that:
    \begin{itemize}
    \item is a linear combinations of the input attributes;
    \item is orthogonal to every previous component;
    \item maximizes the variance of the objects projected onto it.
    \end{itemize}
  \end{definition}

  \pause
  \vfill

  The components are uncorrelated and can be interpreted: they
  sequentially summarize the distribution of the objects.
\end{frame}

\begin{frame}
  \frametitle{PCA: illustration}
  \begin{center}
    \includegraphics[width=.33\linewidth]{img/PCA}
  \end{center}

  \vfill

  \uncover<2>{After normalizing every component using the Z-score,
    the Euclidean distance corresponds to the Mahalanobis distance in
    the original dataset.  It is often used to detect outliers.}

  \vfill

  \begin{block}{\tiny \copyright 2009 Ben FrantzDale (in Wikimedia
      Commons)}
    \tiny \includegraphics[scale=.01]{img/licenses/gnu}\hskip1pt
    This plot is licensed under the GNU Free Documentation License,
    Version 1.2 or any later version.
  \end{block}
\end{frame}

% \begin{frame}
%   \frametitle{Data analysis with PCA}
%   The sequence of components summarizes the distribution.

%   \vfill

%   \begin{center}
%     \includegraphics[width=.67\linewidth]{img/PCA_fish}
%   \end{center}

%   \vfill

%   \begin{block}{\tiny \copyright 2006 Lehalle (in Wikimedia
%       Commons)}
%     \tiny \includegraphics[scale=.15]{img/licenses/cc-by-sa}\hskip1pt
%     This plot is licensed under the the Creative Commons
%     Attribution-ShareAlike 2.5 License.
%   \end{block}
% \end{frame}

\begin{frame}
  \frametitle{Dimensionality reduction with PCA}
  If the last components capture a small fraction of the total
  variance, then they can be removed for dimensionality reduction.

  \vfill
  \pause

  In KNIME, the ``PCA'' node performs Principle Component
  Analysis. ``PCA Compute'' outputs the transformation function
  too. ``PCA Apply'' applies it. ``PCA Inversion'' applies its
  inverse.

  \vfill
  \pause

  Reducing dimensionality, the inversion is inexact.  For the chosen
  dimensionality, PCA minimizes the residual sum of squares.
\end{frame}

\section*{}

\begin{frame}
  \frametitle{License}
  \begin{block}{\copyright 2012--2025 Loïc Cerf}
    \includegraphics[height=1.5ex]{img/licenses/cc-by-sa}\hskip1pt
    These slides are licensed under the Creative Commons
    Attribution-ShareAlike 4.0 International License.
  \end{block}
\end{frame}

\end{document}