\documentclass{beamer}
% \documentclass[handout]{beamer}

\usetheme{dcc}

\usepackage[utf8]{inputenc}
\usepackage[american]{babel}
\usepackage{url}

\newcommand{\ie}{i.\,e.}
\newcommand{\Ie}{I.\,e.}
\newcommand{\eg}{e.\,g.}
\newcommand{\Eg}{E.\,g.}

\title{Mineração de Dados Aplicada}
\subtitle{Data Exploration}

\author{Loïc Cerf}

\date{March, 17th 2025}

\institute
{
  \scriptsize
  DCC -- ICEx -- UFMG
  \bigskip\\
  \includegraphics[width=.334113288\linewidth]{images/dcc}\hfill\includegraphics[width=.5\linewidth]{images/ufmg}
}

\subject{Data Exploration}

\keywords{data types, basic statistics, basic visualizations,
  datasets, sampling bias, Berkson's paradox}

\hypersetup{pdfpagemode=FullScreen}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\begin{frame}[plain]
  \titlepage
\end{frame}

\section*{Summary of the last session}

\begin{frame}
  \frametitle{Data mining}
  \begin{itemize}
  \item Data mining is an empiricism;
  \item It can be seen as a generalization of querying;
  \item It lacks a unified theory;
  \item It implies trades-off between quality and computational
    complexity;
  \item It should always be practiced in an ethical way (privacy
    concerns).
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{The pattern discovery process}
  \begin{itemize}
  \item The pattern discovery process is iterative;
  \item It is interactive, hence statistics and visualization
    techniques, on both the data and the patterns, are essential;
  \item It involves pre-processing steps to get a dataset that:
    \begin{itemize}
    \item is relevant to the analysis;
    \item can be processed by chosen data mining algorithms.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Outline}
  \tableofcontents
\end{frame}

\section{Data structure}

\begin{frame}
  \frametitle{Classical data structure}
  Most datasets can be represented as tables that describe objects
  (the rows, whose order is meaningless) with attributes (the columns,
  whose order is meaningless):
  \begin{displaymath}
    \begin{array}{c|cccc}
      & a_1 & a_2 & \dots & a_n\\
      \hline
      o_1 & d_{1, 1} & d_{1, 2} & \dots & d_{1, n}\\
      o_2 & d_{2, 1} & d_{2, 2} & \dots & d_{2, n}\\
      \vdots & \vdots & \vdots & \ddots & \vdots\\
      o_m & d_{m, 1} & d_{m, 2} & \dots & d_{m, n}
    \end{array}
  \end{displaymath}

  \vfill

  $m$ is called size of the dataset; $n$ its dimensionality.
\end{frame}

\begin{frame}
  \frametitle{Data streams}
  A dataset with an infinite size is a data stream.  The complexity of
  an algorithm processing it cannot depend on the number of objects
  seen so far.
\end{frame}

\begin{frame}
  \frametitle{The dataset is a sample}
  The dataset with a finite size is seen as a \emph{sample}.  Every
  object is a \emph{realization} of the object of study, an
  $n$-dimensional \emph{random variable}, whose \emph{probability
    distribution} is unknown.

  \vfill
  \pause

  It is almost always assumed that, in the sample, the objects are
  \emph{independent and identically distributed}.

  \vfill
  \pause

  It is important to understand whether (or to what extent) the
  assumptions hold. If not, many analyses do not apply.
\end{frame}

\begin{frame}
  \frametitle{Sampling bias}
  Data collected with an inadequate sampling method may not be
  distributed identically to the studied phenomenon:
  \begin{itemize}[<+->]
  \item selection from a specific area (\eg, ``man on the street''
    interview) or following edges in a (\eg, social) network;
  \item survivorship bias (\eg, studying current companies as an
    indicator of business climate);
  \item self-selection bias (\eg, online and phone-in polls);
  \item publication bias (the result of the study biases its
    publication);
  \item Malmquist bias (preferential detection of intrinsically bright
    objects);
  \item ...
  \end{itemize}

  \vfill

  \uncover<6>{Were caves the most common home of prehistoric people or
    just where the data still exist?}
\end{frame}

\begin{frame}
  \frametitle{Berkson's paradox}
  \begin{exampleblock}{Is the adage true?}
    Are beautiful people assholes (or, equivalently, less good-looking
    people nicer)?
  \end{exampleblock}

  \vfill
  \pause

  If you think of people you dated (a sampling bias): yes, they are.
  \uncover<3->{Because you do not date people who are both ugly and
    mean.  And, in this example, because you do not date people who
    are both very beautiful and very nice: they are ``out of your
    league''.}

  \vfill

  \uncover<4>{Joseph Berkson understood such spurious negative
    correlations between diseases in a hospital (missing healthy
    people).}
\end{frame}

\begin{frame}
  \frametitle{Bias and noise}
  \begin{center}
    \includegraphics[width=.4\linewidth]{img/bias_and_noise}
  \end{center}

  \vfill

  \begin{block}{\tiny \copyright 2021 Tim Isaksson (from Wikimedia
      Commons)}
    \tiny \includegraphics[scale=.15]{img/licenses/cc0}\hskip1pt
    This illustration is licensed under the Creative Commons CC0 1.0
    Universal Public Domain Dedication.
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{More structured datasets}
  Patterns can be searched inside one large word, sequence, graph
  (directed or not, weighted or not, labeled or not), with timestamps
  (including sounds), with spacial positions (including pictures),
  etc.

  \vfill
  \pause

  Studying and using a method processing such data can be the
  non-trivial step of your project.

  \vfill
  \pause

  Components of the structured dataset can be studied too.  They are
  the objects of another dataset.  Their attributes can be derived
  from the initial structure (degree in the graph, distance to a
  reference object, statistics on neighbors, etc.).
\end{frame}

\section{Data exploration and completion}

\begin{frame}
  \frametitle{Difficulties}
  Data are usually:
  \begin{itemize}[<+->]
  \item incomplete;
  \item inconsistent;
  \item with some exceptions;
  \item uncertain/noisy or even plainly wrong.
  \end{itemize}

  \vfill

  \uncover<5>{\alert{Never assume the data are perfect.} Detect the
    problems (basic statistics and visualizations help a lot) and
    understand the limitations of the data generation/acquisition
    process.}
\end{frame}

\begin{frame}
  \frametitle{Looking at specific objects}
  In particular, to check the definitions of the attributes, it is
  essential to look at particular objects:
  \begin{itemize}
  \item those taking uncommon or extreme values;
  \item those that you particularly know;
  \item those that everybody knows.
  \end{itemize}

  \vfill
  \pause

  To do so, you can fire SQL queries (if the dataset is in a database)
  or POSIX commands (if in text files) or use GNU datamash or...
\end{frame}

\begin{frame}
  \frametitle{Using background knowledge}
  It is essential to understand the application domain and to consider
  complementing the dataset with additional attributes:
  \begin{itemize}
  \item taken from other sources;
  \item derived from existing attributes.
  \end{itemize}

  \vfill
  \pause

  \begin{exampleblock}{Spam detection}
    Given an email, its date and the IP addresses sending and
    receiving it, \uncover<3>{the presence of some words (``viagra'',
      ``pr0n'', etc.), the country of the sender, the match/mismatch
      between the language of the email and that of the two countries,
      whether the day is holiday, the number of emails from the sender
      in the dataset}, ...  may help its classification as spam/ham.
  \end{exampleblock}
\end{frame}

\section{Analyzing one single attribute}

\begin{frame}
  \frametitle{Typology of an attribute}
  In 1946, Stanley Smith Stevens proposed to categorize the attributes
  into four types, depending on the allowed operations:
  \begin{description}[<+->]
  \item[$=$, $\neq$] Nominal (Boolean if there are only two values);
  \item[$<$, $>$] Ordinal (with a total order);
  \item[$+$, $-$] Interval (differences make sense but an arbitrary
    0);
  \item[$\times$, $/$] Ratio (ratios make sense).
  \end{description}

  \vfill

  \uncover<5->
  {
    Other authors complement that typology with gradations of
    membership, proportions ($\in [0, 1]$), counts ($\in \mathbb{N}$),
    amounts ($\in \mathbb{R}_+$), log intervals, cyclical ratios, etc.
  }

  \vfill

  \uncover<6>
  {
    Identifying the type of every attribute tells
    what operations, statistics and data mining algorithms are
    applicable.
  }
\end{frame}

\begin{frame}
  \frametitle{Structured attributes}
  Objects can be described with attributes whose values are
  matrices/tensors, words, sequences, graphs (directed or not,
  weighted or not, labeled or not), with timestamps (including
  sounds), with spacial positions (including pictures), etc.

  \vfill
  \pause

  Studying and using a method processing such data can be the
  non-trivial step of your project.

  \vfill
  \pause

  Properties of a structured attribute (its size, counts of patterns
  in it, dominant color, BPM, etc.) can substitute it, with loss of
  information.  Metadata (author, creation date, tags, length, etc.)
  are valuable too.
\end{frame}

\begin{frame}
  \frametitle{Centrality, dispersion and skewness}
  Many (but \alert<2>{not all}) attributes take values that are
  distributed around one center.

  \vfill
  \pause

  \begin{exampleblock}{Average human}
    The average number of ovaries is 1.  The average number of
    testicles too.  People with 1 ovary and 1 testicle are rarities.
  \end{exampleblock}

  \vfill
  \pause

  The dispersion of the values (how stretched their distribution) is
  often relevant too.

  \vfill
  \pause

  A skewed distribution leans to one side of its mean.
\end{frame}

\begin{frame}
  \frametitle{Skewness}
  \begin{center}
    \includegraphics[width=\linewidth]{img/skewness}
  \end{center}

  \vfill

  \begin{block}{\tiny \copyright 2008 Rodolfo Hermans (from Wikimedia
      Commons)}
    \tiny \includegraphics[scale=.15]{img/licenses/cc-by-sa}\hskip1pt
    These diagrams are licensed under the Creative Commons Attribution
    ShareAlike 3.0 Unported License.
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Basic statistics}
  Estimating from the dataset the center and the dispersion of the
  values taken by an attribute is very useful. The applicable
  statistics depend on the type of the attribute:
  \begin{description}[<+->]
  \item[nominal] the mode (the most common value) and the entropy;
  \item[ordinal] the mode, the median (50\% of the values are smaller,
    50\% greater), the entropy, the $\min$ and the $\max$;
  \item[interval] all the above plus the arithmetic mean, the range
    ($\max - \min$), the IQR, the standard deviation, etc.
  \item[ratio] all the above plus the geometric (for growths) and
    harmonic (for rates) means, the studentized range (difference of
    the z-scores of the largest and the smallest values), the
    coefficient of variation (ratio of the standard deviation and the
    mean), etc.
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Robustness of a statistic}
  A \emph{robust} statistic is insensitive to extreme values (called
  \emph{outliers}). A non-robust statistic may be very different from
  the one that would be computed from ``all objects'' (not a sample).

  \vfill
  \pause

  The arithmetic/geometric/harmonic means are not robust statistics of
  centrality.  The \emph{truncated mean} is robust.  It is the
  arithmetic mean computed after discarding the most extreme values, a
  fraction to be chosen.  Choosing 50\%, the median is obtained.

  \vfill
  \pause

  The standard deviation, the correlation coeffient and, worse, the
  (studentized) range are not robust statistics of dispersion. The
  \emph{interquartile range} (IQR) is robust.  It is the range for the
  middle 50\% of the values, the third \emph{quartile} minus the
  first.
\end{frame}

\begin{frame}
  \frametitle{Estimating the skewness}
  For the interval type, several statistics aim to measure the
  skewness.  Pearson's skewness statistics are the most popular:
  \begin{itemize}
  \item $\frac{\text{mean} - \text{mode}}{\text{standard deviation}}$;
  \item $\frac{\text{mean} - \text{median}}{\text{standard deviation}}$;
  \item the Pearson's moment coefficient of skewness (complicated
    formula with better statistical foundations).
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Basic visualizations}
  A histogram depicts the distribution of the values taken by an
  attribute, whatever its type.  If ordinal, the domain is partitioned
  into intervals, bases of the rectangles.  If interval/ratio-scaled,
  the areas (rather than the heights) of the rectangles are
  proportional with the number of values in the interval.  Do not use
  pie charts.

  \vfill
  \pause

  For an interval/ratio-scaled attribute, a box plot is an alternative
  visualization of the distribution of its values.  It shows the
  quartiles and the outliers defined as the values 1.5IQR below/above
  the first/third quartile.  That definition can be tuned.
\end{frame}

\begin{frame}
  \frametitle{Histogram}
  \begin{center}
    \includegraphics[width=.75\linewidth]{img/histogram}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Box plot}
  \begin{center}
    \includegraphics[width=.75\linewidth]{img/box_plot}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{A few well-known distributions}
  A known distribution may well model the observed values.

  \vfill

  \uncover<2->
  {
    \begin{columns}
      \begin{column}{.33\linewidth}
        \only<-3>{\only<-2>{Discrete}\only<3>{Continuous} uniform
          distribution (for a\only<-2>{ }\only<3>{n in}finite number
          of possible values)}\only<4>{Binomial
          distribution}\only<5>{Normal distribution}:
      \end{column}
      \begin{column}{.66\linewidth}
        \begin{center}
          \only<-2>{\includegraphics[width=.8\linewidth]{img/uniform_discrete}}
          \only<3>{\includegraphics[width=.8\linewidth]{img/uniform_continuous}}
          \only<4>{\includegraphics[width=\linewidth]{img/binomial}}
          \only<5>{\includegraphics[width=\linewidth]{img/normal}}
        \end{center}
      \end{column}
    \end{columns}

    \vfill

    \begin{block}{\tiny \copyright \only<-3>{2013
          IkamusumeFan}\only<4>{Tayste}\only<5>{Inductiveload} (from
        Wikimedia Commons)}
      \tiny
      \only<-3>{\includegraphics[scale=.15]{img/licenses/cc-by-sa}}\only<4->{\includegraphics[scale=.1]{img/licenses/cc-public_domain}}\hskip1pt
      This plot is \only<-3>{licensed under the Creative Commons
        Attribution ShareAlike 3.0 Unported License}\only<4->{in the
        public domain}.
    \end{block}
  }
\end{frame}

\begin{frame}
  \frametitle{Heavy-tailed distribution}
  \emph{Heavy-tailed} distributions have an infinite variance and
  possibly an infinite mean.

  \vfill
  \pause

  \begin{exampleblock}{Pareto distribution}
    If, across more than three units in abscissa, a straight line well
    fits the scatter plot of the logarithm of the ranks of the values
    in function of the logarithm of the values themselves, then
    $\log(P(a > x)) \approx \log(p) - q\log(x) \Leftrightarrow P(a >
    x) \approx px^{-q}$, a power law defining a Pareto distribution.
    If $q \leq 2$, its variance is infinite.  If $q \leq 1$, so is its
    mean.
  \end{exampleblock}

  \vfill
  \pause

  Preferential attachment can generate power law distributions.
\end{frame}

\begin{frame}
  \frametitle{Zipf's law}
  The Zipf distribution is the discrete Pareto distribution.  Zipf
  observed that, in texts, the most frequent word occurs $k$ times
  more than the $k$th most frequent work.

  \vfill

  \begin{center}
    \includegraphics[width=.5\linewidth]{img/Zipf}
  \end{center}
\end{frame}

\section*{}

\begin{frame}
  \frametitle{License}
  \begin{block}{\copyright 2011--2025 Loïc Cerf}
    \includegraphics[height=1.5ex]{img/licenses/cc-by-sa}\hskip1pt
    These slides are licensed under the Creative Commons
    Attribution-ShareAlike 4.0 International License.
  \end{block}
\end{frame}

\end{document}