\documentclass{beamer}
% \documentclass[handout]{beamer}

\usetheme{dcc}

\usepackage[utf8]{inputenc}
\usepackage[american]{babel}
\usepackage{url}
\usepackage[autolanguage]{numprint}

\newcommand{\ie}{i.\,e.}
\newcommand{\Ie}{I.\,e.}
\newcommand{\eg}{e.\,g.}
\newcommand{\Eg}{E.\,g.}

\newcommand{\pub}{\hskip2pt\raisebox{-3.5pt}{\pgfuseimage{beamericonarticle}}\hskip1pt}

\title{Mineração de Dados Aplicada}
\subtitle{Dependence between Attributes}

\author{Loïc Cerf}

\date{March, 19th 2025}

\institute
{
  \scriptsize
  DCC -- ICEx -- UFMG
  \bigskip\\
  \includegraphics[width=.334113288\linewidth]{images/dcc}\hfill\includegraphics[width=.5\linewidth]{images/ufmg}
}

\subject{Dependence between attributes}

\keywords{basic visualizations, regression, correlation}

\hypersetup{pdfpagemode=FullScreen}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\begin{frame}[plain]
  \titlepage
\end{frame}

\section*{Summary of the last session}

\begin{frame}
  \frametitle{Datasets}
  \begin{itemize}
  \item Most datasets are (potentially infinite) \emph{sets} of
    objects described with attributes;
  \item More structured datasets exist (one large graph, a video,
    etc.) and their components (vertices of a graph, scenes of a
    video, etc.) can be described with attributes derived from the
    structure;
  \item Data are usually incomplete, inconsistent, with some
    exceptions, uncertain/noisy or even plainly wrong;
  \item Understanding the application and the data
    generation/acquisition process is essential.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Attributes}
  \begin{itemize}
  \item Basic statistics (center, dispersion, etc.) and visualizations
    (histograms, box plots, etc.) help a lot in understanding the
    data; looking at specific objects too;
  \item The simplest attributes can be categorized into four types:
    nominal, ordinal, interval and ratio;
  \item The type tells what operations, statistics and data mining
    algorithms are applicable;
  \item More structured attributes exist (sequences, sounds, etc.) and
    simpler attributes (presences of sub-sequences, BPM, etc.) can be
    derived from the structures;
  \item Additional attributes, taken from other sources or derived
    from the existing data, may be more relevant.
  % \item Regression aims to explain an interval-scaled attribute as a
  %   function of other attributes;
  % \item Correlation does not imply causation.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Outline}
  \tableofcontents
\end{frame}

\section{Visualizing a dependence}

\begin{frame}
  \frametitle{Stacked histogram}
  The \emph{stacked histogram} allows to visualize a dependence
  between nominal attributes.

  \vfill

  \includegraphics[width=\linewidth]{img/histogram_color}
\end{frame}

\begin{frame}
  \frametitle{Conditional box plot}
  The \emph{conditional box plot} allows to visualize a dependence
  between a nominal attribute and an interval-scaled attribute.

  \vfill

  \includegraphics[width=\linewidth]{img/conditional_box_plot}
\end{frame}

\begin{frame}
  \frametitle{Scatter plot}
  A \emph{scatter plot} allows to visualize a dependence between two
  (or three) interval-scaled attributes.  In presence of many objects,
  a random sample can be used.

  \vfill
  \pause

  Simultaneously visualizing additional attributes of the objects:
  \begin{description}
  \item[Nominal] shapes and/or colors with different hues;
    \includegraphics[width=\linewidth]{img/hue}
  \item[Interval] areas and/or colors on a brightness gradient.
    \includegraphics[width=\linewidth]{img/saturation}
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Simple scatter plot}
  \begin{center}
    \includegraphics[width=.66\linewidth]{img/regression_before}
  \end{center}
\end{frame}

\section{Regression}

\begin{frame}
  \frametitle{Regression}
  \begin{definition}
    Learning from a set of objects how to predict the value of an
    attribute on an interval scale.
  \end{definition}

  \vfill

  \only<1>
  {
    Input:\\
    \begin{columns}
      \begin{column}{.5\linewidth}
        \begin{displaymath}
          \begin{array}{c|cccc}
            & a_1 & \dots & a_n & \alert{y}\\
            \hline
            o_1 & d_{1, 1} & \dots & d_{1, n} & \alert{y_1}\\
            o_2 & d_{2, 1} & \dots & d_{2, n} & \alert{y_2}\\
            \vdots & \vdots & \ddots & \vdots & \alert{\vdots}\\
            o_m & d_{m, 1} & \dots & d_{m, n} & \alert{y_m}
          \end{array}
        \end{displaymath}
      \end{column}
      \begin{column}{.5\linewidth}
        A parametric function, the \emph{regression model}, that takes
        the values of the \emph{independent attributes} $a_1$, \dots,
        $a_n$ at input and returns a value for the \emph{dependent
          attribute} $\alert{y}$.
      \end{column}
    \end{columns}
  }\only<2->
  {
    Output:\\

    \vfill

    Estimates of the parameters of the regression model.  The
    regression model with its parameters substituted by their
    estimates is the \emph{regression function}.

    \vfill

    \uncover<3>{If the model is \emph{interpretable} and if no
      independent attribute is well predicted from the others, the
      regression function provides an understanding of the effect of
      every independent attribute on $y$.}
  }
\end{frame}

\begin{frame}
  \frametitle{Regression: graphical interpretation}
  \begin{center}
    \only<1>{\includegraphics[width=.66\linewidth]{img/regression_before}}\only<2>{\includegraphics[width=.66\linewidth]{img/regression}}
  \end{center}

  \vfill
  \pause

  Simple linear regression ($y = c_0 + c_1 a + \epsilon$): $\begin{cases}
    \hat{c}_0 \approx 5\\
    \hat{c}_1 \approx 0.15
  \end{cases} \enspace .$
\end{frame}

\begin{frame}
  \frametitle{Making predictions}
  Applying a regression function to an object, with the value of the
  dependent attribute missing, predicts it.

  \vfill

  \only<1>{Input: a regression function of $a_1, \dots, a_n$
    and}\only<2>{Output:}\\
  \begin{displaymath}
    \begin{array}{c|cccc}
      & a_1 & \dots & a_n & \alert<2>{y}\\
      \hline
      o_{m + 1} & d_{m + 1, 1} & \dots & d_{m + 1, n} &
      \uncover<2>{\alert{y_{m + 1}}}\\
      o_{m + 2} & d_{m + 2, 1} & \dots & d_{m + 2, n} &
      \uncover<2>{\alert{y_{m + 2}}}\\
      \vdots & \vdots & \ddots & \vdots &
      \uncover<2>{\alert{\vdots}}\\
      o_p & d_{p, 1} & \dots & d_{p, n} & \uncover<2>{\alert{y_p}}
    \end{array}
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Linear regression}
  \begin{block}{(Multi-attribute) linear regression model}
    $y = c_0 + c_1 a_1 + \dots + c_n a_n + \epsilon$
  \end{block}

  \vfill
  \pause

  The (weighted) least-square method takes $O(mn^2)$ time to estimate
  the values of the parameters $c_0$, $c_1$, \dots, $c_n$ minimizing
  $\left\|\left(\epsilon_1, \epsilon_2, \dots,
      \epsilon_m\right)^T\right\|_2$ (respectively
  $\left\|\left(\frac{\epsilon_1}{\sigma_1},
      \frac{\epsilon_2}{\sigma_2}, \dots,
      \frac{\epsilon_m}{\sigma_m}\right)^T\right\|_2$).

  \vfill
  \pause

  Deriving attributes allows to fit different models.  \Eg, a linear
  regression can estimate the parameters $c_0$, $c_1$ and $c_2$ of the
  model $y = c_0 + c_1 \log{a_1} + c_2 \frac{a_1}{a_2} + \epsilon$
  after the derivation of the attributes $\log{a_1}$ from $a_1$ and
  $\frac{a_1}{a_2}$ from $a_1$ and $a_2$.
\end{frame}

\begin{frame}
  \frametitle{Homo and heteroscedasticity}
  \begin{columns}
    \begin{column}{.5\linewidth}
      \begin{center}
        Homoscedasticity\\
        \includegraphics[width=\linewidth]{img/homoscedasticity}
      \end{center}
    \end{column}
    \begin{column}{.5\linewidth}
      \begin{center}
        Heteroscedasticity\\
        \includegraphics[width=\linewidth]{img/heteroscedasticity}
      \end{center}
    \end{column}
  \end{columns}

  \vfill

  Providing standard deviations of the errors leads to a better
  estimation of the parameters when data show heteroscedasticity.
\end{frame}

\begin{frame}
  \frametitle{Other regression models}
  Other regression models do not assume that the dependent attribute
  \emph{linearly} depends on the independent attributes.  Some models
  (\eg, regression trees) are interpretable.  Others (\eg, SVR) are
  not but may provide better predictions.

  \vfill
  \pause

  When the dependent attribute is nominal, the regression is said
  \emph{logistic}.  It is also called \emph{supervised
    classification}.  We will study that topic at the end of the
  semester.
\end{frame}

\section{Correlation}

\begin{frame}
  \frametitle{Goodness of fit}
  If the chosen model is wrong or if there is no actual dependence
  between the independent attributes and the dependent attribute, the
  fit is bad, and so are predictions.

  \vfill
  \pause

  The \emph{coefficient of determination}, in $[0; 1]$, quantifies how
  much the regression function fits the data.  It is the proportion of
  the variance of the dependent attribute that the regression function
  explains.  It tells nothing about the estimates of the parameters.

  \vfill
  \pause

  With one single independent attribute, the term \emph{correlation}
  is preferred.  Correlation measures are symmetric.  They are in
  $[-1; 1]$, where the sign is that of the estimate of the parameter
  multiplying the independent attribute.
\end{frame}

\begin{frame}
  \frametitle{Correlation: graphical interpretation}
  \begin{center}
    \includegraphics[width=.66\linewidth]{img/regression}
  \end{center}

  \vfill

  Pearson product-moment correlation coefficient: $0.91$ (strong
  positive correlation).
\end{frame}

\begin{frame}
  \frametitle{Over-fitting and over-generalizing}
  The more parameters in the regression model, the better the fit.
  However, a model with too many parameters is bad: the regression
  function over-fits the data and its predictions are imprecise.

  \vfill

  \begin{description}[<+->]
  \item[Over-fitting] Fitting too much the data is learning the noise:
    the regression function poorly generalizes to new objects;
  \item[Over-generalizing] Not fitting enough the data is missing
    significant details to explain the dependent variable.
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Over-fitting: graphical interpretation}
  \begin{center}
    \includegraphics[width=.66\linewidth]{img/over-fitting}
  \end{center}

  \vfill

  The blue regression function, $y = \sum_{j = 0}^{10} \hat{c}_j a^j$,
  perfectly fits the data but over-fits them.  It generalizes worse
  than the black function, $y = \hat{c}_0 + \hat{c}_1 a$.
\end{frame}

\begin{frame}
  \frametitle{Learning the good amount}
  To compare regression models, measures such as the Bayesian and the
  Akaike information criteria quantify the trade-off between the
  goodness of fit and the number of parameters.

  \vfill
  \pause

  \emph{Stepwise regression} is greedily optimizing such a measure to
  select a model.  It is criticized for leading to over-fitting by
  data dredging.

  \vfill
  \pause

  The fit can be computed on objects that are \emph{not} used to
  estimate the parameters of the regression model.  Different
  techniques will be detailed during the courses about supervised
  classification.
\end{frame}

\begin{frame}
  \frametitle{Popular correlation measures}
  Correlation measures depend on the type of both attributes:
  \begin{description}
  \item[nominal] the (adjusted) Rand index;
  \item[ordinal] the \emph{Kendall's tau rank correlation coefficient}
    and the \emph{Spearman's rho rank correlation coefficient};
  \item[interval] the \emph{Pearson product-moment correlation
      coefficient}, $\mbox{sgn}(\hat{c}_1)\sqrt{r^2}$, where $r^2$ is
    the coefficient of determination of the regression function
    $y = \hat{c}_0 + \hat{c}_1 a$.
  \end{description}

  \vfill
  \pause

  Computing from $m$ objects the (adjusted) Rand index or Kendall's
  tau takes $O(m^2)$ time; $O(m)$ for Spearman's rho and Pearson's r.
\end{frame}

\begin{frame}
  \frametitle{Pearson correlation coefficient}
  If the two attributes are not linearly dependent, the Pearson
  correlation coefficient does not apply.  However, ignoring the error
  term, nonlinear models can be turned linear.  \Eg,
  $y = c_0 e^{c_1 a}$ becomes $\ln{y} = \ln{c_0} + c_1 a$, a linear
  dependence between $a$ and $\ln{y}$.

  \vfill
  \pause

  Pearson's r is not robust.  Kendall's tau and Spearman's rho are
  robust since they are based on ranks and not on actual values.
\end{frame}

\section*{}

\begin{frame}
  \frametitle{License}
  \begin{block}{\copyright 2012--2025 Loïc Cerf}
    \includegraphics[height=1.5ex]{img/licenses/cc-by-sa}\hskip1pt
    These slides are licensed under the Creative Commons
    Attribution-ShareAlike 4.0 International License.
  \end{block}
\end{frame}

\end{document}