\documentclass{beamer}
% \documentclass[handout]{beamer}

\usetheme{dcc}

\usepackage[utf8]{inputenc}
\usepackage[american]{babel}

\newcommand{\ie}{i.\,e.}
\newcommand{\Ie}{I.\,e.}
\newcommand{\eg}{e.\,g.}
\newcommand{\Eg}{E.\,g.}

\title{Mineração de Dados Aplicada}
\subtitle{A Few Words about Efficiency}

\author{Loïc Cerf}

\date{April 23rd 2025}

\institute
{
  \scriptsize
  DCC -- ICEx -- UFMG
  \bigskip\\
  \includegraphics[width=.334113288\linewidth]{images/dcc}\hfill\includegraphics[width=.5\linewidth]{images/ufmg}
}

\subject{A Few Words about Efficiency}

\keywords{efficiency, parallelization, compression, data}

\hypersetup{pdfpagemode=FullScreen}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\begin{frame}[plain]
  \titlepage
\end{frame}

\begin{frame}
  \frametitle{Outline for the next sessions}
  \begin{description}
  \item[Today] A few words about efficiency + project work;
  \item[28/04] Four ``Traditional'' Data Mining Tasks;
  \item[30/04] \alert<2->{Exam on the POSIX text-processing
      commands.}
  \end{description}

  \vfill
  \pause

  Previous exam, to train: \url{dcc.ufmg.br/~lcerf/twitch.tar.xz}

  \vfill
  \pause

  For the exam (exercises, as in the slides, on a new dataset):
  \begin{itemize}
  \item no Internet connection;
  \item bringing a USB flash drive is allowed.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Outline for this session}
  % \frametitle{Outline}
  \tableofcontents
\end{frame}

% \section{Correction of last week exercises}

% \begin{frame}
%   \frametitle{Two-step per-team normalization}
%   \begin{exampleblock}{Computing the coefficients
%       (\only<-2>{POSIX-compliant}\only<3>{using \texttt{gawk} or
%         \texttt{mawk}})}
%     \texttt{awk '\{ teams[\$2] += \$1 \}}\\
%     \only<-2>{\texttt{END \{ for (t in teams) \{ ++nb; avg +=
%         teams[t] \}}\\
%       \texttt{avg /= nb}\\}
%     \only<3>{\texttt{END \{ for (t in teams) avg += teams[t]}\\
%       \texttt{avg /= length(teams)}\\}
%     \texttt{for (t in teams) print t, avg / teams[t] \}'}\\
%     \texttt{retweets.txt > coeff}
%   \end{exampleblock}

%   \vfill
%   \pause

%   \begin{exampleblock}{Applying the coefficients
%       (\only<-2>{POSIX-compliant}\only<3>{using \texttt{gawk}})}
%     \only<-2>{\texttt{awk 'FNR == 1 \{ ++argind \}}\\
%       \texttt{argind == 1 \{ coeff[\$1] = \$2 \}}\\
%       \texttt{argind == 2 \{ \$1 *= coeff[\$2]; print \}'}\\}
%     \only<3>{\texttt{awk '}\\
%       \texttt{ARGIND == 1 \{ coeff[\$1] = \$2 \}}\\
%       \texttt{ARGIND == 2 \{ \$1 *= coeff[\$2]; print \}'}\\}
%     \texttt{coeff retweets.txt}
%   \end{exampleblock}
% \end{frame}

% \begin{frame}
%   \frametitle{One-step per-team normalization}
%   \begin{exampleblock}{Per-team normalization
%       (\only<1>{POSIX-compliant}\only<2>{using \texttt{gawk} or
%         \texttt{mawk}})}
%     \texttt{awk 'BEGIN \{}\\
%     \texttt{ while (getline < ARGV[1]) coeff[\$2] += \$1}\\
%     \only<1>{\texttt{for (t in coeff) \{ ++nb; avg += coeff[t] \}}\\
%       \texttt{avg /= nb}\\}
%     \only<2>{\texttt{for (t in coeff) avg += coeff[t]}\\
%       \texttt{avg /= length(teams)}\\}
%     \texttt{for (t in coeff)}\\
%     \texttt{coeff[t] = avg / coeff[t] \}}\\
%     \texttt{\{ \$1 *= coeff[\$2]}\\
%     \texttt{print \}' retweets.txt}
%   \end{exampleblock}
% \end{frame}

\section{A few words about efficiency}

\begin{frame}
  \frametitle{Program optimization}
  \begin{alertblock}{Remember}
    Donald Knuth. In \emph{Computing Surveys}, 6(4):261--301, Dec.
    1974:
    \begin{quotation}
      We should forget about small efficiencies, say about 97\% of the
      time: premature optimization is the root of all evil. Yet we
      should not pass up our opportunities in that critical 3\%. A
      good programmer will not be lulled into complacency by such
      reasoning, he will be wise to look carefully at the critical
      code; but only after that code has been identified.
    \end{quotation}
  \end{alertblock}
\end{frame}

\begin{frame}
  \frametitle{Choice of commands}
  Writing your own programs to \emph{linearly} process text files is
  almost always a bad idea.

  \vfill
  \pause

  The most specific commands (\texttt{head}, \texttt{tail},
  \texttt{shuf}, \texttt{cat}, \texttt{tr}, \texttt{wc}, \texttt{cut},
  \texttt{paste}, \texttt{comm}, \texttt{join}, \texttt{uniq},
  \texttt{sort}, \texttt{grep}) are the fastest.

  \vfill
  \pause

  \begin{alertblock}{Remember}
    Prefer specific commands.
  \end{alertblock}

  \vfill
  \pause

  Programs in \texttt{sed} or \texttt{awk} may not be as fast as
  compiled programs but greatly reduce the development effort.
\end{frame}

\begin{frame}
  \frametitle{Ordering the commands}
  The two solutions below produce the same output:
  \begin{enumerate}
  \item \texttt{grep -io '[\^{ } ]*galo.*' retweets.txt | sort -u}
  \item \texttt{sort -uk 4 retweets.txt | grep -io '[\^{ } ]*galo.*'}
  \end{enumerate}

  \vfill
  \pause

  The first solution is almost certainly faster because it reduces
  more the input (unless there are many repetitions in the last
  column).


  \vfill
  \pause

  \begin{alertblock}{Remember}
    First, reduce the input.
  \end{alertblock}
\end{frame}

\begin{frame}
  \frametitle{Temporary file system}
  \begin{block}{HDD vs. SSD vs. RAM transfer rates}
    \begin{description}
    \item[SATA 3 HDD] 0.142 GB/s (benchmarked with \texttt{hdparm -t});
    \item[M.2 SSD] 0.430 GB/s (benchmarked with \texttt{hdparm -t});
    \item[DDR4] 21.053 GB/s (benchmarked with \texttt{memtest86+}).
    \end{description}
  \end{block}

  \vfill
  \pause

  \begin{alertblock}{Remember}
    If the temporary files do not require too much space, store them
    on a \emph{temporary file system} (\ie, in RAM).
  \end{alertblock}

  \vfill
  \pause

  The most popular one, in the Linux and BSD kernels, is
  tmpfs. \Eg, to mount a 1 GB tmpfs in a directory named \texttt{tmp}:\\
  \texttt{\# mount -t tmpfs -o size=1G tmpfs tmp}
\end{frame}

\begin{frame}
  \frametitle{Reading compressed data}
  Reading/writing compressed data on the disk is often faster, despite
  the decompression/compression step.

  \vfill
  \pause

  \begin{alertblock}{Remember}
    Start with \texttt{(z|bz|xz|zstd)cat} or
    \texttt{(z|bz|xz|zstd)grep}, pipe the output to
    \texttt{gzip}/\texttt{bzip2}/\texttt{xz}/\texttt{zstd}, and notice
    that \texttt{less} directly reads \texttt{(gz|bz2|xz)}-compressed
    files (\texttt{zstd} soon?).
  \end{alertblock}

  \vfill
  \pause

  Data can be stored on modern filesystems, such as ZFS and Btrfs,
  with transparent compression enabled.
\end{frame}

\begin{frame}
  \frametitle{Temporary files vs. pipes}
  The two solutions below produce the same output but the second is
  faster (even if \texttt{selec} is stored in RAM):
  \begin{enumerate}
  \item \texttt{grep -io '[\^{ } ]*galo.*' retweets.txt > selec\\
      sort -u selec}
  \item \texttt{grep -io '[\^{ } ]* galo.*' retweets.txt | sort -u}
  \end{enumerate}

  \vfill
  \pause

  Pipes are buffers. In the second solution, \texttt{sort} can process
  a line as soon as \texttt{grep} outputs it.

  \vfill
  \pause

  \texttt{mkfifo} creates a named pipe. If \texttt{mkfifo selec}
  created \texttt{selec}, the two solutions are identical. However,
  the first line of the first solution must now end with \texttt{\&}
  to run in the background:\\
  \texttt{grep -io '[\^{ } ]* galo.*' retweets.txt > selec \&}
\end{frame}

\begin{frame}
  \frametitle{Forking the processing flow}
  \texttt{tee} copies the standard input to the standard output and to
  files in arguments.

  \vfill
  \pause

  Given a regular file, \texttt{tee} saves intermediary data.

  \vfill
  \pause

  Given a named pipe, \texttt{tee} forks the processing flow. \Eg:\\
  \texttt{mkfifo selec\\
    grep -i galo retweets.txt | tee selec |\\
    \hspace{2em}sort -uk 4 -o galo\_users \&\\
    awk '\{ sum += \$1 \} END \{ print sum \}' selec}
\end{frame}

\begin{frame}
  \frametitle{Pipes}
  \begin{alertblock}{Remember}
    \begin{itemize}[<+->]
    \item Prefer pipes to temporary files...
    \item but avoid useless pipes such as \texttt{cat file |},
      \texttt{shuf | head}, \texttt{sort | uniq} and \texttt{grep
        regexp | wc -l}.
    \item Write programs that read/write on the standard I/O line by
      line.
    \end{itemize}
  \end{alertblock}
\end{frame}

\begin{frame}
  \frametitle{Parallelization}
  \texttt{GNU parallel} executes commands or scripts on several cores,
  possibly in several computers. The jobs must be independent,
  typically the same computation on many files.

  \vfill
  \pause

  \texttt{GNU parallel} looks both easier to use and more powerful
  than alternatives (\texttt{pexec}, \texttt{xjobs}, etc.).
\end{frame}

\begin{frame}
  \frametitle{Specifically processing data tables}
  \texttt{GNU datamash} performs basic numeric, textual and
  statistical operations on input textual data tables.

  \vfill

  \begin{exampleblock}{Per-sex medians of the weight (9th field)}
    \texttt{\$ datamash -sHt \textbackslash ; groupby SEX iqr 9 < bears.csv}\\
    \texttt{GroupBy(SEX);iqr(WEIGHT)}\\
    \texttt{1;213}\\
    \texttt{2;112,5}
  \end{exampleblock}

  \vfill
  \pause

  \texttt{GNU R} is (one of) the most-widely used environment for
  statistical computing and graphics.
\end{frame}

\begin{frame}
  \frametitle{Caveats}
  \begin{alertblock}{Remember}
    Enclose an argument in single quotes to prevent the shell from
    interpreting it; in double quotes to still interpret \texttt{\$}
    and \texttt{`}.
  \end{alertblock}

  \vfill
  \pause

  \begin{alertblock}{Remember}
    The length of the command line is limited (to 2,097,152 characters
    on my system) and matching many files with \texttt{*} can become
    problematic.  \texttt{find} brings a solution.  \Eg, to execute
    \texttt{command} on every regular file
    below the working directory:\\
    \texttt{find . -type f -exec command '\{\}' +}
  \end{alertblock}
\end{frame}

\section{Project work}

\section*{}

\begin{frame}
  \frametitle{License}
  \begin{block}{\copyright 2011--2025 Loïc Cerf}
    \includegraphics[height=1.5ex]{img/licenses/cc-by-sa}\hskip1pt
    These slides are licensed under the Creative Commons
    Attribution-ShareAlike 4.0 International License.
  \end{block}
\end{frame}

\end{document}