%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%Tutorial slides on Python.

%

% Author: FOSSEE

% Copyright (c) 2009, FOSSEE, IIT Bombay

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\documentclass[14pt,compress]{beamer}

%\documentclass[draft]{beamer}

%\documentclass[compress,handout]{beamer}

%\usepackage{pgfpages} 

%\pgfpagesuselayout{2 on 1}[a4paper,border shrink=5mm]

% Modified from: generic-ornate-15min-45min.de.tex

\mode<presentation>

{

  \usetheme{Warsaw}

  \useoutertheme{infolines}

  \setbeamercovered{transparent}

}

\usepackage[english]{babel}

\usepackage[latin1]{inputenc}

%\usepackage{times}

\usepackage[T1]{fontenc}

% Taken from Fernando's slides.

\usepackage{ae,aecompl}

\usepackage{mathpazo,courier,euler}

\usepackage[scaled=.95]{helvet}

\usepackage{amsmath}

\definecolor{darkgreen}{rgb}{0,0.5,0}

\usepackage{listings}

\lstset{language=Python,

    basicstyle=\ttfamily\bfseries,

    commentstyle=\color{red}\itshape,

  stringstyle=\color{darkgreen},

  showstringspaces=false,

  keywordstyle=\color{blue}\bfseries}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Macros

\setbeamercolor{emphbar}{bg=blue!20, fg=black}

\newcommand{\emphbar}[1]

{\begin{beamercolorbox}[rounded=true]{emphbar} 

      {#1}

 \end{beamercolorbox}

}

\newcounter{time}

\setcounter{time}{0}

\newcommand{\inctime}[1]{\addtocounter{time}{#1}{\tiny \thetime\ m}}

\newcommand{\typ}[1]{\lstinline{#1}}

\newcommand{\kwrd}[1]{ \texttt{\textbf{\color{blue}{#1}}}  }

%%% This is from Fernando's setup.

% \usepackage{color}

% \definecolor{orange}{cmyk}{0,0.4,0.8,0.2}

% % Use and configure listings package for nicely formatted code

% \usepackage{listings}

% \lstset{

%    language=Python,

%    basicstyle=\small\ttfamily,

%    commentstyle=\ttfamily\color{blue},

%    stringstyle=\ttfamily\color{orange},

%    showstringspaces=false,

%    breaklines=true,

%    postbreak = \space\dots

% }

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Title page

\title[Statistics]{Python for Science and Engg:\\ Basic data processing}

\author[FOSSEE] {FOSSEE}

\institute[IIT Bombay] {Department of Aerospace Engineering\\IIT Bombay}

\date[] {SciPy 2010, Introductory tutorials,\\Day 1, Session 3}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%\pgfdeclareimage[height=0.75cm]{iitmlogo}{iitmlogo}

%\logo{\pgfuseimage{iitmlogo}}

%% Delete this, if you do not want the table of contents to pop up at

%% the beginning of each subsection:

\AtBeginSubsection[]

{

  \begin{frame}<beamer>

    \frametitle{Outline}

    \tableofcontents[currentsection,currentsubsection]

  \end{frame}

}

\AtBeginSection[]

{

  \begin{frame}<beamer>

    \frametitle{Outline}

    \tableofcontents[currentsection,currentsubsection]

  \end{frame}

}

\newcommand{\num}{\texttt{numpy}}

% If you wish to uncover everything in a step-wise fashion, uncomment

% the following command: 

%\beamerdefaultoverlayspecification{<+->}

%\includeonlyframes{current,current1,current2,current3,current4,current5,current6}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% DOCUMENT STARTS

\begin{document}

\begin{frame}

  \maketitle

\end{frame}

%% \begin{frame}

%%   \frametitle{Outline}

%%   \tableofcontents

%%   % You might wish to add the option [pausesections]

%% \end{frame}

\section{Computing the mean}

\begin{frame}

  \frametitle{Value of acceleration due to gravity?}

  \begin{itemize}

    \item We already have \typ{pendulum.txt}

    \item We know that $ T = 2\pi \sqrt{\frac{L}{g}} $

    \item So $ g = \frac{4 \pi^2 L}{T^2}  $

    \item Calculate $g$ - acceleration due to gravity for each pair of

        $L$ and $T$

    \item Hence calculate mean $g$

  \end{itemize}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Acceleration due to gravity - $g$\ldots}

  \begin{lstlisting}

In []: g_list = []

In []: for line in open('pendulum.txt'):

  ....     point = line.split()

  ....     L = float(point[0])

  ....     t = float(point[1])

  ....     g = 4 * pi * pi * L / (t * t)

  ....     g_list.append(g)

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Mean $g$ - Classical method}

  \begin{lstlisting}

In []: total = 0

In []: for g in g_list:

 ....:     total += g

 ....:

In []: g_mean = total / len(g_list)

In []: print 'Mean: ', g_mean

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Mean $g$ - Slightly improved method}

  \begin{lstlisting}

In []: g_mean = sum(g_list) / len(g_list)

In []: print 'Mean: ', g_mean

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Mean $g$ - One liner}

  \begin{lstlisting}

In []: g_mean = mean(g_list)

In []: print 'Mean: ', g_mean

  \end{lstlisting}

  \inctime{10}

\end{frame}

\section{Processing voluminous data}

\begin{frame}

  \frametitle{More on data processing}

  \begin{block}{}

    We have a huge data file--180,000 records.\\How do we do

    \emph{efficient} statistical computations, i.e. find mean, median,

    standard deviation etc.;\\How do we draw pie charts?

  \end{block}

\end{frame}

\begin{frame}

  \frametitle{Structure of the file}

  Understanding the structure of \typ{sslc1.txt}

  \begin{itemize}

    \item Each line in the file has a student's details(record)

    \item Each record consists of fields separated by ';'

  \end{itemize}

\emphbar{A;015162;JENIL T P;081;060;77;41;74;333;P;;}

\end{frame}

\begin{frame}

  \frametitle{Structure of the file \ldots}

\emphbar{A;015163;JOSEPH RAJ S;083;042;47;AA;72;244;;;}

  Each record consists of:

  \begin{itemize}

    \item Region Code

    \item Roll Number

    \item Name

    \item Marks of 5 subjects: second lang, first lang., Math, Science,

        Social Studies

    \item Total marks

    \item Pass/Fail (P/F)

    \item Withheld (W)

  \end{itemize}

  \inctime{5}

\end{frame}

\begin{frame}

  \frametitle{Statistical Analysis: Problem statement}

  1. Read the data supplied in the file \typ{sslc1.txt} and carry out the following:

  \begin{itemize}

    \item[a] Draw a pie chart representing proportion of students who scored more than 90\% in each region in Science.

    \item[b] Print mean, median and standard deviation of math scores for all regions combined.

  \end{itemize}

\end{frame}

\begin{frame}

  \frametitle{Problem statement: explanation}

    \emphbar{a. Draw a pie chart representing proportion of students who scored more than 90\% in each region in Science.}

\begin{columns}

    \column{5.25\textwidth}

    \hspace*{.5in}

\includegraphics[height=2.6in, interpolate=true]{data/science}

    \column{0.8\textwidth}

\end{columns}

\end{frame}

\begin{frame}

  \frametitle{Machinery Required}

  \begin{itemize}

    \item File reading

    \item Parsing

    \item Dictionaries 

    \item Arrays

    \item Statistical operations

  \end{itemize}

\end{frame}

\subsection{Data processing}

\begin{frame}[fragile]

  \frametitle{File reading and parsing \ldots}

\emphbar{Reading files line by line is the same as we had done with the pendulum example.}

  \begin{lstlisting}

for record in open('sslc1.txt'):

    fields = record.split(';')

  \end{lstlisting}

\end{frame}

\subsection{Dictionaries}

\begin{frame}[fragile]

  \frametitle{Dictionaries: Introduction}

  \begin{itemize}

    \item Lists index using integers\\

Recall \typ{p = [2, 3, 5, 7]} and\\

\typ{p[1]} is equal to \typ{3}

    \item Dictionaries index using strings

  \end{itemize}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Dictionaries \ldots}

  \begin{lstlisting}

In []: d = {'png' : 'image file',

      'txt' : 'text file', 

      'py' : 'python code',

      'java': 'bad code', 

      'cpp': 'complex code'}

In []: d['txt']

Out[]: 'text file'

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Dictionaries \ldots}

  \begin{lstlisting}

In []: 'py' in d

Out[]: True

In []: 'jpg' in d

Out[]: False

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Dictionaries \ldots}

  \begin{small}

    \begin{lstlisting}

In []: d.keys()

Out[]: ['cpp', 'py', 'txt', 'java', 'png']

In []: d.values()

Out[]: ['complex code', 'python code',

        'text file', 'bad code', 

        'image file']

    \end{lstlisting}

  \end{small}

  \inctime{10}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Inserting elements into dictionary}

  \emphbar{\alert{d[key] = value}}

  \begin{lstlisting}

    In []: d['bin'] = 'binary file'

    In []: d

    Out[]: 

    {'bin': 'binary file',

     'cpp': 'complex code',

     'java': 'bad code',

     'png': 'image file',

     'py': 'python code',

     'txt': 'text file'}

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Getting back to the problem}

  Let our dictionary be:

  \begin{lstlisting}

science = {}

  \end{lstlisting}

\begin{itemize}

    \item Keys will be region codes

    \item Values will be the number students who scored more than 90\% in that region in Science

  \end{itemize}

  \begin{block}{Sample \typ{science} dictionary}

    \{'A': 729, 'C': 764, 'B': 1120,'E': 414, 'D': 603, 'F': 500\}

  \end{block}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Building parsed data \ldots}

  \begin{lstlisting}

science = {}

for record in open('sslc1.txt'):

    fields = record.split(';')

    region_code = fields[0].strip()

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Building parsed data \ldots}

  \begin{lstlisting}

    if region_code not in science:

        science[region_code] = 0

    score_str = fields[6].strip()

    score = 0

    if score_str != 'AA':

        score = int(score_str)

    if score > 90:

        science[region_code] += 1

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Building parsed data \ldots}

  \begin{lstlisting}

print science

print science.keys()

print science.values()

  \end{lstlisting}

\end{frame}

\subsection{Visualizing data}

\begin{frame}[fragile]

  \frametitle{Pie Chart}

  \begin{lstlisting}

    pie(science.values())

  \end{lstlisting}

\includegraphics[height=2in, interpolate=true]{data/science_nolabel}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Pie chart}

  \small

  \begin{lstlisting}

pie(science.values(), 

    labels = science.keys())

title('Students scoring 90% and above 

      in science by region')

savefig('science.png')

  \end{lstlisting}

\begin{columns}

    \column{5.25\textwidth}

    \hspace*{1.1in}

\includegraphics[height=2in, interpolate=true]{data/science}

    \column{0.8\textwidth}

\end{columns}

  \inctime{10}

\end{frame}

\begin{frame}

  \frametitle{Problem statement}

    \emphbar{b. Print mean, median and standard deviation of math scores for all regions combined.}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Building data for statistics}

  \begin{lstlisting}

math_scores = []

for record in open('sslc1.txt'):

    fields = record.split(';')

    score_str = fields[5].strip()

    score = 0

    if score_str != 'AA':

        score = int(score_str)

    math_scores.append(score)

  \end{lstlisting}

\end{frame}

\subsection{Obtaining statistics}

\begin{frame}[fragile]

  \frametitle{Obtaining statistics}

  \begin{lstlisting}

print 'Mean: ', mean(math_scores)

print 'Median: ', median(math_scores)

print 'Standard Deviation: ',

              std(math_scores)

  \end{lstlisting}

  \inctime{10}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Obtaining statistics: efficiently!}

  \begin{lstlisting}

math_array = array(math_scores)

print 'Mean: ', mean(math_array)

print 'Median: ', median(math_array)

print 'Standard Deviation: ',

              std(math_array)

  \end{lstlisting}

  \inctime{5}

\end{frame}

\begin{frame}[fragile]

  \frametitle{IPython tip: Timing}

Try the following:

  \begin{lstlisting}

In []: %timeit mean(math_scores)

In []: %timeit mean(math_array)

In []: %timeit?

  \end{lstlisting}

  \begin{itemize}

      \item \typ{\%timeit}: accurate, many measurements

      \item Can also use \typ{\%time}

      \item \typ{\%time}: less accurate, one measurement 

  \end{itemize}

  \inctime{5}

\end{frame}

\begin{frame}[fragile]

  \frametitle{What tools did we use?}

  \begin{itemize}

   \item More parsing data

   \item Dictionaries for storing data