%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%Tutorial slides on Python.

%

% Author: FOSSEE

% Copyright (c) 2009, FOSSEE, IIT Bombay

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\documentclass[14pt,compress]{beamer}

%\documentclass[draft]{beamer}

%\documentclass[compress,handout]{beamer}

%\usepackage{pgfpages} 

%\pgfpagesuselayout{2 on 1}[a4paper,border shrink=5mm]

% Modified from: generic-ornate-15min-45min.de.tex

\mode<presentation>

{

  \usetheme{Warsaw}

  \useoutertheme{infolines}

  \setbeamercovered{transparent}

}

\usepackage[english]{babel}

\usepackage[latin1]{inputenc}

%\usepackage{times}

\usepackage[T1]{fontenc}

% Taken from Fernando's slides.

\usepackage{ae,aecompl}

\usepackage{mathpazo,courier,euler}

\usepackage[scaled=.95]{helvet}

\usepackage{amsmath}

\definecolor{darkgreen}{rgb}{0,0.5,0}

\usepackage{listings}

\lstset{language=Python,

    basicstyle=\ttfamily\bfseries,

    commentstyle=\color{red}\itshape,

  stringstyle=\color{darkgreen},

  showstringspaces=false,

  keywordstyle=\color{blue}\bfseries}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Macros

\setbeamercolor{emphbar}{bg=blue!20, fg=black}

\newcommand{\emphbar}[1]

{\begin{beamercolorbox}[rounded=true]{emphbar} 

      {#1}

 \end{beamercolorbox}

}

\newcounter{time}

\setcounter{time}{0}

\newcommand{\inctime}[1]{\addtocounter{time}{#1}{\tiny \thetime\ m}}

\newcommand{\typ}[1]{\lstinline{#1}}

\newcommand{\kwrd}[1]{ \texttt{\textbf{\color{blue}{#1}}}  }

%%% This is from Fernando's setup.

% \usepackage{color}

% \definecolor{orange}{cmyk}{0,0.4,0.8,0.2}

% % Use and configure listings package for nicely formatted code

% \usepackage{listings}

% \lstset{

%    language=Python,

%    basicstyle=\small\ttfamily,

%    commentstyle=\ttfamily\color{blue},

%    stringstyle=\ttfamily\color{orange},

%    showstringspaces=false,

%    breaklines=true,

%    postbreak = \space\dots

% }

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Title page

\title[Statistics]{Python for Science and Engg: Statistics}

\author[FOSSEE] {FOSSEE}

\institute[IIT Bombay] {Department of Aerospace Engineering\\IIT Bombay}

\date[] {31, October 2009\\Day 1, Session 3}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%\pgfdeclareimage[height=0.75cm]{iitmlogo}{iitmlogo}

%\logo{\pgfuseimage{iitmlogo}}

%% Delete this, if you do not want the table of contents to pop up at

%% the beginning of each subsection:

\AtBeginSubsection[]

{

  \begin{frame}<beamer>

    \frametitle{Outline}

    \tableofcontents[currentsection,currentsubsection]

  \end{frame}

}

\AtBeginSection[]

{

  \begin{frame}<beamer>

    \frametitle{Outline}

    \tableofcontents[currentsection,currentsubsection]

  \end{frame}

}

\newcommand{\num}{\texttt{numpy}}

% If you wish to uncover everything in a step-wise fashion, uncomment

% the following command: 

%\beamerdefaultoverlayspecification{<+->}

%\includeonlyframes{current,current1,current2,current3,current4,current5,current6}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% DOCUMENT STARTS

\begin{document}

\begin{frame}

  \maketitle

\end{frame}

%% \begin{frame}

%%   \frametitle{Outline}

%%   \tableofcontents

%%   % You might wish to add the option [pausesections]

%% \end{frame}

\section{Processing voluminous data}

\begin{frame}

  \frametitle{More on data processing}

  \begin{block}{}

    We have a huge--1m records--data file.\\How do we do \emph{efficient} statistical computations, that is find mean, median, mode, standard deveiation etc; draw pie charts?

  \end{block}

\end{frame}

\begin{frame}

  \frametitle{Statistical Analysis: Problem statement}

  Read the data supplied in \emph{sslc1.txt} and carry out the following:

  \begin{enumerate}

    \item Draw a pie chart representing the proportion of students who scored more than 90\% in each region in Science.

    \item Draw a pie chart representing the proportion of students who scored more than 90\% in each subject across regions.

    \item Print mean, median, mode and standard deviation of math scores for all regions combined.

  \end{enumerate}

\end{frame}

\begin{frame}

  \frametitle{Problem statement: explanation}

    \emphbar{Draw a pie chart representing the proportion of students who scored more than 90\% in each region in Science.}

    \begin{enumerate}

      \item Complete(100\%) data - Number of students who scored more than 90\% in Science

      \item Each slice - Number of students who scored more than 90\% in Science in one region

    \end{enumerate}

\end{frame}

\begin{frame}

  \frametitle{Problem statement: explanation}

    \emphbar{Draw a pie chart representing the proportion of students who scored more than 90\% in each subject across regions.}

    \begin{enumerate}

      \item Complete(100\%) data - Number of students who scored more than 90\% across all regions

      \item Each slice - Number of students who scored more than 90\% in each subject across all regions

    \end{enumerate}

\end{frame}

\begin{frame}

  \frametitle{Statistical Analysis and Parsing \ldots}

  Machinery Required -

  \begin{itemize}

    \item File reading

    \item Parsing

    \item Dictionaries

    \item NumPy arrays

    \item Statistical operations

  \end{itemize}

\end{frame}

\begin{frame}

  \frametitle{File reading and parsing}

  Understanding the structure of sslc1.txt

  \begin{itemize}

    \item One line in file corresponds to a student's details

    \item aka record

    \item Each record consists of fields separated by ';'

  \end{itemize}

\end{frame}

\begin{frame}

  \frametitle{File reading and parsing \ldots}

  Each record consists of:

  \begin{itemize}

    \item Region Code

    \item Roll Number

    \item Name

    \item Marks of 5 subjects: English, Hindi, Maths, Science, Social

    \item Total marks

    \item Pass/Fail (P/F)

    \item Withdrawn (W)

  \end{itemize}

  \inctime{5}

\end{frame}

\subsection{Data processing}

\begin{frame}[fragile]

  \frametitle{File reading and parsing \ldots}

  \begin{lstlisting}

for record in open('sslc1.txt'):

    fields = record.split(';')

  \end{lstlisting}

\end{frame}

\subsection{Dictionary}

\begin{frame}[fragile]

  \frametitle{Dictionary: Introduction}

  \begin{itemize}

    \item lists index: 0 \ldots n

    \item dictionaries index using strings

  \end{itemize}

  \begin{block}{Example}

d = \{ ``Hitchhiker's guide'' : 42,

     ``Terminator'' : ``I'll be back''\}\\

d[``Terminator''] => ``I'll be back''

  \end{block}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Dictionary: Introduction}

  \begin{lstlisting}

In [1]: d = {"Hitchhiker's guide" : 42,

      "Terminator" : "I'll be back"}

In [2]: d["Hitchhiker's guide"]

Out[2]: 42

In [3]: "Hitchhiker's guide" in d

Out[3]: True

In [4]: "Guido" in d

Out[4]: False

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Dictionary: Introduction}

  \begin{lstlisting}

In [5]: d.keys()

Out[5]: ['Terminator', "Hitchhiker's 

                              guide"]

In [6]: d.values()

Out[6]: ["I'll be back", 42]

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Back to lists: Iterating}

  \begin{itemize}

    \item Python's \kwrd{for} loop iterates through list items

    \item In other languages (C/C++) we run through indices and pick items from the array using these indices

    \item In Python, while iterating through list items current position is not available

  \end{itemize}

  \begin{block}{Iterating through indices}

    What if we want the index of an item of a list?

  \end{block}

\end{frame}

\begin{frame}[fragile]

  \frametitle{enumerate: Iterating through list indices}

  \begin{lstlisting}

In [1]: names = ["Guido","Alex", "Tim"]

In [2]: for i, name in enumerate(names):

   ...:     print i, name

   ...: 

0 Guido

1 Alex

2 Tim

  \end{lstlisting}

  \inctime{5}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Continuing with our Dictionary}

  Let our dictionary be:

  \begin{lstlisting}

science = {} # is an empty dictionary

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Dictionary - Building parsed data}

  \begin{itemize}

    \item \emph{Keys} of \emph{science} will be region codes

    \item Value of a \emph{science} will be the number students who scored more than 90\% in that region

  \end{itemize}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Building parsed data \ldots}

  \begin{lstlisting}

from pylab import pie

science = {}

for record in open('sslc1.txt'):

    record = record.strip()

    fields = record.split(';')

    region_code = fields[0].strip()

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Building parsed data \ldots}

  \begin{lstlisting}

if region_code not in science:

    science[region_code] = 0

score_str = fields[4].strip()

score = int(score_str) if

    score_str != 'AA' else 0

if score > 90:

    science[region_code] += 1

  \end{lstlisting}

\end{frame}

\subsection{Visualizing data}

\begin{frame}[fragile]

  \frametitle{Pie charts}

  \small

  \begin{lstlisting}

figure(1)

pie(science.values(), 

    labels=science.keys())

title('Students scoring 90% and above 

      in science by region')

savefig('/tmp/science.png')

  \end{lstlisting}

\begin{columns}

    \column{5.25\textwidth}

    \hspace*{1.1in}

\includegraphics[height=2in, interpolate=true]{data/science}

    \column{0.8\textwidth}

\end{columns}

  \inctime{5}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Building data for all subjects \ldots}

  \begin{lstlisting}

from pylab import pie

from scipy import mean, median, std

from scipy import stats

scores = [[], [], [], [], []]

ninety_percents = [{}, {}, {}, {}, {}]

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Building data for all subjects \ldots}

  \begin{lstlisting}

for record in open('sslc1.txt'):

    record = record.strip()

    fields = record.split(';')

    region_code = fields[0].strip()

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Building data for all subjects \ldots}

  \small

  \begin{lstlisting}

for i, field in enumerate(fields[3:8]):

    if region_code not in ninety_percents[i]:

        ninety_percents[i][region_code] = 0

    score_str = field.strip()

    score = int(score_str) if

      score_str != 'AA' else 0

    scores[i].append(score)

    if score > 90:

        ninety_percents[i][region_code] += 1

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Consolidating data}

  \begin{lstlisting}

subj_total = []

for subject in ninety_percents:

    subj_total.append(sum(

         subject.values()))

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Pie charts}

  \begin{lstlisting}

figure(2)

pie(subj_total, labels=['English',

    'Hindi', 'Maths', 'Science',

    'Social'])

title('Students scoring more than

      90% by subject(All regions

      combined).')

savefig('/tmp/all_regions.png')

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Pie charts}

  \includegraphics[height=3in, interpolate=true]{data/all_regions}

\end{frame}

\subsection{Obtaining statistics}

\begin{frame}[fragile]

  \frametitle{Obtaining statistics}

  \begin{block}{Statistics: Mean}

    Obtain the mean of Math scores

  \end{block}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Obtaining statistics: Solution}

  \begin{block}{Statistics: Mean}

    Obtain the mean of Math scores

  \end{block}

  \begin{lstlisting}

math_scores = scores[2]

total = 0

for i, score in enumerate(math_scores):

    total += score

mean = total / (i + 1)

print "Mean: ", mean

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Obtaining statistics: Another solution}

  \begin{block}{Statistics: Mean}

    Obtain the mean of Math scores

  \end{block}

  \begin{lstlisting}

math_scores = scores[2]

mean = sum(math_scores) /

          len(math_scores)

  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

\frametitle{NumPy arrays}

  \begin{itemize}

    \item NumPy provides arrays

    \item arrays are very efficient and powerful 

    \item Very easy to perform element-wise operations - \typ{+, -, *, /, \%}

    \begin{lstlisting}

In [1]: a = array([1, 2, 3])

In [2]: b = array([4, 5, 6])

In [3]: a + b

Out[3]: array([5, 7, 9])

    \end{lstlisting}

    \item Very easy to compute statistics

  \end{itemize}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Obtaining statistics}

  \begin{lstlisting}

math_scores = array(scores[2])

print "Mean: ", mean(math_scores)

print "Median: ", median(math_scores)

print "Mode: ", stats.mode(math_scores)

print "Standard Deviation: ",

              std(math_scores)

  \end{lstlisting}

  \inctime{15}

\end{frame}

\begin{frame}[fragile]

  \frametitle{What tools did we use?}

  \begin{itemize}

   \item Dictionaries for storing data

   \item Facilities for drawing pie charts