1 #!/usr/bin/python

     2 #

     3 #             Perforce Defect Tracking Integration Project

     4 #              <http://www.ravenbrook.com/project/p4dti/>

     5 #

     6 #                   COVERAGE.PY -- COVERAGE TESTING

     7 #

     8 #             Gareth Rees, Ravenbrook Limited, 2001-12-04

     9 #                     Ned Batchelder, 2004-12-12

    10 #         http://nedbatchelder.com/code/modules/coverage.html

    11 #

    12 #

    13 # 1. INTRODUCTION

    14 #

    15 # This module provides coverage testing for Python code.

    16 #

    17 # The intended readership is all Python developers.

    18 #

    19 # This document is not confidential.

    20 #

    21 # See [GDR 2001-12-04a] for the command-line interface, programmatic

    22 # interface and limitations.  See [GDR 2001-12-04b] for requirements and

    23 # design.

    24 

    25 r"""Usage:

    26 

    27 coverage.py -x [-p] MODULE.py [ARG1 ARG2 ...]

    28     Execute module, passing the given command-line arguments, collecting

    29     coverage data. With the -p option, write to a temporary file containing

    30     the machine name and process ID.

    31 

    32 coverage.py -e

    33     Erase collected coverage data.

    34 

    35 coverage.py -c

    36     Collect data from multiple coverage files (as created by -p option above)

    37     and store it into a single file representing the union of the coverage.

    38 

    39 coverage.py -r [-m] [-o dir1,dir2,...] FILE1 FILE2 ...

    40     Report on the statement coverage for the given files.  With the -m

    41     option, show line numbers of the statements that weren't executed.

    42 

    43 coverage.py -a [-d dir] [-o dir1,dir2,...] FILE1 FILE2 ...

    44     Make annotated copies of the given files, marking statements that

    45     are executed with > and statements that are missed with !.  With

    46     the -d option, make the copies in that directory.  Without the -d

    47     option, make each copy in the same directory as the original.

    48 

    49 -o dir,dir2,...

    50   Omit reporting or annotating files when their filename path starts with

    51   a directory listed in the omit list.

    52   e.g. python coverage.py -i -r -o c:\python23,lib\enthought\traits

    53 

    54 Coverage data is saved in the file .coverage by default.  Set the

    55 COVERAGE_FILE environment variable to save it somewhere else."""

    56 

    57 __version__ = "2.85.20080914"    # see detailed history at the end of this file.

    58 

    59 import compiler

    60 import compiler.visitor

    61 import glob

    62 import os

    63 import re

    64 import string

    65 import symbol

    66 import sys

    67 import threading

    68 import token

    69 import types

    70 import zipimport

    71 from socket import gethostname

    72 

    73 # Python version compatibility

    74 try:

    75     strclass = basestring   # new to 2.3

    76 except:

    77     strclass = str

    78 

    79 # 2. IMPLEMENTATION

    80 #

    81 # This uses the "singleton" pattern.

    82 #

    83 # The word "morf" means a module object (from which the source file can

    84 # be deduced by suitable manipulation of the __file__ attribute) or a

    85 # filename.

    86 #

    87 # When we generate a coverage report we have to canonicalize every

    88 # filename in the coverage dictionary just in case it refers to the

    89 # module we are reporting on.  It seems a shame to throw away this

    90 # information so the data in the coverage dictionary is transferred to

    91 # the 'cexecuted' dictionary under the canonical filenames.

    92 #

    93 # The coverage dictionary is called "c" and the trace function "t".  The

    94 # reason for these short names is that Python looks up variables by name

    95 # at runtime and so execution time depends on the length of variables!

    96 # In the bottleneck of this application it's appropriate to abbreviate

    97 # names to increase speed.

    98 

    99 class StatementFindingAstVisitor(compiler.visitor.ASTVisitor):

   100     """ A visitor for a parsed Abstract Syntax Tree which finds executable

   101         statements.

   102     """

   103     def __init__(self, statements, excluded, suite_spots):

   104         compiler.visitor.ASTVisitor.__init__(self)

   105         self.statements = statements

   106         self.excluded = excluded

   107         self.suite_spots = suite_spots

   108         self.excluding_suite = 0

   109         

   110     def doRecursive(self, node):

   111         for n in node.getChildNodes():

   112             self.dispatch(n)

   113 

   114     visitStmt = visitModule = doRecursive

   115     

   116     def doCode(self, node):

   117         if hasattr(node, 'decorators') and node.decorators:

   118             self.dispatch(node.decorators)

   119             self.recordAndDispatch(node.code)

   120         else:

   121             self.doSuite(node, node.code)

   122             

   123     visitFunction = visitClass = doCode

   124 

   125     def getFirstLine(self, node):

   126         # Find the first line in the tree node.

   127         lineno = node.lineno

   128         for n in node.getChildNodes():

   129             f = self.getFirstLine(n)

   130             if lineno and f:

   131                 lineno = min(lineno, f)

   132             else:

   133                 lineno = lineno or f

   134         return lineno

   135 

   136     def getLastLine(self, node):

   137         # Find the first line in the tree node.

   138         lineno = node.lineno

   139         for n in node.getChildNodes():

   140             lineno = max(lineno, self.getLastLine(n))

   141         return lineno

   142     

   143     def doStatement(self, node):

   144         self.recordLine(self.getFirstLine(node))

   145 

   146     visitAssert = visitAssign = visitAssTuple = visitPrint = \

   147         visitPrintnl = visitRaise = visitSubscript = visitDecorators = \

   148         doStatement

   149     

   150     def visitPass(self, node):

   151         # Pass statements have weird interactions with docstrings.  If this

   152         # pass statement is part of one of those pairs, claim that the statement

   153         # is on the later of the two lines.

   154         l = node.lineno

   155         if l:

   156             lines = self.suite_spots.get(l, [l,l])

   157             self.statements[lines[1]] = 1

   158         

   159     def visitDiscard(self, node):

   160         # Discard nodes are statements that execute an expression, but then

   161         # discard the results.  This includes function calls, so we can't 

   162         # ignore them all.  But if the expression is a constant, the statement

   163         # won't be "executed", so don't count it now.

   164         if node.expr.__class__.__name__ != 'Const':

   165             self.doStatement(node)

   166 

   167     def recordNodeLine(self, node):

   168         # Stmt nodes often have None, but shouldn't claim the first line of

   169         # their children (because the first child might be an ignorable line

   170         # like "global a").

   171         if node.__class__.__name__ != 'Stmt':

   172             return self.recordLine(self.getFirstLine(node))

   173         else:

   174             return 0

   175     

   176     def recordLine(self, lineno):

   177         # Returns a bool, whether the line is included or excluded.

   178         if lineno:

   179             # Multi-line tests introducing suites have to get charged to their

   180             # keyword.

   181             if lineno in self.suite_spots:

   182                 lineno = self.suite_spots[lineno][0]

   183             # If we're inside an excluded suite, record that this line was

   184             # excluded.

   185             if self.excluding_suite:

   186                 self.excluded[lineno] = 1

   187                 return 0

   188             # If this line is excluded, or suite_spots maps this line to

   189             # another line that is exlcuded, then we're excluded.

   190             elif self.excluded.has_key(lineno) or \

   191                  self.suite_spots.has_key(lineno) and \

   192                  self.excluded.has_key(self.suite_spots[lineno][1]):

   193                 return 0

   194             # Otherwise, this is an executable line.

   195             else:

   196                 self.statements[lineno] = 1

   197                 return 1

   198         return 0

   199     

   200     default = recordNodeLine

   201     

   202     def recordAndDispatch(self, node):

   203         self.recordNodeLine(node)

   204         self.dispatch(node)

   205 

   206     def doSuite(self, intro, body, exclude=0):

   207         exsuite = self.excluding_suite

   208         if exclude or (intro and not self.recordNodeLine(intro)):

   209             self.excluding_suite = 1

   210         self.recordAndDispatch(body)

   211         self.excluding_suite = exsuite

   212         

   213     def doPlainWordSuite(self, prevsuite, suite):

   214         # Finding the exclude lines for else's is tricky, because they aren't

   215         # present in the compiler parse tree.  Look at the previous suite,

   216         # and find its last line.  If any line between there and the else's

   217         # first line are excluded, then we exclude the else.

   218         lastprev = self.getLastLine(prevsuite)

   219         firstelse = self.getFirstLine(suite)

   220         for l in range(lastprev+1, firstelse):

   221             if self.suite_spots.has_key(l):

   222                 self.doSuite(None, suite, exclude=self.excluded.has_key(l))

   223                 break

   224         else:

   225             self.doSuite(None, suite)

   226         

   227     def doElse(self, prevsuite, node):

   228         if node.else_:

   229             self.doPlainWordSuite(prevsuite, node.else_)

   230     

   231     def visitFor(self, node):

   232         self.doSuite(node, node.body)

   233         self.doElse(node.body, node)

   234 

   235     visitWhile = visitFor

   236 

   237     def visitIf(self, node):

   238         # The first test has to be handled separately from the rest.

   239         # The first test is credited to the line with the "if", but the others

   240         # are credited to the line with the test for the elif.

   241         self.doSuite(node, node.tests[0][1])

   242         for t, n in node.tests[1:]:

   243             self.doSuite(t, n)

   244         self.doElse(node.tests[-1][1], node)

   245 

   246     def visitTryExcept(self, node):

   247         self.doSuite(node, node.body)

   248         for i in range(len(node.handlers)):

   249             a, b, h = node.handlers[i]

   250             if not a:

   251                 # It's a plain "except:".  Find the previous suite.

   252                 if i > 0:

   253                     prev = node.handlers[i-1][2]

   254                 else:

   255                     prev = node.body

   256                 self.doPlainWordSuite(prev, h)

   257             else:

   258                 self.doSuite(a, h)

   259         self.doElse(node.handlers[-1][2], node)

   260     

   261     def visitTryFinally(self, node):

   262         self.doSuite(node, node.body)

   263         self.doPlainWordSuite(node.body, node.final)

   264         

   265     def visitWith(self, node):

   266         self.doSuite(node, node.body)

   267         

   268     def visitGlobal(self, node):

   269         # "global" statements don't execute like others (they don't call the

   270         # trace function), so don't record their line numbers.

   271         pass

   272 

   273 the_coverage = None

   274 

   275 class CoverageException(Exception):

   276     pass

   277 

   278 class coverage:

   279     # Name of the cache file (unless environment variable is set).

   280     cache_default = ".coverage"

   281 

   282     # Environment variable naming the cache file.

   283     cache_env = "COVERAGE_FILE"

   284 

   285     # A dictionary with an entry for (Python source file name, line number

   286     # in that file) if that line has been executed.

   287     c = {}

   288     

   289     # A map from canonical Python source file name to a dictionary in

   290     # which there's an entry for each line number that has been

   291     # executed.

   292     cexecuted = {}

   293 

   294     # Cache of results of calling the analysis2() method, so that you can

   295     # specify both -r and -a without doing double work.

   296     analysis_cache = {}

   297 

   298     # Cache of results of calling the canonical_filename() method, to

   299     # avoid duplicating work.

   300     canonical_filename_cache = {}

   301 

   302     def __init__(self):

   303         global the_coverage

   304         if the_coverage:

   305             raise CoverageException("Only one coverage object allowed.")

   306         self.usecache = 1

   307         self.cache = None

   308         self.parallel_mode = False

   309         self.exclude_re = ''

   310         self.nesting = 0

   311         self.cstack = []

   312         self.xstack = []

   313         self.relative_dir = self.abs_file(os.curdir)+os.sep

   314         self.exclude('# *pragma[: ]*[nN][oO] *[cC][oO][vV][eE][rR]')

   315 

   316     # t(f, x, y).  This method is passed to sys.settrace as a trace function.  

   317     # See [van Rossum 2001-07-20b, 9.2] for an explanation of sys.settrace and 

   318     # the arguments and return value of the trace function.

   319     # See [van Rossum 2001-07-20a, 3.2] for a description of frame and code

   320     # objects.

   321     

   322     def t(self, f, w, unused):                                 #pragma: no cover

   323         if w == 'line':

   324             self.c[(f.f_code.co_filename, f.f_lineno)] = 1

   325             #-for c in self.cstack:

   326             #-    c[(f.f_code.co_filename, f.f_lineno)] = 1

   327         return self.t

   328     

   329     def help(self, error=None):     #pragma: no cover

   330         if error:

   331             print error

   332             print

   333         print __doc__

   334         sys.exit(1)

   335 

   336     def command_line(self, argv, help_fn=None):

   337         import getopt

   338         help_fn = help_fn or self.help

   339         settings = {}

   340         optmap = {

   341             '-a': 'annotate',

   342             '-c': 'collect',

   343             '-d:': 'directory=',

   344             '-e': 'erase',

   345             '-h': 'help',

   346             '-i': 'ignore-errors',

   347             '-m': 'show-missing',

   348             '-p': 'parallel-mode',

   349             '-r': 'report',

   350             '-x': 'execute',

   351             '-o:': 'omit=',

   352             }

   353         short_opts = string.join(map(lambda o: o[1:], optmap.keys()), '')

   354         long_opts = optmap.values()

   355         options, args = getopt.getopt(argv, short_opts, long_opts)

   356         for o, a in options:

   357             if optmap.has_key(o):

   358                 settings[optmap[o]] = 1

   359             elif optmap.has_key(o + ':'):

   360                 settings[optmap[o + ':']] = a

   361             elif o[2:] in long_opts:

   362                 settings[o[2:]] = 1

   363             elif o[2:] + '=' in long_opts:

   364                 settings[o[2:]+'='] = a

   365             else:       #pragma: no cover

   366                 pass    # Can't get here, because getopt won't return anything unknown.

   367 

   368         if settings.get('help'):

   369             help_fn()

   370 

   371         for i in ['erase', 'execute']:

   372             for j in ['annotate', 'report', 'collect']:

   373                 if settings.get(i) and settings.get(j):

   374                     help_fn("You can't specify the '%s' and '%s' "

   375                               "options at the same time." % (i, j))

   376 

   377         args_needed = (settings.get('execute')

   378                        or settings.get('annotate')

   379                        or settings.get('report'))

   380         action = (settings.get('erase') 

   381                   or settings.get('collect')

   382                   or args_needed)

   383         if not action:

   384             help_fn("You must specify at least one of -e, -x, -c, -r, or -a.")

   385         if not args_needed and args:

   386             help_fn("Unexpected arguments: %s" % " ".join(args))

   387         

   388         self.parallel_mode = settings.get('parallel-mode')

   389         self.get_ready()

   390 

   391         if settings.get('erase'):

   392             self.erase()

   393         if settings.get('execute'):

   394             if not args:

   395                 help_fn("Nothing to do.")

   396             sys.argv = args

   397             self.start()

   398             import __main__

   399             sys.path[0] = os.path.dirname(sys.argv[0])

   400             execfile(sys.argv[0], __main__.__dict__)

   401         if settings.get('collect'):

   402             self.collect()

   403         if not args:

   404             args = self.cexecuted.keys()

   405         

   406         ignore_errors = settings.get('ignore-errors')

   407         show_missing = settings.get('show-missing')

   408         directory = settings.get('directory=')

   409 

   410         omit = settings.get('omit=')

   411         if omit is not None:

   412             omit = [self.abs_file(p) for p in omit.split(',')]

   413         else:

   414             omit = []

   415         

   416         if settings.get('report'):

   417             self.report(args, show_missing, ignore_errors, omit_prefixes=omit)

   418         if settings.get('annotate'):

   419             self.annotate(args, directory, ignore_errors, omit_prefixes=omit)

   420 

   421     def use_cache(self, usecache, cache_file=None):

   422         self.usecache = usecache

   423         if cache_file and not self.cache:

   424             self.cache_default = cache_file

   425         

   426     def get_ready(self, parallel_mode=False):

   427         if self.usecache and not self.cache:

   428             self.cache = os.environ.get(self.cache_env, self.cache_default)

   429             if self.parallel_mode:

   430                 self.cache += "." + gethostname() + "." + str(os.getpid())

   431             self.restore()

   432         self.analysis_cache = {}

   433         

   434     def start(self, parallel_mode=False):

   435         self.get_ready()

   436         if self.nesting == 0:                               #pragma: no cover

   437             sys.settrace(self.t)

   438             if hasattr(threading, 'settrace'):

   439                 threading.settrace(self.t)

   440         self.nesting += 1

   441         

   442     def stop(self):

   443         self.nesting -= 1

   444         if self.nesting == 0:                               #pragma: no cover

   445             sys.settrace(None)

   446             if hasattr(threading, 'settrace'):

   447                 threading.settrace(None)

   448 

   449     def erase(self):

   450         self.get_ready()

   451         self.c = {}

   452         self.analysis_cache = {}

   453         self.cexecuted = {}

   454         if self.cache and os.path.exists(self.cache):

   455             os.remove(self.cache)

   456 

   457     def exclude(self, re):

   458         if self.exclude_re:

   459             self.exclude_re += "|"

   460         self.exclude_re += "(" + re + ")"

   461 

   462     def begin_recursive(self):

   463         self.cstack.append(self.c)

   464         self.xstack.append(self.exclude_re)

   465         

   466     def end_recursive(self):

   467         self.c = self.cstack.pop()

   468         self.exclude_re = self.xstack.pop()

   469 

   470     # save().  Save coverage data to the coverage cache.

   471 

   472     def save(self):

   473         if self.usecache and self.cache:

   474             self.canonicalize_filenames()

   475             cache = open(self.cache, 'wb')

   476             import marshal

   477             marshal.dump(self.cexecuted, cache)

   478             cache.close()

   479 

   480     # restore().  Restore coverage data from the coverage cache (if it exists).

   481 

   482     def restore(self):

   483         self.c = {}

   484         self.cexecuted = {}

   485         assert self.usecache

   486         if os.path.exists(self.cache):

   487             self.cexecuted = self.restore_file(self.cache)

   488 

   489     def restore_file(self, file_name):

   490         try:

   491             cache = open(file_name, 'rb')

   492             import marshal

   493             cexecuted = marshal.load(cache)

   494             cache.close()

   495             if isinstance(cexecuted, types.DictType):

   496                 return cexecuted

   497             else:

   498                 return {}

   499         except:

   500             return {}

   501 

   502     # collect(). Collect data in multiple files produced by parallel mode

   503 

   504     def collect(self):

   505         cache_dir, local = os.path.split(self.cache)

   506         for f in os.listdir(cache_dir or '.'):

   507             if not f.startswith(local):

   508                 continue

   509 

   510             full_path = os.path.join(cache_dir, f)

   511             cexecuted = self.restore_file(full_path)

   512             self.merge_data(cexecuted)

   513 

   514     def merge_data(self, new_data):

   515         for file_name, file_data in new_data.items():

   516             if self.cexecuted.has_key(file_name):

   517                 self.merge_file_data(self.cexecuted[file_name], file_data)

   518             else:

   519                 self.cexecuted[file_name] = file_data

   520 

   521     def merge_file_data(self, cache_data, new_data):

   522         for line_number in new_data.keys():

   523             if not cache_data.has_key(line_number):

   524                 cache_data[line_number] = new_data[line_number]

   525 

   526     def abs_file(self, filename):

   527         """ Helper function to turn a filename into an absolute normalized

   528             filename.

   529         """

   530         return os.path.normcase(os.path.abspath(os.path.realpath(filename)))

   531 

   532     def get_zip_data(self, filename):

   533         """ Get data from `filename` if it is a zip file path, or return None

   534             if it is not.

   535         """

   536         markers = ['.zip'+os.sep, '.egg'+os.sep]

   537         for marker in markers:

   538             if marker in filename:

   539                 parts = filename.split(marker)

   540                 try:

   541                     zi = zipimport.zipimporter(parts[0]+marker[:-1])

   542                 except zipimport.ZipImportError:

   543                     continue

   544                 try:

   545                     data = zi.get_data(parts[1])

   546                 except IOError:

   547                     continue

   548                 return data

   549         return None

   550 

   551     # canonical_filename(filename).  Return a canonical filename for the

   552     # file (that is, an absolute path with no redundant components and

   553     # normalized case).  See [GDR 2001-12-04b, 3.3].

   554 

   555     def canonical_filename(self, filename):

   556         if not self.canonical_filename_cache.has_key(filename):

   557             f = filename

   558             if os.path.isabs(f) and not os.path.exists(f):

   559                 if not self.get_zip_data(f):

   560                     f = os.path.basename(f)

   561             if not os.path.isabs(f):

   562                 for path in [os.curdir] + sys.path:

   563                     g = os.path.join(path, f)

   564                     if os.path.exists(g):

   565                         f = g

   566                         break

   567             cf = self.abs_file(f)

   568             self.canonical_filename_cache[filename] = cf

   569         return self.canonical_filename_cache[filename]

   570 

   571     # canonicalize_filenames().  Copy results from "c" to "cexecuted", 

   572     # canonicalizing filenames on the way.  Clear the "c" map.

   573 

   574     def canonicalize_filenames(self):

   575         for filename, lineno in self.c.keys():

   576             if filename == '<string>':

   577                 # Can't do anything useful with exec'd strings, so skip them.

   578                 continue

   579             f = self.canonical_filename(filename)

   580             if not self.cexecuted.has_key(f):

   581                 self.cexecuted[f] = {}

   582             self.cexecuted[f][lineno] = 1

   583         self.c = {}

   584 

   585     # morf_filename(morf).  Return the filename for a module or file.

   586 

   587     def morf_filename(self, morf):

   588         if hasattr(morf, '__file__'):

   589             f = morf.__file__

   590         else:

   591             f = morf

   592         return self.canonical_filename(f)

   593 

   594     # analyze_morf(morf).  Analyze the module or filename passed as

   595     # the argument.  If the source code can't be found, raise an error.

   596     # Otherwise, return a tuple of (1) the canonical filename of the

   597     # source code for the module, (2) a list of lines of statements

   598     # in the source code, (3) a list of lines of excluded statements,

   599     # and (4), a map of line numbers to multi-line line number ranges, for

   600     # statements that cross lines.

   601     

   602     def analyze_morf(self, morf):

   603         if self.analysis_cache.has_key(morf):

   604             return self.analysis_cache[morf]

   605         filename = self.morf_filename(morf)

   606         ext = os.path.splitext(filename)[1]

   607         source, sourcef = None, None

   608         if ext == '.pyc':

   609             if not os.path.exists(filename[:-1]):

   610                 source = self.get_zip_data(filename[:-1])

   611                 if not source:

   612                     raise CoverageException(

   613                         "No source for compiled code '%s'." % filename

   614                         )

   615             filename = filename[:-1]

   616         if not source:

   617             sourcef = open(filename, 'rU')

   618             source = sourcef.read()

   619         try:

   620             lines, excluded_lines, line_map = self.find_executable_statements(

   621                 source, exclude=self.exclude_re

   622                 )

   623         except SyntaxError, synerr:

   624             raise CoverageException(

   625                 "Couldn't parse '%s' as Python source: '%s' at line %d" %

   626                     (filename, synerr.msg, synerr.lineno)

   627                 )

   628         if sourcef:

   629             sourcef.close()

   630         result = filename, lines, excluded_lines, line_map

   631         self.analysis_cache[morf] = result

   632         return result

   633 

   634     def first_line_of_tree(self, tree):

   635         while True:

   636             if len(tree) == 3 and type(tree[2]) == type(1):

   637                 return tree[2]

   638             tree = tree[1]

   639     

   640     def last_line_of_tree(self, tree):

   641         while True:

   642             if len(tree) == 3 and type(tree[2]) == type(1):

   643                 return tree[2]

   644             tree = tree[-1]

   645     

   646     def find_docstring_pass_pair(self, tree, spots):

   647         for i in range(1, len(tree)):

   648             if self.is_string_constant(tree[i]) and self.is_pass_stmt(tree[i+1]):

   649                 first_line = self.first_line_of_tree(tree[i])

   650                 last_line = self.last_line_of_tree(tree[i+1])

   651                 self.record_multiline(spots, first_line, last_line)

   652         

   653     def is_string_constant(self, tree):

   654         try:

   655             return tree[0] == symbol.stmt and tree[1][1][1][0] == symbol.expr_stmt

   656         except:

   657             return False

   658         

   659     def is_pass_stmt(self, tree):

   660         try:

   661             return tree[0] == symbol.stmt and tree[1][1][1][0] == symbol.pass_stmt

   662         except:

   663             return False

   664 

   665     def record_multiline(self, spots, i, j):

   666         for l in range(i, j+1):

   667             spots[l] = (i, j)

   668             

   669     def get_suite_spots(self, tree, spots):

   670         """ Analyze a parse tree to find suite introducers which span a number

   671             of lines.

   672         """

   673         for i in range(1, len(tree)):

   674             if type(tree[i]) == type(()):

   675                 if tree[i][0] == symbol.suite:

   676                     # Found a suite, look back for the colon and keyword.

   677                     lineno_colon = lineno_word = None

   678                     for j in range(i-1, 0, -1):

   679                         if tree[j][0] == token.COLON:

   680                             # Colons are never executed themselves: we want the

   681                             # line number of the last token before the colon.

   682                             lineno_colon = self.last_line_of_tree(tree[j-1])

   683                         elif tree[j][0] == token.NAME:

   684                             if tree[j][1] == 'elif':

   685                                 # Find the line number of the first non-terminal

   686                                 # after the keyword.

   687                                 t = tree[j+1]

   688                                 while t and token.ISNONTERMINAL(t[0]):

   689                                     t = t[1]

   690                                 if t:

   691                                     lineno_word = t[2]

   692                             else:

   693                                 lineno_word = tree[j][2]

   694                             break

   695                         elif tree[j][0] == symbol.except_clause:

   696                             # "except" clauses look like:

   697                             # ('except_clause', ('NAME', 'except', lineno), ...)

   698                             if tree[j][1][0] == token.NAME:

   699                                 lineno_word = tree[j][1][2]

   700                                 break

   701                     if lineno_colon and lineno_word:

   702                         # Found colon and keyword, mark all the lines

   703                         # between the two with the two line numbers.

   704                         self.record_multiline(spots, lineno_word, lineno_colon)

   705 

   706                     # "pass" statements are tricky: different versions of Python

   707                     # treat them differently, especially in the common case of a

   708                     # function with a doc string and a single pass statement.

   709                     self.find_docstring_pass_pair(tree[i], spots)

   710                     

   711                 elif tree[i][0] == symbol.simple_stmt:

   712                     first_line = self.first_line_of_tree(tree[i])

   713                     last_line = self.last_line_of_tree(tree[i])

   714                     if first_line != last_line:

   715                         self.record_multiline(spots, first_line, last_line)

   716                 self.get_suite_spots(tree[i], spots)

   717 

   718     def find_executable_statements(self, text, exclude=None):

   719         # Find lines which match an exclusion pattern.

   720         excluded = {}

   721         suite_spots = {}

   722         if exclude:

   723             reExclude = re.compile(exclude)

   724             lines = text.split('\n')

   725             for i in range(len(lines)):

   726                 if reExclude.search(lines[i]):

   727                     excluded[i+1] = 1

   728 

   729         # Parse the code and analyze the parse tree to find out which statements

   730         # are multiline, and where suites begin and end.

   731         import parser

   732         tree = parser.suite(text+'\n\n').totuple(1)

   733         self.get_suite_spots(tree, suite_spots)

   734         #print "Suite spots:", suite_spots

   735         

   736         # Use the compiler module to parse the text and find the executable

   737         # statements.  We add newlines to be impervious to final partial lines.

   738         statements = {}

   739         ast = compiler.parse(text+'\n\n')

   740         visitor = StatementFindingAstVisitor(statements, excluded, suite_spots)

   741         compiler.walk(ast, visitor, walker=visitor)

   742 

   743         lines = statements.keys()

   744         lines.sort()

   745         excluded_lines = excluded.keys()

   746         excluded_lines.sort()

   747         return lines, excluded_lines, suite_spots

   748 

   749     # format_lines(statements, lines).  Format a list of line numbers

   750     # for printing by coalescing groups of lines as long as the lines

   751     # represent consecutive statements.  This will coalesce even if

   752     # there are gaps between statements, so if statements =

   753     # [1,2,3,4,5,10,11,12,13,14] and lines = [1,2,5,10,11,13,14] then

   754     # format_lines will return "1-2, 5-11, 13-14".

   755 

   756     def format_lines(self, statements, lines):

   757         pairs = []

   758         i = 0

   759         j = 0

   760         start = None

   761         pairs = []

   762         while i < len(statements) and j < len(lines):

   763             if statements[i] == lines[j]:

   764                 if start == None:

   765                     start = lines[j]

   766                 end = lines[j]

   767                 j = j + 1

   768             elif start:

   769                 pairs.append((start, end))

   770                 start = None

   771             i = i + 1

   772         if start:

   773             pairs.append((start, end))

   774         def stringify(pair):

   775             start, end = pair

   776             if start == end:

   777                 return "%d" % start

   778             else:

   779                 return "%d-%d" % (start, end)

   780         ret = string.join(map(stringify, pairs), ", ")

   781         return ret

   782 

   783     # Backward compatibility with version 1.

   784     def analysis(self, morf):

   785         f, s, _, m, mf = self.analysis2(morf)

   786         return f, s, m, mf

   787 

   788     def analysis2(self, morf):

   789         filename, statements, excluded, line_map = self.analyze_morf(morf)

   790         self.canonicalize_filenames()

   791         if not self.cexecuted.has_key(filename):

   792             self.cexecuted[filename] = {}

   793         missing = []

   794         for line in statements:

   795             lines = line_map.get(line, [line, line])

   796             for l in range(lines[0], lines[1]+1):

   797                 if self.cexecuted[filename].has_key(l):

   798                     break

   799             else:

   800                 missing.append(line)

   801         return (filename, statements, excluded, missing,

   802                 self.format_lines(statements, missing))

   803 

   804     def relative_filename(self, filename):

   805         """ Convert filename to relative filename from self.relative_dir.

   806         """

   807         return filename.replace(self.relative_dir, "")

   808 

   809     def morf_name(self, morf):

   810         """ Return the name of morf as used in report.

   811         """

   812         if hasattr(morf, '__name__'):

   813             return morf.__name__

   814         else:

   815             return self.relative_filename(os.path.splitext(morf)[0])

   816 

   817     def filter_by_prefix(self, morfs, omit_prefixes):

   818         """ Return list of morfs where the morf name does not begin

   819             with any one of the omit_prefixes.

   820         """

   821         filtered_morfs = []

   822         for morf in morfs:

   823             for prefix in omit_prefixes:

   824                 if self.morf_name(morf).startswith(prefix):

   825                     break

   826             else:

   827                 filtered_morfs.append(morf)

   828 

   829         return filtered_morfs

   830 

   831     def morf_name_compare(self, x, y):

   832         return cmp(self.morf_name(x), self.morf_name(y))

   833 

   834     def report(self, morfs, show_missing=1, ignore_errors=0, file=None, omit_prefixes=[]):

   835         if not isinstance(morfs, types.ListType):

   836             morfs = [morfs]

   837         # On windows, the shell doesn't expand wildcards.  Do it here.

   838         globbed = []

   839         for morf in morfs:

   840             if isinstance(morf, strclass):

   841                 globbed.extend(glob.glob(morf))

   842             else:

   843                 globbed.append(morf)

   844         morfs = globbed

   845         

   846         morfs = self.filter_by_prefix(morfs, omit_prefixes)

   847         morfs.sort(self.morf_name_compare)

   848 

   849         max_name = max([5,] + map(len, map(self.morf_name, morfs)))

   850         fmt_name = "%%- %ds  " % max_name

   851         fmt_err = fmt_name + "%s: %s"

   852         header = fmt_name % "Name" + " Stmts   Exec  Cover"

   853         fmt_coverage = fmt_name + "% 6d % 6d % 5d%%"

   854         if show_missing:

   855             header = header + "   Missing"

   856             fmt_coverage = fmt_coverage + "   %s"

   857         if not file:

   858             file = sys.stdout

   859         print >>file, header

   860         print >>file, "-" * len(header)

   861         total_statements = 0

   862         total_executed = 0

   863         for morf in morfs:

   864             name = self.morf_name(morf)

   865             try:

   866                 _, statements, _, missing, readable  = self.analysis2(morf)

   867                 n = len(statements)

   868                 m = n - len(missing)

   869                 if n > 0:

   870                     pc = 100.0 * m / n

   871                 else:

   872                     pc = 100.0

   873                 args = (name, n, m, pc)

   874                 if show_missing:

   875                     args = args + (readable,)

   876                 print >>file, fmt_coverage % args

   877                 total_statements = total_statements + n

   878                 total_executed = total_executed + m

   879             except KeyboardInterrupt:                       #pragma: no cover

   880                 raise

   881             except:

   882                 if not ignore_errors:

   883                     typ, msg = sys.exc_info()[:2]

   884                     print >>file, fmt_err % (name, typ, msg)

   885         if len(morfs) > 1:

   886             print >>file, "-" * len(header)

   887             if total_statements > 0:

   888                 pc = 100.0 * total_executed / total_statements

   889             else:

   890                 pc = 100.0

   891             args = ("TOTAL", total_statements, total_executed, pc)

   892             if show_missing:

   893                 args = args + ("",)

   894             print >>file, fmt_coverage % args

   895 

   896     # annotate(morfs, ignore_errors).

   897 

   898     blank_re = re.compile(r"\s*(#|$)")

   899     else_re = re.compile(r"\s*else\s*:\s*(#|$)")

   900 

   901     def annotate(self, morfs, directory=None, ignore_errors=0, omit_prefixes=[]):

   902         morfs = self.filter_by_prefix(morfs, omit_prefixes)

   903         for morf in morfs:

   904             try:

   905                 filename, statements, excluded, missing, _ = self.analysis2(morf)

   906                 self.annotate_file(filename, statements, excluded, missing, directory)

   907             except KeyboardInterrupt:

   908                 raise

   909             except:

   910                 if not ignore_errors:

   911                     raise

   912                 

   913     def annotate_file(self, filename, statements, excluded, missing, directory=None):

   914         source = open(filename, 'r')

   915         if directory:

   916             dest_file = os.path.join(directory,

   917                                      os.path.basename(filename)

   918                                      + ',cover')

   919         else:

   920             dest_file = filename + ',cover'

   921         dest = open(dest_file, 'w')

   922         lineno = 0

   923         i = 0

   924         j = 0

   925         covered = 1

   926         while 1:

   927             line = source.readline()

   928             if line == '':

   929                 break

   930             lineno = lineno + 1

   931             while i < len(statements) and statements[i] < lineno:

   932                 i = i + 1

   933             while j < len(missing) and missing[j] < lineno:

   934                 j = j + 1

   935             if i < len(statements) and statements[i] == lineno:

   936                 covered = j >= len(missing) or missing[j] > lineno

   937             if self.blank_re.match(line):

   938                 dest.write('  ')

   939             elif self.else_re.match(line):

   940                 # Special logic for lines containing only 'else:'.  

   941                 # See [GDR 2001-12-04b, 3.2].

   942                 if i >= len(statements) and j >= len(missing):

   943                     dest.write('! ')

   944                 elif i >= len(statements) or j >= len(missing):

   945                     dest.write('> ')

   946                 elif statements[i] == missing[j]:

   947                     dest.write('! ')

   948                 else:

   949                     dest.write('> ')

   950             elif lineno in excluded:

   951                 dest.write('- ')

   952             elif covered:

   953                 dest.write('> ')

   954             else:

   955                 dest.write('! ')

   956             dest.write(line)

   957         source.close()

   958         dest.close()

   959 

   960 # Singleton object.

   961 the_coverage = coverage()

   962 

   963 # Module functions call methods in the singleton object.

   964 def use_cache(*args, **kw): 

   965     return the_coverage.use_cache(*args, **kw)

   966 

   967 def start(*args, **kw): 

   968     return the_coverage.start(*args, **kw)

   969 

   970 def stop(*args, **kw): 

   971     return the_coverage.stop(*args, **kw)

   972 

   973 def erase(*args, **kw): 

   974     return the_coverage.erase(*args, **kw)

   975 

   976 def begin_recursive(*args, **kw): 

   977     return the_coverage.begin_recursive(*args, **kw)

   978 

   979 def end_recursive(*args, **kw): 

   980     return the_coverage.end_recursive(*args, **kw)

   981 

   982 def exclude(*args, **kw): 

   983     return the_coverage.exclude(*args, **kw)

   984 

   985 def analysis(*args, **kw): 

   986     return the_coverage.analysis(*args, **kw)

   987 

   988 def analysis2(*args, **kw): 

   989     return the_coverage.analysis2(*args, **kw)

   990 

   991 def report(*args, **kw): 

   992     return the_coverage.report(*args, **kw)

   993 

   994 def annotate(*args, **kw): 

   995     return the_coverage.annotate(*args, **kw)

   996 

   997 def annotate_file(*args, **kw): 

   998     return the_coverage.annotate_file(*args, **kw)

   999 

  1000 # Save coverage data when Python exits.  (The atexit module wasn't

  1001 # introduced until Python 2.0, so use sys.exitfunc when it's not

  1002 # available.)

  1003 try:

  1004     import atexit

  1005     atexit.register(the_coverage.save)

  1006 except ImportError:

  1007     sys.exitfunc = the_coverage.save

  1008 

  1009 def main():

  1010     the_coverage.command_line(sys.argv[1:])

  1011     

  1012 # Command-line interface.

  1013 if __name__ == '__main__':

  1014     main()

  1015 

  1016 

  1017 # A. REFERENCES

  1018 #

  1019 # [GDR 2001-12-04a] "Statement coverage for Python"; Gareth Rees;

  1020 # Ravenbrook Limited; 2001-12-04;

  1021 # <http://www.nedbatchelder.com/code/modules/rees-coverage.html>.

  1022 #

  1023 # [GDR 2001-12-04b] "Statement coverage for Python: design and

  1024 # analysis"; Gareth Rees; Ravenbrook Limited; 2001-12-04;

  1025 # <http://www.nedbatchelder.com/code/modules/rees-design.html>.

  1026 #

  1027 # [van Rossum 2001-07-20a] "Python Reference Manual (releae 2.1.1)";

  1028 # Guide van Rossum; 2001-07-20;

  1029 # <http://www.python.org/doc/2.1.1/ref/ref.html>.

  1030 #

  1031 # [van Rossum 2001-07-20b] "Python Library Reference"; Guido van Rossum;

  1032 # 2001-07-20; <http://www.python.org/doc/2.1.1/lib/lib.html>.

  1033 #

  1034 #

  1035 # B. DOCUMENT HISTORY

  1036 #

  1037 # 2001-12-04 GDR Created.

  1038 #

  1039 # 2001-12-06 GDR Added command-line interface and source code

  1040 # annotation.

  1041 #

  1042 # 2001-12-09 GDR Moved design and interface to separate documents.

  1043 #

  1044 # 2001-12-10 GDR Open cache file as binary on Windows.  Allow

  1045 # simultaneous -e and -x, or -a and -r.

  1046 #

  1047 # 2001-12-12 GDR Added command-line help.  Cache analysis so that it

  1048 # only needs to be done once when you specify -a and -r.

  1049 #

  1050 # 2001-12-13 GDR Improved speed while recording.  Portable between

  1051 # Python 1.5.2 and 2.1.1.

  1052 #

  1053 # 2002-01-03 GDR Module-level functions work correctly.

  1054 #

  1055 # 2002-01-07 GDR Update sys.path when running a file with the -x option,

  1056 # so that it matches the value the program would get if it were run on

  1057 # its own.

  1058 #

  1059 # 2004-12-12 NMB Significant code changes.

  1060 # - Finding executable statements has been rewritten so that docstrings and

  1061 #   other quirks of Python execution aren't mistakenly identified as missing

  1062 #   lines.

  1063 # - Lines can be excluded from consideration, even entire suites of lines.

  1064 # - The filesystem cache of covered lines can be disabled programmatically.

  1065 # - Modernized the code.

  1066 #

  1067 # 2004-12-14 NMB Minor tweaks.  Return 'analysis' to its original behavior

  1068 # and add 'analysis2'.  Add a global for 'annotate', and factor it, adding

  1069 # 'annotate_file'.

  1070 #

  1071 # 2004-12-31 NMB Allow for keyword arguments in the module global functions.

  1072 # Thanks, Allen.

  1073 #

  1074 # 2005-12-02 NMB Call threading.settrace so that all threads are measured.

  1075 # Thanks Martin Fuzzey. Add a file argument to report so that reports can be 

  1076 # captured to a different destination.

  1077 #

  1078 # 2005-12-03 NMB coverage.py can now measure itself.

  1079 #

  1080 # 2005-12-04 NMB Adapted Greg Rogers' patch for using relative filenames,

  1081 # and sorting and omitting files to report on.

  1082 #

  1083 # 2006-07-23 NMB Applied Joseph Tate's patch for function decorators.

  1084 #

  1085 # 2006-08-21 NMB Applied Sigve Tjora and Mark van der Wal's fixes for argument

  1086 # handling.

  1087 #

  1088 # 2006-08-22 NMB Applied Geoff Bache's parallel mode patch.

  1089 #

  1090 # 2006-08-23 NMB Refactorings to improve testability.  Fixes to command-line

  1091 # logic for parallel mode and collect.

  1092 #

  1093 # 2006-08-25 NMB "#pragma: nocover" is excluded by default.

  1094 #

  1095 # 2006-09-10 NMB Properly ignore docstrings and other constant expressions that

  1096 # appear in the middle of a function, a problem reported by Tim Leslie.

  1097 # Minor changes to avoid lint warnings.

  1098 #

  1099 # 2006-09-17 NMB coverage.erase() shouldn't clobber the exclude regex.

  1100 # Change how parallel mode is invoked, and fix erase() so that it erases the

  1101 # cache when called programmatically.

  1102 #

  1103 # 2007-07-21 NMB In reports, ignore code executed from strings, since we can't

  1104 # do anything useful with it anyway.

  1105 # Better file handling on Linux, thanks Guillaume Chazarain.

  1106 # Better shell support on Windows, thanks Noel O'Boyle.

  1107 # Python 2.2 support maintained, thanks Catherine Proulx.

  1108 #

  1109 # 2007-07-22 NMB Python 2.5 now fully supported. The method of dealing with

  1110 # multi-line statements is now less sensitive to the exact line that Python

  1111 # reports during execution. Pass statements are handled specially so that their

  1112 # disappearance during execution won't throw off the measurement.

  1113 #

  1114 # 2007-07-23 NMB Now Python 2.5 is *really* fully supported: the body of the

  1115 # new with statement is counted as executable.

  1116 #

  1117 # 2007-07-29 NMB Better packaging.

  1118 #

  1119 # 2007-09-30 NMB Don't try to predict whether a file is Python source based on

  1120 # the extension. Extensionless files are often Pythons scripts. Instead, simply

  1121 # parse the file and catch the syntax errors.  Hat tip to Ben Finney.

  1122 #

  1123 # 2008-05-25 NMB Open files in rU mode to avoid line ending craziness.

  1124 # Thanks, Edward Loper.

  1125 #

  1126 # 2008-09-14 NMB Add support for finding source files in eggs.

  1127 # Don't check for morf's being instances of ModuleType, instead use duck typing

  1128 # so that pseudo-modules can participate. Thanks, Imri Goldberg.

  1129 # Use os.realpath as part of the fixing of filenames so that symlinks won't

  1130 # confuse things.  Thanks, Patrick Mezard.

  1131 #

  1132 #

  1133 # C. COPYRIGHT AND LICENCE

  1134 #

  1135 # Copyright 2001 Gareth Rees.  All rights reserved.

  1136 # Copyright 2004-2008 Ned Batchelder.  All rights reserved.