#!/usr/bin/python2.5

#

# Copyright 2008 the Melange authors.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#   http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

# __doc__ string is slightly unconventional because it is used as usage text

"""%prog [OPTIONS] [FIND_REGEX] [REPLACE_FORMAT]

Script to list, search, and modify files using Python regex patterns.

OPTIONS:  optional command-line flags; see %prog --help

FIND_REGEX:  an optional valid Python regular expression pattern;

  if supplied, only files containing at least one match will be processed;

  matching file paths will be printed; if supplied, REPLACE_FORMAT will be

  used to convert the match groups into formatted output.

REPLACE_FORMAT:  an optional valid Python format string;

  FIND_REGEX must be supplied first if REPLACE_FORMAT is supplied;

  positional arguments will be replaced with ordered groups from

  FIND_REGEX matches, and named arguments will be replaced with named

  groups from FIND_REGEX matches."""

__authors__ = [

  '"Todd Larsen" <tlarsen@google.com>',

]

import dircache

import errno

import os

import optparse

import re

import sre_constants

import sys

class Error(Exception):

  """Base class of all exceptions in this module.

  """

  pass

def compileRegex(pattern):

  """Compiles a Python regex pattern into a regex object.

  Args:

    pattern: valid Python regex pattern string, or an already-compiled

      regex object (in which case this function is is a no-op)

  Returns:

    regex object compiled from pattern

  Raises:

    Error if pattern could not be compiled.

  """

  try:

    return re.compile(pattern)

  except sre_constants.error, error:

    msg = 're.compile: %s\n%s' % (error.args[0], pattern)

    raise Error(errno.EINVAL, msg)

def findAll(text_to_search, pattern):

  """Returns all matches of a regex in a string.

  Args:

    text_to_search: string in which to find matches

    pattern: Python regex pattern (or already-compiled regex object)

      indicating which matches to retrieve

  Returns:

    a (possibly empty) list of the matches found, as strings 

  """

  matches = []

  def _captureMatchText(match):

    match_text = match.group()

    matches.append(match_text)

    return match_text

  compileRegex(pattern).sub(_captureMatchText, text_to_search)

  return matches 

def getFileContents(file_path):

  """Reads the contents of a file as a single string, then closes the file.

  Args:

    file_path: path to the file to read its contents into a string

  Returns:

    a single string containing the entire contents of the file

  """

  file_to_read = open(file_path)

  file_contents = file_to_read.read()

  file_to_read.close()

  return file_contents

def findAllInFile(file_path, pattern, *ignored_args, **ignored_kwargs): 

  """Action to return a list of all pattern matches in a file.

  Args:

    file_path: path of file to manipulate

    pattern: see findAll()

    *ignored_args: other positional arguments which are ignored

      command-line arguments not used by this action callable

    **ignored_kwargs: other keyword arguments which are ignored

      command-line options not used by this action callable

  Returns:

    two-tuple of boolean indicating if any match was found and a

    (possibly empty) list of the matches found, as strings (to be used

    as printable output of the action)

  """

  matches = findAll(getFileContents(file_path), pattern)

  if matches:

    found = True

  else:

    found = False

  return found, matches

def replaceAll(original, pattern, format):

  """Substitutes formatted text for all matches in a string.

  Args:

    original: original string in which to find and replace matches

    pattern: Python regex pattern (or already-compiled regex object)

      indicating which matches to replace

    format: Python format string specifying how to format the

      replacement text; how this format string is interpreted depends

      on the contents of the pattern;  if the pattern contains:

        named groups: format is expected to contain named format specifiers

        unnamed groups: format is expected to contain exactly the same

          number of unnamed format specifiers as the number of groups in

          pattern

        no groups: format is expected to contain a single format specifier

          (in which case the entire match is supplied to it), or no format

          specifier at all (in which case the "format" string simply

          replaces the match with no substitutions from the match itself)

  Returns:

    two-tuple of the text with all matches replaced as specified by

    pattern and format, and a list of the original matches, each followed

    by its replacement 

  """

  matches_and_replacements = []

  def _replaceWithFormat(match):

    formatted_match = None

    if match.groupdict():

      try:

        formatted_match = format % match.groupdict()

      except TypeError:

        pass

    if (not formatted_match) and match.groups():

      try:

        formatted_match = format % match.groups()

      except TypeError:

        pass

    if (not formatted_match):

      try:

        formatted_match = format % match.group()

      except TypeError:

        formatted_match = format

    matches_and_replacements.append(match.group())

    matches_and_replacements.append(formatted_match)

    return formatted_match

  replaced = compileRegex(pattern).sub(_replaceWithFormat, original)

  return replaced, matches_and_replacements

def writeAltFileIfExt(path, ext, contents):

  """Writes a file if path and additional extension are supplied.

  If path or ext are not supplied, no file is written.

  Args:

    path: path of file to be written, to which ext will be appended

    ext: additional file extension that will be appended to path

    contents: contents of file to be written, as a string

  """

  if (not path) or (not ext):

    return

  if ext.startswith('.'):

    ext = ext[1:]

  alt_path = '%s.%s' % (path, ext)

  alt_file = open(alt_path, 'w')

  alt_file.write(contents) 

  alt_file.close()

def replaceAllInFile(file_path, pattern, format,

                     new_ext=None, backup_ext=None,

                     overwrite_files=False,

                     *ignored_args, **ignored_kwargs): 

  """Substitutes formatted text for all matches in a file.

  Args:

    file_path: path of file to manipulate

    pattern, format: see replaceAll()

    *ignored_args: other positional arguments which are ignored

      command-line arguments not used by this action callable

    **ignored_kwargs: other keyword arguments which are ignored

      command-line options not used by this action callable

  Returns:

    two-tuple of boolean indicating if any match was found and a

    list of printable output text lines containing pairs of original

    pattern matches each followed by the formatted replacement

  """

  original = getFileContents(file_path)

  replaced, matches_and_replacements = replaceAll(

    original, pattern, format)

  if matches_and_replacements:

    found = True

    writeAltFileIfExt(file_path, new_ext, replaced)

    writeAltFileIfExt(file_path, backup_ext, original)

    if overwrite_files:

      if replaced != original:

        replaced_file = open(file_path, 'w')

        replaced_file.write(replaced)

        replaced_file.close()

  else:

    found = False

  return found, matches_and_replacements

def listFile(*ignored_args, **ignored_kwargs): 

  """No-op action callable that ignores arguments and returns (True, []).

  """

  return True, []  # match only based on file names, which was done by caller

def applyActionToFiles(action, action_args,

                       start_path='', abs_path=False, files_pattern='',

                       recurse_dirs=False, dirs_pattern='',

                       follow_symlinks=False, quiet_output=False,

                       hide_paths=False, hide_text=False, **action_options):

  """Applies a callable action to files, based on options and arguments.

  Args:

    action: callable that expects a file path argument, positional arguments

      (action_args), and keyword options from the command-line options dict;

      and returns a "matched" boolean and a list of output strings

    action_args: list of positional arguments, if any; passed to action

      callable unchanged

    start_path: required path of initial directory to visit

    abs_path: optional boolean indicating to use absolute paths

    files_pattern: required Python regex (object or pattern) which selects

      which files to pass to the action callable

    recurse_dirs: boolean indicating if subdirectories should be traversed 

    dirs_pattern: Python regex (object or pattern) which selects which

      subdirectories to traverse if recurse_dirs is True

    follow_symlinks: boolean indicating if symlinks should be traversed

    quiet_output: optional boolean indicating if output should be suppressed

    hide_paths: optional boolean indicating to omit file paths from output

    hide_text: optional boolean indicating to omit find/replace text from

      output

    **action_options: remaining keyword arguments that are passed unchanged

      to the action callable

  Returns:

    two-tuple containing an exit code and a (possibly empty) list of

    output strings

  Raises:

    Error exception if problems occur (file I/O, invalid regex, etc.).

  """

  exit_code = errno.ENOENT

  output = []

  start_path = os.path.expandvars(os.path.expanduser(start_path))

  if abs_path:

    start_path = os.path.abspath(start_path)

  paths = [start_path]

  files_regex = compileRegex(files_pattern)

  if recurse_dirs:

    dirs_regex = compileRegex(dirs_pattern)

  while paths:

    sub_paths = []

    for path in paths:

      # expand iterator into an actual list and sort it

      try:

        items = dircache.listdir(path)[:]

      except (IOError, OSError), error:

        raise Error(error.args[0], '%s: %s' % (

                    error.__class__.__name__, error.args[1]))

      items.sort()

      for item in items:

        item_path = os.path.join(path, item)

        if os.path.islink(item_path):

          if not follow_symlinks:

            continue  # do not follow symlinks (ignore them)

        if os.path.isdir(item_path):

          if recurse_dirs:

            if dirs_regex.match(item):

              sub_paths.append(item_path)

          continue

        if os.path.isfile(item_path) and files_regex.match(item):

          try:

            matched, found_output = action(item_path, *action_args,

                                           **action_options)

          except (IOError, OSError), error:

            raise Error(error.args[0], '%s: %s' % (

                        error.__class__.__name__, error.args[1]))

          if matched:

            exit_code = 0  # at least one matched file has now been found

            if (not quiet_output) and (not hide_paths):

              output.append(item_path)

          if (not quiet_output) and (not hide_text):

            output.extend(found_output)

    paths = sub_paths

  return exit_code, output

class _ErrorOptionParser(optparse.OptionParser):

  """Customized optparse.OptionParser that does not call sys.exit().

  """

  def error(self, msg):

    """Raises an Error exception, instead of calling sys.exit().

    """

    raise Error(errno.EINVAL, msg)

def _buildParser():

  """Returns a custom OptionParser for parsing command-line arguments.

  """

  parser = _ErrorOptionParser(__doc__)

  filter_group = optparse.OptionGroup(parser,

    'File Options',

    'Options used to select which files to process.')

  filter_group.add_option(

    '-f', '--files', dest='files_pattern',

    default='(?!^.*\.pyc|.*\.ico|.*\.gif|.*\.png|.*\.jpg$)',

    metavar='FILES_REGEX',

    help=('Python regex pattern (*not* a glob!) defining files to process'

          ' in each directory [default: %default]'))

  filter_group.add_option(

    '-F', '--follow', dest='follow_symlinks', default=False,

    action='store_true',

    help=('follow file and subdirectory symlinks (possibly *DANGEROUS*)'

          ' [default: %default]'))

  parser.add_option_group(filter_group)

  dir_group = optparse.OptionGroup(parser,

    'Directory Options',

    'Options used to indicate which directories to traverse.')

  dir_group.add_option(

    '-s', '--start', dest='start_path', default=os.curdir, metavar='PATH',

    help='directory in which to start processing files [default: %default]')

  dir_group.add_option(

    '-R', '--recursive', dest='recurse_dirs', default=False,

    action='store_true',

    help='recurse into subdirectories [default: %default]')

  dir_group.add_option(

    '-d', '--dirs', dest='dirs_pattern', default='^[^.].*$',

    metavar='SUBDIRS_REGEX',

    help=('Python regex pattern (*not* a glob!) defining subdirectories to'

          ' recurse into (if --recursive) [default: %default]'))

  parser.add_option_group(dir_group)

  output_group = optparse.OptionGroup(parser,

    'Output Options',

    'Options used to control program output.')

  output_group.add_option(

    '-a', '--abspath', dest='abs_path', default=False, action='store_true',

    help=('output absolute paths instead of relative paths'

          ' [default: %default]'))

  output_group.add_option(

    '', '--nopaths', dest='hide_paths', default=False, action='store_true',

    help=('suppress printing of file path names for successfully matched'

          ' files to stdout [default: %default]'))

  output_group.add_option(

    '', '--notext', dest='hide_text', default=False, action='store_true',

    help=('suppress find/replace text output to stdout (but still print'

          ' paths if not --nopath, and still perform replacements if'

          ' specified) [default: %default]'))

  output_group.add_option(

    '-q', '--quiet', dest='quiet_output', default=False, action='store_true',

    help=('suppress *all* printed output to stdout (but still perform'

          ' replacements if specified) [default: %default]'))

  parser.add_option_group(output_group)

  replace_group = optparse.OptionGroup(parser,

    'Replace Options',

    'Options applied when matches in files are replaced with substitutions.'

    ' (Only possible if REPLACE_FORMAT is supplied.)')

  replace_group.add_option(

    '-o', '--overwrite', dest='overwrite_files', default=False,

    action='store_true',

    help=('overwrite original files with formatted text substituted for'

          ' matches [default: %default]'))  

  replace_group.add_option(

    '-b', '--backup', dest='backup_ext', default='', metavar='EXTENSION',

    help=('if supplied, and file would be overwritten, backup original'

          ' file with the supplied extension [default is no backups of'

          ' overwritten files are kept]'))

  replace_group.add_option(

    '-n', '--new', dest='new_ext', default='', metavar='EXTENSION',

    help=('if supplied, and file has matches and and is altered by'

          ' substitutions, create a new file with the supplied extension'

          ' [default is no new file is created]'))

  parser.add_option_group(replace_group)

  return parser

def _parseArgs(cmd_line_args):

  """Builds a command-line option parser and parses command-line arguments.

  Args:

    cmd_line_args: command-line arguments, excluding the argv[0] program name

  Returns:

    four-tuple of action callable, supplied command-line options (including

    those defined by defaults in the command-line parser) as a dict,

    remaining positional command-line arguments, and the parser itself

  Raises:

    Error if problems occurred during commmand-line argument parsing.

  """

  parser = _buildParser()

  options, args = parser.parse_args(args=cmd_line_args)

  if not args:

    # no FIND_REGEX or REPLACE_PATTERN supplied, so just match based

    # on file name and subdirectory name patterns

    action = listFile

  elif len(args) == 1:

    # FIND_REGEX supplied, but not REPLACE_PATTERN, so just match based

    # on file name and subdirectory name patterns, and then on file

    # contents

    action = findAllInFile

  elif len(args) == 2:

    # FIND_REGEX and REPLACE_PATTERN both supplied, so match based

    # on file name and subdirectory name patterns, and then do a find and

    # replace on file contents

    action = replaceAllInFile

  else:

    raise Error(errno.EINVAL,'too many (%d) arguments supplied:\n%s' % (

                len(args), ' '.join(args)))

  return action, vars(options), args, parser

def _main(argv):

  """Wrapper that catches exceptions, prints output, and returns exit status.

  Normal program output is printed to stdout.  Error output (including

  exception text) is printed to stderr.

  Args:

    argv: script arguments, usually sys.argv; argv[0] is expected to be the

      program name

  Returns:

    exit code suitable for sys.exit()

  """

  options = {}  # empty options, used if _parseArgs() fails

  parser = None

  try:

    action, options, args, parser = _parseArgs(argv[1:])