#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# __doc__ string is slightly unconventional because it is used as usage text
"""%prog [OPTIONS] [FIND_REGEX] [REPLACE_FORMAT]
Script to list, search, and modify files using Python regex patterns.
OPTIONS: optional command-line flags; see %prog --help
FIND_REGEX: an optional valid Python regular expression pattern;
if supplied, only files containing at least one match will be processed;
matching file paths will be printed; if supplied, REPLACE_FORMAT will be
used to convert the match groups into formatted output.
REPLACE_FORMAT: an optional valid Python format string;
FIND_REGEX must be supplied first if REPLACE_FORMAT is supplied;
positional arguments will be replaced with ordered groups from
FIND_REGEX matches, and named arguments will be replaced with named
groups from FIND_REGEX matches."""
__authors__ = [
'"Todd Larsen" <tlarsen@google.com>',
]
import dircache
import errno
import os
import optparse
import re
import sre_constants
import sys
class Error(Exception):
"""Base class of all exceptions in this module.
"""
pass
def compileRegex(pattern):
"""Compiles a Python regex pattern into a regex object.
Args:
pattern: valid Python regex pattern string, or an already-compiled
regex object (in which case this function is is a no-op)
Returns:
regex object compiled from pattern
Raises:
Error if pattern could not be compiled.
"""
try:
return re.compile(pattern)
except sre_constants.error, error:
msg = 're.compile: %s\n%s' % (error.args[0], pattern)
raise Error(errno.EINVAL, msg)
def findAll(text_to_search, pattern):
"""Returns all matches of a regex in a string.
Args:
text_to_search: string in which to find matches
pattern: Python regex pattern (or already-compiled regex object)
indicating which matches to retrieve
Returns:
a (possibly empty) list of the matches found, as strings
"""
matches = []
def _captureMatchText(match):
match_text = match.group()
matches.append(match_text)
return match_text
compileRegex(pattern).sub(_captureMatchText, text_to_search)
return matches
def getFileContents(file_path):
"""Reads the contents of a file as a single string, then closes the file.
Args:
file_path: path to the file to read its contents into a string
Returns:
a single string containing the entire contents of the file
"""
file_to_read = open(file_path)
file_contents = file_to_read.read()
file_to_read.close()
return file_contents
def findAllInFile(file_path, pattern, *ignored_args, **ignored_kwargs):
"""Action to return a list of all pattern matches in a file.
Args:
file_path: path of file to manipulate
pattern: see findAll()
*ignored_args: other positional arguments which are ignored
command-line arguments not used by this action callable
**ignored_kwargs: other keyword arguments which are ignored
command-line options not used by this action callable
Returns:
two-tuple of boolean indicating if any match was found and a
(possibly empty) list of the matches found, as strings (to be used
as printable output of the action)
"""
matches = findAll(getFileContents(file_path), pattern)
if matches:
found = True
else:
found = False
return found, matches
def replaceAll(original, pattern, format):
"""Substitutes formatted text for all matches in a string.
Args:
original: original string in which to find and replace matches
pattern: Python regex pattern (or already-compiled regex object)
indicating which matches to replace
format: Python format string specifying how to format the
replacement text; how this format string is interpreted depends
on the contents of the pattern; if the pattern contains:
named groups: format is expected to contain named format specifiers
unnamed groups: format is expected to contain exactly the same
number of unnamed format specifiers as the number of groups in
pattern
no groups: format is expected to contain a single format specifier
(in which case the entire match is supplied to it), or no format
specifier at all (in which case the "format" string simply
replaces the match with no substitutions from the match itself)
Returns:
two-tuple of the text with all matches replaced as specified by
pattern and format, and a list of the original matches, each followed
by its replacement
"""
matches_and_replacements = []
def _replaceWithFormat(match):
formatted_match = None
if match.groupdict():
try:
formatted_match = format % match.groupdict()
except TypeError:
pass
if (not formatted_match) and match.groups():
try:
formatted_match = format % match.groups()
except TypeError:
pass
if (not formatted_match):
try:
formatted_match = format % match.group()
except TypeError:
formatted_match = format
matches_and_replacements.append(match.group())
matches_and_replacements.append(formatted_match)
return formatted_match
replaced = compileRegex(pattern).sub(_replaceWithFormat, original)
return replaced, matches_and_replacements
def writeAltFileIfExt(path, ext, contents):
"""Writes a file if path and additional extension are supplied.
If path or ext are not supplied, no file is written.
Args:
path: path of file to be written, to which ext will be appended
ext: additional file extension that will be appended to path
contents: contents of file to be written, as a string
"""
if (not path) or (not ext):
return
if ext.startswith('.'):
ext = ext[1:]
alt_path = '%s.%s' % (path, ext)
alt_file = open(alt_path, 'w')
alt_file.write(contents)
alt_file.close()
def replaceAllInFile(file_path, pattern, format,
new_ext=None, backup_ext=None,
overwrite_files=False,
*ignored_args, **ignored_kwargs):
"""Substitutes formatted text for all matches in a file.
Args:
file_path: path of file to manipulate
pattern, format: see replaceAll()
*ignored_args: other positional arguments which are ignored
command-line arguments not used by this action callable
**ignored_kwargs: other keyword arguments which are ignored
command-line options not used by this action callable
Returns:
two-tuple of boolean indicating if any match was found and a
list of printable output text lines containing pairs of original
pattern matches each followed by the formatted replacement
"""
original = getFileContents(file_path)
replaced, matches_and_replacements = replaceAll(
original, pattern, format)
if matches_and_replacements:
found = True
writeAltFileIfExt(file_path, new_ext, replaced)
writeAltFileIfExt(file_path, backup_ext, original)
if overwrite_files:
if replaced != original:
replaced_file = open(file_path, 'w')
replaced_file.write(replaced)
replaced_file.close()
else:
found = False
return found, matches_and_replacements
def listFile(*ignored_args, **ignored_kwargs):
"""No-op action callable that ignores arguments and returns (True, []).
"""
return True, [] # match only based on file names, which was done by caller
def applyActionToFiles(action, action_args,
start_path='', abs_path=False, files_pattern='',
recurse_dirs=False, dirs_pattern='',
follow_symlinks=False, quiet_output=False,
hide_paths=False, hide_text=False, **action_options):
"""Applies a callable action to files, based on options and arguments.
Args:
action: callable that expects a file path argument, positional arguments
(action_args), and keyword options from the command-line options dict;
and returns a "matched" boolean and a list of output strings
action_args: list of positional arguments, if any; passed to action
callable unchanged
start_path: required path of initial directory to visit
abs_path: optional boolean indicating to use absolute paths
files_pattern: required Python regex (object or pattern) which selects
which files to pass to the action callable
recurse_dirs: boolean indicating if subdirectories should be traversed
dirs_pattern: Python regex (object or pattern) which selects which
subdirectories to traverse if recurse_dirs is True
follow_symlinks: boolean indicating if symlinks should be traversed
quiet_output: optional boolean indicating if output should be suppressed
hide_paths: optional boolean indicating to omit file paths from output
hide_text: optional boolean indicating to omit find/replace text from
output
**action_options: remaining keyword arguments that are passed unchanged
to the action callable
Returns:
two-tuple containing an exit code and a (possibly empty) list of
output strings
Raises:
Error exception if problems occur (file I/O, invalid regex, etc.).
"""
exit_code = errno.ENOENT
output = []
start_path = os.path.expandvars(os.path.expanduser(start_path))
if abs_path:
start_path = os.path.abspath(start_path)
paths = [start_path]
files_regex = compileRegex(files_pattern)
if recurse_dirs:
dirs_regex = compileRegex(dirs_pattern)
while paths:
sub_paths = []
for path in paths:
# expand iterator into an actual list and sort it
try:
items = dircache.listdir(path)[:]
except (IOError, OSError), error:
raise Error(error.args[0], '%s: %s' % (
error.__class__.__name__, error.args[1]))
items.sort()
for item in items:
item_path = os.path.join(path, item)
if os.path.islink(item_path):
if not follow_symlinks:
continue # do not follow symlinks (ignore them)
if os.path.isdir(item_path):
if recurse_dirs:
if dirs_regex.match(item):
sub_paths.append(item_path)
continue
if os.path.isfile(item_path) and files_regex.match(item):
try:
matched, found_output = action(item_path, *action_args,
**action_options)
except (IOError, OSError), error:
raise Error(error.args[0], '%s: %s' % (
error.__class__.__name__, error.args[1]))
if matched:
exit_code = 0 # at least one matched file has now been found
if (not quiet_output) and (not hide_paths):
output.append(item_path)
if (not quiet_output) and (not hide_text):
output.extend(found_output)
paths = sub_paths
return exit_code, output
class _ErrorOptionParser(optparse.OptionParser):
"""Customized optparse.OptionParser that does not call sys.exit().
"""
def error(self, msg):
"""Raises an Error exception, instead of calling sys.exit().
"""
raise Error(errno.EINVAL, msg)
def _buildParser():
"""Returns a custom OptionParser for parsing command-line arguments.
"""
parser = _ErrorOptionParser(__doc__)
filter_group = optparse.OptionGroup(parser,
'File Options',
'Options used to select which files to process.')
filter_group.add_option(
'-f', '--files', dest='files_pattern',
default='(?!^.*\.pyc|.*\.ico|.*\.gif|.*\.png|.*\.jpg$)',
metavar='FILES_REGEX',
help=('Python regex pattern (*not* a glob!) defining files to process'
' in each directory [default: %default]'))
filter_group.add_option(
'-F', '--follow', dest='follow_symlinks', default=False,
action='store_true',
help=('follow file and subdirectory symlinks (possibly *DANGEROUS*)'
' [default: %default]'))
parser.add_option_group(filter_group)
dir_group = optparse.OptionGroup(parser,
'Directory Options',
'Options used to indicate which directories to traverse.')
dir_group.add_option(
'-s', '--start', dest='start_path', default=os.curdir, metavar='PATH',
help='directory in which to start processing files [default: %default]')
dir_group.add_option(
'-R', '--recursive', dest='recurse_dirs', default=False,
action='store_true',
help='recurse into subdirectories [default: %default]')
dir_group.add_option(
'-d', '--dirs', dest='dirs_pattern', default='^[^.].*$',
metavar='SUBDIRS_REGEX',
help=('Python regex pattern (*not* a glob!) defining subdirectories to'
' recurse into (if --recursive) [default: %default]'))
parser.add_option_group(dir_group)
output_group = optparse.OptionGroup(parser,
'Output Options',
'Options used to control program output.')
output_group.add_option(
'-a', '--abspath', dest='abs_path', default=False, action='store_true',
help=('output absolute paths instead of relative paths'
' [default: %default]'))
output_group.add_option(
'', '--nopaths', dest='hide_paths', default=False, action='store_true',
help=('suppress printing of file path names for successfully matched'
' files to stdout [default: %default]'))
output_group.add_option(
'', '--notext', dest='hide_text', default=False, action='store_true',
help=('suppress find/replace text output to stdout (but still print'
' paths if not --nopath, and still perform replacements if'
' specified) [default: %default]'))
output_group.add_option(
'-q', '--quiet', dest='quiet_output', default=False, action='store_true',
help=('suppress *all* printed output to stdout (but still perform'
' replacements if specified) [default: %default]'))
parser.add_option_group(output_group)
replace_group = optparse.OptionGroup(parser,
'Replace Options',
'Options applied when matches in files are replaced with substitutions.'
' (Only possible if REPLACE_FORMAT is supplied.)')
replace_group.add_option(
'-o', '--overwrite', dest='overwrite_files', default=False,
action='store_true',
help=('overwrite original files with formatted text substituted for'
' matches [default: %default]'))
replace_group.add_option(
'-b', '--backup', dest='backup_ext', default='', metavar='EXTENSION',
help=('if supplied, and file would be overwritten, backup original'
' file with the supplied extension [default is no backups of'
' overwritten files are kept]'))
replace_group.add_option(
'-n', '--new', dest='new_ext', default='', metavar='EXTENSION',
help=('if supplied, and file has matches and and is altered by'
' substitutions, create a new file with the supplied extension'
' [default is no new file is created]'))
parser.add_option_group(replace_group)
return parser
def _parseArgs(cmd_line_args):
"""Builds a command-line option parser and parses command-line arguments.
Args:
cmd_line_args: command-line arguments, excluding the argv[0] program name
Returns:
four-tuple of action callable, supplied command-line options (including
those defined by defaults in the command-line parser) as a dict,
remaining positional command-line arguments, and the parser itself
Raises:
Error if problems occurred during commmand-line argument parsing.
"""
parser = _buildParser()
options, args = parser.parse_args(args=cmd_line_args)
if not args:
# no FIND_REGEX or REPLACE_PATTERN supplied, so just match based
# on file name and subdirectory name patterns
action = listFile
elif len(args) == 1:
# FIND_REGEX supplied, but not REPLACE_PATTERN, so just match based
# on file name and subdirectory name patterns, and then on file
# contents
action = findAllInFile
elif len(args) == 2:
# FIND_REGEX and REPLACE_PATTERN both supplied, so match based
# on file name and subdirectory name patterns, and then do a find and
# replace on file contents
action = replaceAllInFile
else:
raise Error(errno.EINVAL,'too many (%d) arguments supplied:\n%s' % (
len(args), ' '.join(args)))
return action, vars(options), args, parser
def _main(argv):
"""Wrapper that catches exceptions, prints output, and returns exit status.
Normal program output is printed to stdout. Error output (including
exception text) is printed to stderr.
Args:
argv: script arguments, usually sys.argv; argv[0] is expected to be the
program name
Returns:
exit code suitable for sys.exit()
"""
options = {} # empty options, used if _parseArgs() fails
parser = None
try:
action, options, args, parser = _parseArgs(argv[1:])
exit_code, output = applyActionToFiles(action, args, **options)
if output: print '\n'.join(output)
except Error, error:
if not options.get('quiet_output'):
print >>sys.stderr, '\nERROR: (%s: %s) %s\n' % (
error.args[0], os.strerror(error.args[0]), error.args[1])
if parser:
print >>sys.stderr, parser.get_usage()
exit_code = error.args[0]
return exit_code
if __name__ == '__main__':
sys.exit(_main(sys.argv))