py_tasks_melange: comparison thirdparty/google_appengine/google/appengine/tools/appcfg.py

equal deleted inserted replaced

-:c61d96e72e6f
+:d0b7dac5325c
 import google
 import yaml
 from google.appengine.cron import groctimespecification
 from google.appengine.api import appinfo
 from google.appengine.api import croninfo
+from google.appengine.api import queueinfo
 from google.appengine.api import validation
 from google.appengine.api import yaml_errors
 from google.appengine.api import yaml_object
 from google.appengine.datastore import datastore_index
 from google.appengine.tools import appengine_rpc
 from google.appengine.tools import bulkloader
 MAX_FILES_TO_CLONE = 100
-LIST_DELIMITER = "\n"
+LIST_DELIMITER = '\n'
-TUPLE_DELIMITER = "|"
+TUPLE_DELIMITER = '|'
-VERSION_FILE = "../VERSION"
+VERSION_FILE = '../VERSION'
 UPDATE_CHECK_TIMEOUT = 3
-NAG_FILE = ".appcfg_nag"
+NAG_FILE = '.appcfg_nag'
 MAX_LOG_LEVEL = 4
 verbosity = 1
-appinfo.AppInfoExternal.ATTRIBUTES[appinfo.RUNTIME] = "python"
+appinfo.AppInfoExternal.ATTRIBUTES[appinfo.RUNTIME] = 'python'
 _api_versions = os.environ.get('GOOGLE_TEST_API_VERSIONS', '1')
 _options = validation.Options(*_api_versions.split(','))
 appinfo.AppInfoExternal.ATTRIBUTES[appinfo.API_VERSION] = _options
 del _api_versions, _options
 The mime type string.  For example, 'text/plain' or 'image/gif'.
 None if this is not a static file.
 """
 for handler in config.handlers:
 handler_type = handler.GetHandlerType()
-if handler_type in ("static_dir", "static_files"):
+if handler_type in ('static_dir', 'static_files'):
-if handler_type == "static_dir":
+if handler_type == 'static_dir':
-regex = os.path.join(re.escape(handler.GetHandler()), ".*")
+regex = os.path.join(re.escape(handler.GetHandler()), '.*')
 else:
 regex = handler.upload
 if re.match(regex, filename):
 if handler.mime_type is not None:
 return handler.mime_type
 else:
 guess = mimetypes.guess_type(filename)[0]
 if guess is None:
-default = "application/octet-stream"
+default = 'application/octet-stream'
-print >>sys.stderr, ("Could not guess mimetype for %s.  Using %s."
+print >>sys.stderr, ('Could not guess mimetype for %s.  Using %s.'
 % (filename, default))
 return default
 return guess
 return None
 opt_in: True if the user wants to check for updates on dev_appserver
 start.  False if not.  May be None if we have not asked the user yet.
 """
 ATTRIBUTES = {
-"timestamp": validation.TYPE_FLOAT,
+'timestamp': validation.TYPE_FLOAT,
-"opt_in": validation.Optional(validation.TYPE_BOOL),
+'opt_in': validation.Optional(validation.TYPE_BOOL),
 }
 @staticmethod
 def Load(nag_file):
 """Load a single NagFile object where one and only one is expected.
 A Yaml object or None if the VERSION file does not exist.
 """
 version_filename = os.path.join(os.path.dirname(google.__file__),
 VERSION_FILE)
 if not isfile(version_filename):
-logging.error("Could not find version file at %s", version_filename)
+logging.error('Could not find version file at %s', version_filename)
 return None
-version_fh = open_fn(version_filename, "r")
+version_fh = open_fn(version_filename, 'r')
 try:
 version = yaml.safe_load(version_fh)
 finally:
 version_fh.close()
 return version
-def RetryWithBackoff(initial_delay, backoff_factor, max_tries, callable):
-"""Calls a function multiple times, backing off more and more each time.
+def RetryWithBackoff(initial_delay, backoff_factor, max_tries, callable_func):
+"""Calls a function multiple times, backing off more and more each time.
-Args:
-initial_delay: Initial delay after first try, in seconds.
+Args:
-backoff_factor: Delay will be multiplied by this factor after each try.
+initial_delay: Initial delay after first try, in seconds.
-max_tries: Maximum number of tries.
+backoff_factor: Delay will be multiplied by this factor after each try.
-callable: The method to call, will pass no arguments.
+max_tries: Maximum number of tries.
+callable_func: The method to call, will pass no arguments.
-Returns:
-True if the function succeded in one of its tries.
+Returns:
+True if the function succeded in one of its tries.
-Raises:
-Whatever the function raises--an exception will immediately stop retries.
+Raises:
-"""
+Whatever the function raises--an exception will immediately stop retries.
-delay = initial_delay
+"""
-while not callable() and max_tries > 0:
+delay = initial_delay
-StatusUpdate("Will check again in %s seconds." % delay)
+while not callable_func() and max_tries > 0:
-time.sleep(delay)
+StatusUpdate('Will check again in %s seconds.' % delay)
-delay *= backoff_factor
+time.sleep(delay)
-max_tries -= 1
+delay *= backoff_factor
-return max_tries > 0
+max_tries -= 1
+return max_tries > 0
 class UpdateCheck(object):
 """Determines if the local SDK is the latest version.
 self.open = open_fn
 @staticmethod
 def MakeNagFilename():
 """Returns the filename for the nag file for this user."""
-user_homedir = os.path.expanduser("~/")
+user_homedir = os.path.expanduser('~/')
 if not os.path.isdir(user_homedir):
 drive, unused_tail = os.path.splitdrive(os.__file__)
 if drive:
-os.environ["HOMEDRIVE"] = drive
+os.environ['HOMEDRIVE'] = drive
-return os.path.expanduser("~/" + NAG_FILE)
+return os.path.expanduser('~/' + NAG_FILE)
 def _ParseVersionFile(self):
 """Parse the local VERSION file.
 Returns:
 Raises:
 SystemExit if the api_version is not supported.
 """
 version = self._ParseVersionFile()
 if version is None:
-logging.error("Could not determine if the SDK supports the api_version "
+logging.error('Could not determine if the SDK supports the api_version '
-"requested in app.yaml.")
+'requested in app.yaml.')
 return
-if self.config.api_version not in version["api_versions"]:
+if self.config.api_version not in version['api_versions']:
-logging.critical("The api_version specified in app.yaml (%s) is not "
+logging.critical('The api_version specified in app.yaml (%s) is not '
-"supported by this release of the SDK.  The supported "
+'supported by this release of the SDK.  The supported '
-"api_versions are %s.",
+'api_versions are %s.',
-self.config.api_version, version["api_versions"])
+self.config.api_version, version['api_versions'])
 sys.exit(1)
 def CheckForUpdates(self):
 """Queries the server for updates and nags the user if appropriate.
 Queries the server for the latest SDK version at the same time reporting
 the local SDK version.  The server will respond with a yaml document
 containing the fields:
-"release": The name of the release (e.g. 1.2).
+'release': The name of the release (e.g. 1.2).
-"timestamp": The time the release was created (YYYY-MM-DD HH:MM AM/PM TZ).
+'timestamp': The time the release was created (YYYY-MM-DD HH:MM AM/PM TZ).
-"api_versions": A list of api_version strings (e.g. ['1', 'beta']).
+'api_versions': A list of api_version strings (e.g. ['1', 'beta']).
 We will nag the user with increasing severity if:
 - There is a new release.
 - There is a new release with a new api_version.
 - There is a new release that does not support the api_version named in
 self.config.
 """
 version = self._ParseVersionFile()
 if version is None:
-logging.info("Skipping update check")
+logging.info('Skipping update check')
 return
-logging.info("Checking for updates to the SDK.")
+logging.info('Checking for updates to the SDK.')
 try:
-response = self.server.Send("/api/updatecheck",
+response = self.server.Send('/api/updatecheck',
 timeout=UPDATE_CHECK_TIMEOUT,
-release=version["release"],
+release=version['release'],
-timestamp=version["timestamp"],
+timestamp=version['timestamp'],
-api_versions=version["api_versions"])
+api_versions=version['api_versions'])
 except urllib2.URLError, e:
-logging.info("Update check failed: %s", e)
+logging.info('Update check failed: %s', e)
 return
 latest = yaml.safe_load(response)
-if latest["release"] == version["release"]:
+if latest['release'] == version['release']:
-logging.info("The SDK is up to date.")
+logging.info('The SDK is up to date.')
 return
-api_versions = latest["api_versions"]
+api_versions = latest['api_versions']
 if self.config.api_version not in api_versions:
 self._Nag(
-"The api version you are using (%s) is obsolete!  You should\n"
+'The api version you are using (%s) is obsolete!  You should\n'
-"upgrade your SDK and test that your code works with the new\n"
+'upgrade your SDK and test that your code works with the new\n'
-"api version." % self.config.api_version,
+'api version.' % self.config.api_version,
 latest, version, force=True)
 return
 if self.config.api_version != api_versions[len(api_versions) - 1]:
 self._Nag(
-"The api version you are using (%s) is deprecated. You should\n"
+'The api version you are using (%s) is deprecated. You should\n'
-"upgrade your SDK to try the new functionality." %
+'upgrade your SDK to try the new functionality.' %
 self.config.api_version, latest, version)
 return
-self._Nag("There is a new release of the SDK available.",
+self._Nag('There is a new release of the SDK available.',
 latest, version)
 def _ParseNagFile(self):
 """Parses the nag file.
 Returns:
 A NagFile if the file was present else None.
 """
 nag_filename = UpdateCheck.MakeNagFilename()
 if self.isfile(nag_filename):
-fh = self.open(nag_filename, "r")
+fh = self.open(nag_filename, 'r')
 try:
 nag = NagFile.Load(fh)
 finally:
 fh.close()
 return nag
 Args:
 nag: The NagFile to write.
 """
 nagfilename = UpdateCheck.MakeNagFilename()
 try:
-fh = self.open(nagfilename, "w")
+fh = self.open(nagfilename, 'w')
 try:
 fh.write(nag.ToYAML())
 finally:
 fh.close()
 except (OSError, IOError), e:
-logging.error("Could not write nag file to %s. Error: %s", nagfilename, e)
+logging.error('Could not write nag file to %s. Error: %s', nagfilename, e)
 def _Nag(self, msg, latest, version, force=False):
 """Prints a nag message and updates the nag file's timestamp.
 Because we don't want to nag the user everytime, we store a simple
 """
 nag = self._ParseNagFile()
 if nag and not force:
 last_nag = datetime.datetime.fromtimestamp(nag.timestamp)
 if datetime.datetime.now() - last_nag < datetime.timedelta(weeks=1):
-logging.debug("Skipping nag message")
+logging.debug('Skipping nag message')
 return
 if nag is None:
 nag = NagFile()
 nag.timestamp = time.time()
 self._WriteNagFile(nag)
-print "****************************************************************"
+print '****************************************************************'
 print msg
-print "-----------"
+print '-----------'
-print "Latest SDK:"
+print 'Latest SDK:'
 print yaml.dump(latest)
-print "-----------"
+print '-----------'
-print "Your SDK:"
+print 'Your SDK:'
 print yaml.dump(version)
-print "-----------"
+print '-----------'
-print "Please visit http://code.google.com/appengine for the latest SDK"
+print 'Please visit http://code.google.com/appengine for the latest SDK'
-print "****************************************************************"
+print '****************************************************************'
 def AllowedToCheckForUpdates(self, input_fn=raw_input):
 """Determines if the user wants to check for updates.
 On startup, the dev_appserver wants to check for updates to the SDK.
 if nag is None:
 nag = NagFile()
 nag.timestamp = time.time()
 if nag.opt_in is None:
-answer = input_fn("Allow dev_appserver to check for updates on startup? "
+answer = input_fn('Allow dev_appserver to check for updates on startup? '
-"(Y/n): ")
+'(Y/n): ')
 answer = answer.strip().lower()
-if answer == "n" or answer == "no":
+if answer == 'n' or answer == 'no':
-print ("dev_appserver will not check for updates on startup.  To "
+print ('dev_appserver will not check for updates on startup.  To '
-"change this setting, edit %s" % UpdateCheck.MakeNagFilename())
+'change this setting, edit %s' % UpdateCheck.MakeNagFilename())
 nag.opt_in = False
 else:
-print ("dev_appserver will check for updates on startup.  To change "
+print ('dev_appserver will check for updates on startup.  To change '
-"this setting, edit %s" % UpdateCheck.MakeNagFilename())
+'this setting, edit %s' % UpdateCheck.MakeNagFilename())
 nag.opt_in = True
 self._WriteNagFile(nag)
 return nag.opt_in
 self.config = config
 self.definitions = definitions
 def DoUpload(self):
 """Uploads the index definitions."""
-StatusUpdate("Uploading index definitions.")
+StatusUpdate('Uploading index definitions.')
-self.server.Send("/api/datastore/index/add",
+self.server.Send('/api/datastore/index/add',
 app_id=self.config.application,
 version=self.config.version,
 payload=self.definitions.ToYAML())
 self.config = config
 self.cron = cron
 def DoUpload(self):
 """Uploads the cron entries."""
-StatusUpdate("Uploading cron entries.")
+StatusUpdate('Uploading cron entries.')
-self.server.Send("/api/datastore/cron/update",
+self.server.Send('/api/datastore/cron/update',
 app_id=self.config.application,
 version=self.config.version,
 payload=self.cron.ToYAML())
+class QueueEntryUpload(object):
+"""Provides facilities to upload task queue entries to the hosting service."""
+def __init__(self, server, config, queue):
+"""Creates a new QueueEntryUpload.
+Args:
+server: The RPC server to use.  Should be an instance of a subclass of
+AbstractRpcServer
+config: The AppInfoExternal object derived from the app.yaml file.
+queue: The QueueInfoExternal object loaded from the queue.yaml file.
+"""
+self.server = server
+self.config = config
+self.queue = queue
+def DoUpload(self):
+"""Uploads the task queue entries."""
+StatusUpdate('Uploading task queue entries.')
+self.server.Send('/api/queue/update',
+app_id=self.config.application,
+version=self.config.version,
+payload=self.queue.ToYAML())
 class IndexOperation(object):
 """Provide facilities for writing Index operation commands."""
 is the set of indexes that are present in the index.yaml file but missing
 from the server.  The second record is the set of indexes that are
 present on the server but missing from the index.yaml file (indicating
 that these indexes should probably be vacuumed).
 """
-StatusUpdate("Fetching index definitions diff.")
+StatusUpdate('Fetching index definitions diff.')
-response = self.server.Send("/api/datastore/index/diff",
+response = self.server.Send('/api/datastore/index/diff',
 app_id=self.config.application,
 payload=definitions.ToYAML())
 return datastore_index.ParseMultipleIndexDefinitions(response)
 def DoDelete(self, definitions):
 A single datstore_index.IndexDefinitions containing indexes that were
 not deleted, probably because they were already removed.  This may
 be normal behavior as there is a potential race condition between fetching
 the index-diff and sending deletion confirmation through.
 """
-StatusUpdate("Deleting selected index definitions.")
+StatusUpdate('Deleting selected index definitions.')
-response = self.server.Send("/api/datastore/index/delete",
+response = self.server.Send('/api/datastore/index/delete',
 app_id=self.config.application,
 payload=definitions.ToYAML())
 return datastore_index.ParseIndexDefinitions(response)
 Returns:
 True if user enters 'y' or 'a'.  False if user enter 'n'.
 """
 while True:
-print "This index is no longer defined in your index.yaml file."
+print 'This index is no longer defined in your index.yaml file.'
 print
 print index.ToYAML()
 print
 confirmation = self.confirmation_fn(
-"Are you sure you want to delete this index? (N/y/a): ")
+'Are you sure you want to delete this index? (N/y/a): ')
 confirmation = confirmation.strip().lower()
-if confirmation == "y":
+if confirmation == 'y':
 return True
-elif confirmation == "n" or not confirmation:
+elif confirmation == 'n' or not confirmation:
 return False
-elif confirmation == "a":
+elif confirmation == 'a':
 self.force = True
 return True
 else:
-print "Did not understand your response."
+print 'Did not understand your response.'
 def DoVacuum(self, definitions):
 """Vacuum indexes in datastore.
 This method will query the server to determine which indexes are not
 not_deleted = self.DoDelete(deletions)
 if not_deleted.indexes:
 not_deleted_count = len(not_deleted.indexes)
 if not_deleted_count == 1:
-warning_message = ("An index was not deleted.  Most likely this is "
+warning_message = ('An index was not deleted.  Most likely this is '
-"because it no longer exists.\n\n")
+'because it no longer exists.\n\n')
 else:
-warning_message = ("%d indexes were not deleted.  Most likely this "
+warning_message = ('%d indexes were not deleted.  Most likely this '
-"is because they no longer exist.\n\n"
+'is because they no longer exist.\n\n'
 % not_deleted_count)
 for index in not_deleted.indexes:
 warning_message += index.ToYAML()
 logging.warning(warning_message)
 class LogsRequester(object):
 """Provide facilities to export request logs."""
 def __init__(self, server, config, output_file,
-num_days, append, severity, now, vhost):
+num_days, append, severity, now, vhost, include_vhost):
 """Constructor.
 Args:
 server: The RPC server to use.  Should be an instance of HttpRpcServer
 or TestRpcServer.
 num_days: Number of days worth of logs to export; 0 for all available.
 append: True if appending to an existing file.
 severity: App log severity to request (0-4); None for no app logs.
 now: POSIX timestamp used for calculating valid dates for num_days.
 vhost: The virtual host of log messages to get. None for all hosts.
+include_vhost: If true, the virtual host is included in log messages.
 """
 self.server = server
 self.config = config
 self.output_file = output_file
 self.append = append
 self.num_days = num_days
 self.severity = severity
 self.vhost = vhost
-self.version_id = self.config.version + ".1"
+self.include_vhost = include_vhost
+self.version_id = self.config.version + '.1'
 self.sentinel = None
-self.write_mode = "w"
+self.write_mode = 'w'
 if self.append:
 self.sentinel = FindSentinel(self.output_file)
-self.write_mode = "a"
+self.write_mode = 'a'
 self.valid_dates = None
 if self.num_days:
 patterns = []
 now = PacificTime(now)
 for i in xrange(self.num_days):
 then = time.gmtime(now - 24*3600 * i)
-patterns.append(re.escape(time.strftime("%d/%m/%Y", then)))
+patterns.append(re.escape(time.strftime('%d/%m/%Y', then)))
-patterns.append(re.escape(time.strftime("%d/%b/%Y", then)))
+patterns.append(re.escape(time.strftime('%d/%b/%Y', then)))
-self.valid_dates = re.compile(r"[^[]+\[(" + "|".join(patterns) + r"):")
+self.valid_dates = re.compile(r'[^[]+\[(' + '|'.join(patterns) + r'):')
 def DownloadLogs(self):
 """Download the requested logs.
 This will write the logs to the file designated by
 self.output_file, or to stdout if the filename is '-'.
 Multiple roundtrips to the server may be made.
 """
-StatusUpdate("Downloading request logs for %s %s." %
+StatusUpdate('Downloading request logs for %s %s.' %
 (self.config.application, self.version_id))
 tf = tempfile.TemporaryFile()
 offset = None
 try:
 while True:
 try:
 offset = self.RequestLogLines(tf, offset)
 if not offset:
 break
 except KeyboardInterrupt:
-StatusUpdate("Keyboard interrupt; saving data downloaded so far.")
+StatusUpdate('Keyboard interrupt; saving data downloaded so far.')
 break
-StatusUpdate("Copying request logs to %r." % self.output_file)
+StatusUpdate('Copying request logs to %r.' % self.output_file)
-if self.output_file == "-":
+if self.output_file == '-':
 of = sys.stdout
 else:
 try:
 of = open(self.output_file, self.write_mode)
 except IOError, err:
-StatusUpdate("Can't write %r: %s." % (self.output_file, err))
+StatusUpdate('Can\'t write %r: %s.' % (self.output_file, err))
 sys.exit(1)
 try:
 line_count = CopyReversedLines(tf, of)
 finally:
 of.flush()
 if of is not sys.stdout:
 of.close()
 finally:
 tf.close()
-StatusUpdate("Copied %d records." % line_count)
+StatusUpdate('Copied %d records.' % line_count)
 def RequestLogLines(self, tf, offset):
 """Make a single roundtrip to the server.
 Args:
 Returns:
 The offset string to be used for the next request, if another
 request should be issued; or None, if not.
 """
-logging.info("Request with offset %r.", offset)
+logging.info('Request with offset %r.', offset)
-kwds = {"app_id": self.config.application,
+kwds = {'app_id': self.config.application,
-"version": self.version_id,
+'version': self.version_id,
-"limit": 100,
+'limit': 100,
 }
 if offset:
-kwds["offset"] = offset
+kwds['offset'] = offset
 if self.severity is not None:
-kwds["severity"] = str(self.severity)
+kwds['severity'] = str(self.severity)
 if self.vhost is not None:
-kwds["vhost"] = str(self.vhost)
+kwds['vhost'] = str(self.vhost)
-response = self.server.Send("/api/request_logs", payload=None, **kwds)
+if self.include_vhost is not None:
-response = response.replace("\r", "\0")
+kwds['include_vhost'] = str(self.include_vhost)
+response = self.server.Send('/api/request_logs', payload=None, **kwds)
+response = response.replace('\r', '\0')
 lines = response.splitlines()
-logging.info("Received %d bytes, %d records.", len(response), len(lines))
+logging.info('Received %d bytes, %d records.', len(response), len(lines))
 offset = None
-if lines and lines[0].startswith("#"):
+if lines and lines[0].startswith('#'):
-match = re.match(r"^#\s*next_offset=(\S+)\s*$", lines[0])
+match = re.match(r'^#\s*next_offset=(\S+)\s*$', lines[0])
 del lines[0]
 if match:
 offset = match.group(1)
-if lines and lines[-1].startswith("#"):
+if lines and lines[-1].startswith('#'):
 del lines[-1]
 valid_dates = self.valid_dates
 sentinel = self.sentinel
 len_sentinel = None
 if sentinel:
 len_sentinel = len(sentinel)
 for line in lines:
 if ((sentinel and
 line.startswith(sentinel) and
-line[len_sentinel : len_sentinel+1] in ("", "\0")) or
+line[len_sentinel : len_sentinel+1] in ('', '\0')) or
 (valid_dates and not valid_dates.match(line))):
 return None
-tf.write(line + "\n")
+tf.write(line + '\n')
 if not lines:
 return None
 return offset
 def CopyReversedLines(instream, outstream, blocksize=2**16):
 r"""Copy lines from input stream to output stream in reverse order.
 As a special feature, null bytes in the input are turned into
-newlines followed by tabs in the output, but these "sub-lines"
+newlines followed by tabs in the output, but these 'sub-lines'
 separated by null bytes are not reversed.  E.g. If the input is
-"A\0B\nC\0D\n", the output is "C\n\tD\nA\n\tB\n".
+'A\0B\nC\0D\n', the output is 'C\n\tD\nA\n\tB\n'.
 Args:
 instream: A seekable stream open for reading in binary mode.
 outstream: A stream open for writing; doesn't have to be seekable or binary.
 blocksize: Optional block size for buffering, for unit testing.
 The number of lines copied.
 """
 line_count = 0
 instream.seek(0, 2)
 last_block = instream.tell() // blocksize
-spillover = ""
+spillover = ''
 for iblock in xrange(last_block + 1, -1, -1):
 instream.seek(iblock * blocksize)
 data = instream.read(blocksize)
 lines = data.splitlines(True)
-lines[-1:] = "".join(lines[-1:] + [spillover]).splitlines(True)
+lines[-1:] = ''.join(lines[-1:] + [spillover]).splitlines(True)
-if lines and not lines[-1].endswith("\n"):
+if lines and not lines[-1].endswith('\n'):
-lines[-1] += "\n"
+lines[-1] += '\n'
 lines.reverse()
 if lines and iblock > 0:
 spillover = lines.pop()
 if lines:
 line_count += len(lines)
-data = "".join(lines).replace("\0", "\n\t")
+data = ''.join(lines).replace('\0', '\n\t')
 outstream.write(data)
 return line_count
 def FindSentinel(filename, blocksize=2**16):
 The contents of the last line in the file that doesn't start with
 a tab, with its trailing newline stripped; or None if the file
 couldn't be opened or no such line could be found by inspecting
 the last 'blocksize' bytes of the file.
 """
-if filename == "-":
+if filename == '-':
-StatusUpdate("Can't combine --append with output to stdout.")
+StatusUpdate('Can\'t combine --append with output to stdout.')
 sys.exit(2)
 try:
-fp = open(filename, "rb")
+fp = open(filename, 'rb')
 except IOError, err:
-StatusUpdate("Append mode disabled: can't read %r: %s." % (filename, err))
+StatusUpdate('Append mode disabled: can\'t read %r: %s.' % (filename, err))
 return None
 try:
 fp.seek(0, 2)
 fp.seek(max(0, fp.tell() - blocksize))
 lines = fp.readlines()
 del lines[:1]
 sentinel = None
 for line in lines:
-if not line.startswith("\t"):
+if not line.startswith('\t'):
 sentinel = line
 if not sentinel:
-StatusUpdate("Append mode disabled: can't find sentinel in %r." %
+StatusUpdate('Append mode disabled: can\'t find sentinel in %r.' %
 filename)
 return None
-return sentinel.rstrip("\n")
+return sentinel.rstrip('\n')
 finally:
 fp.close()
 class AppVersionUpload(object):
 Returns:
 The string representation of the hash.
 """
 h = sha.new(content).hexdigest()
-return "%s_%s_%s_%s_%s" % (h[0:8], h[8:16], h[16:24], h[24:32], h[32:40])
+return '%s_%s_%s_%s_%s' % (h[0:8], h[8:16], h[16:24], h[24:32], h[32:40])
 def AddFile(self, path, file_handle):
 """Adds the provided file to the list to be pushed to the server.
 Args:
 path: The path the file should be uploaded as.
 file_handle: A stream containing data to upload.
 """
-assert not self.in_transaction, "Already in a transaction."
+assert not self.in_transaction, 'Already in a transaction.'
 assert file_handle is not None
 reason = appinfo.ValidFilename(path)
 if reason:
 logging.error(reason)
 Returns:
 A list of pathnames for files that should be uploaded using UploadFile()
 before Commit() can be called.
 """
-assert not self.in_transaction, "Already in a transaction."
+assert not self.in_transaction, 'Already in a transaction.'
-StatusUpdate("Initiating update.")
+StatusUpdate('Initiating update.')
-self.server.Send("/api/appversion/create", app_id=self.app_id,
+self.server.Send('/api/appversion/create', app_id=self.app_id,
 version=self.version, payload=self.config.ToYAML())
 self.in_transaction = True
 files_to_clone = []
 blobs_to_clone = []
 file_type: the type of the files
 """
 if not files:
 return
-StatusUpdate("Cloning %d %s file%s." %
+StatusUpdate('Cloning %d %s file%s.' %
-(len(files), file_type, len(files) != 1 and "s" or ""))
+(len(files), file_type, len(files) != 1 and 's' or ''))
 for i in xrange(0, len(files), MAX_FILES_TO_CLONE):
 if i > 0 and i % MAX_FILES_TO_CLONE == 0:
-StatusUpdate("Cloned %d files." % i)
+StatusUpdate('Cloned %d files.' % i)
 chunk = files[i:min(len(files), i + MAX_FILES_TO_CLONE)]
 result = self.server.Send(url,
 app_id=self.app_id, version=self.version,
 payload=BuildClonePostBody(chunk))
 if result:
 files_to_upload.update(dict(
 (f, self.files[f]) for f in result.split(LIST_DELIMITER)))
-CloneFiles("/api/appversion/cloneblobs", blobs_to_clone, "static")
+CloneFiles('/api/appversion/cloneblobs', blobs_to_clone, 'static')
-CloneFiles("/api/appversion/clonefiles", files_to_clone, "application")
+CloneFiles('/api/appversion/clonefiles', files_to_clone, 'application')
-logging.info("Files to upload: " + str(files_to_upload))
+logging.info('Files to upload: ' + str(files_to_upload))
 self.files = files_to_upload
 return sorted(files_to_upload.iterkeys())
 def UploadFile(self, path, file_handle):
 file_handle: A file-like object containing the data to upload.
 Raises:
 KeyError: The provided file is not amongst those to be uploaded.
 """
-assert self.in_transaction, "Begin() must be called before UploadFile()."
+assert self.in_transaction, 'Begin() must be called before UploadFile().'
 if path not in self.files:
-raise KeyError("File '%s' is not in the list of files to be uploaded."
+raise KeyError('File \'%s\' is not in the list of files to be uploaded.'
 % path)
 del self.files[path]
 mime_type = GetMimeTypeIfStaticFile(self.config, path)
 if mime_type is not None:
-self.server.Send("/api/appversion/addblob", app_id=self.app_id,
+self.server.Send('/api/appversion/addblob', app_id=self.app_id,
 version=self.version, path=path, content_type=mime_type,
 payload=file_handle.read())
 else:
-self.server.Send("/api/appversion/addfile", app_id=self.app_id,
+self.server.Send('/api/appversion/addfile', app_id=self.app_id,
 version=self.version, path=path,
 payload=file_handle.read())
 def Commit(self):
 """Commits the transaction, making the new app version available.
 This tries the new 'deploy' method; if that fails it uses the old 'commit'.
 Raises:
 Exception: Some required files were not uploaded.
 """
-assert self.in_transaction, "Begin() must be called before Commit()."
+assert self.in_transaction, 'Begin() must be called before Commit().'
 if self.files:
-raise Exception("Not all required files have been uploaded.")
+raise Exception('Not all required files have been uploaded.')
 try:
 self.Deploy()
 if not RetryWithBackoff(1, 2, 8, self.IsReady):
-logging.warning("Version still not ready to serve, aborting.")
+logging.warning('Version still not ready to serve, aborting.')
-raise Exception("Version not ready.")
+raise Exception('Version not ready.')
 self.StartServing()
 except urllib2.HTTPError, e:
 if e.code != 404:
 raise
-StatusUpdate("Closing update.")
+StatusUpdate('Closing update.')
-self.server.Send("/api/appversion/commit", app_id=self.app_id,
+self.server.Send('/api/appversion/commit', app_id=self.app_id,
 version=self.version)
 self.in_transaction = False
 def Deploy(self):
 """Deploys the new app version but does not make it default.
 before Deploy() can be called.
 Raises:
 Exception: Some required files were not uploaded.
 """
-assert self.in_transaction, "Begin() must be called before Deploy()."
+assert self.in_transaction, 'Begin() must be called before Deploy().'
 if self.files:
-raise Exception("Not all required files have been uploaded.")
+raise Exception('Not all required files have been uploaded.')
-StatusUpdate("Deploying new version.")
+StatusUpdate('Deploying new version.')
-self.server.Send("/api/appversion/deploy", app_id=self.app_id,
+self.server.Send('/api/appversion/deploy', app_id=self.app_id,
 version=self.version)
 self.deployed = True
 def IsReady(self):
 """Check if the new app version is ready to serve traffic.
 Exception: Deploy has not yet been called.
 Returns:
 True if the server returned the app is ready to serve.
 """
-assert self.deployed, "Deploy() must be called before IsReady()."
+assert self.deployed, 'Deploy() must be called before IsReady().'
-StatusUpdate("Checking if new version is ready to serve.")
+StatusUpdate('Checking if new version is ready to serve.')
-result = self.server.Send("/api/appversion/isready", app_id=self.app_id,
+result = self.server.Send('/api/appversion/isready', app_id=self.app_id,
 version=self.version)
-return result == "1"
+return result == '1'
 def StartServing(self):
 """Start serving with the newly created version.
 Raises:
 Exception: Deploy has not yet been called.
 """
-assert self.deployed, "Deploy() must be called before IsReady()."
+assert self.deployed, 'Deploy() must be called before IsReady().'
-StatusUpdate("Closing update: new version is ready to start serving.")
+StatusUpdate('Closing update: new version is ready to start serving.')
-self.server.Send("/api/appversion/startserving",
+self.server.Send('/api/appversion/startserving',
 app_id=self.app_id, version=self.version)
 self.in_transaction = False
 def Rollback(self):
 """Rolls back the transaction if one is in progress."""
 if not self.in_transaction:
 return
-StatusUpdate("Rolling back the update.")
+StatusUpdate('Rolling back the update.')
-self.server.Send("/api/appversion/rollback", app_id=self.app_id,
+self.server.Send('/api/appversion/rollback', app_id=self.app_id,
 version=self.version)
 self.in_transaction = False
 self.files = {}
 def DoUpload(self, paths, max_size, openfunc):
 Args:
 paths: An iterator that yields the relative paths of the files to upload.
 max_size: The maximum size file to upload.
 openfunc: A function that takes a path and returns a file-like object.
 """
-logging.info("Reading app configuration.")
+logging.info('Reading app configuration.')
-path = ""
+path = ''
 try:
-StatusUpdate("Scanning files on local disk.")
+StatusUpdate('Scanning files on local disk.')
 num_files = 0
 for path in paths:
 file_handle = openfunc(path)
 try:
 if self.config.skip_files.match(path):
-logging.info("Ignoring file '%s': File matches ignore regex.",
+logging.info('Ignoring file \'%s\': File matches ignore regex.',
 path)
 else:
 file_length = GetFileLength(file_handle)
 if file_length > max_size:
-logging.error("Ignoring file '%s': Too long "
+logging.error('Ignoring file \'%s\': Too long '
-"(max %d bytes, file is %d bytes)",
+'(max %d bytes, file is %d bytes)',
 path, max_size, file_length)
 else:
-logging.info("Processing file '%s'", path)
+logging.info('Processing file \'%s\'', path)
 self.AddFile(path, file_handle)
 finally:
 file_handle.close()
 num_files += 1
 if num_files % 500 == 0:
-StatusUpdate("Scanned %d files." % num_files)
+StatusUpdate('Scanned %d files.' % num_files)
 except KeyboardInterrupt:
-logging.info("User interrupted. Aborting.")
+logging.info('User interrupted. Aborting.')
 raise
 except EnvironmentError, e:
-logging.error("An error occurred processing file '%s': %s. Aborting.",
+logging.error('An error occurred processing file \'%s\': %s. Aborting.',
 path, e)
 raise
 try:
 missing_files = self.Begin()
 if missing_files:
-StatusUpdate("Uploading %d files." % len(missing_files))
+StatusUpdate('Uploading %d files.' % len(missing_files))
 num_files = 0
 for missing_file in missing_files:
-logging.info("Uploading file '%s'" % missing_file)
+logging.info('Uploading file \'%s\'' % missing_file)
 file_handle = openfunc(missing_file)
 try:
 self.UploadFile(missing_file, file_handle)
 finally:
 file_handle.close()
 num_files += 1
 if num_files % 500 == 0:
-StatusUpdate("Uploaded %d files." % num_files)
+StatusUpdate('Uploaded %d files.' % num_files)
 self.Commit()
 except KeyboardInterrupt:
-logging.info("User interrupted. Aborting.")
+logging.info('User interrupted. Aborting.')
 self.Rollback()
 raise
 except:
-logging.exception("An unexpected error occurred. Aborting.")
+logging.exception('An unexpected error occurred. Aborting.')
 self.Rollback()
 raise
-logging.info("Done!")
+logging.info('Done!')
 def FileIterator(base, separator=os.path.sep):
 """Walks a directory tree, returning all the files. Follows symlinks.
 separator: Path separator used by the running system's platform.
 Yields:
 Paths of files found, relative to base.
 """
-dirs = [""]
+dirs = ['']
 while dirs:
 current_dir = dirs.pop()
 for entry in os.listdir(os.path.join(base, current_dir)):
 name = os.path.join(current_dir, entry)
 fullname = os.path.join(base, name)
 if os.path.isfile(fullname):
-if separator == "\\":
+if separator == '\\':
-name = name.replace("\\", "/")
+name = name.replace('\\', '/')
 yield name
 elif os.path.isdir(fullname):
 dirs.append(name)
 get_platform: Used for testing.
 Returns:
 String containing the 'user-agent' header value, which includes the SDK
 version, the platform information, and the version of Python;
-e.g., "appcfg_py/1.0.1 Darwin/9.2.0 Python/2.5.2".
+e.g., 'appcfg_py/1.0.1 Darwin/9.2.0 Python/2.5.2'.
 """
 product_tokens = []
-sdk_name = os.environ.get("APPCFG_SDK_NAME")
+sdk_name = os.environ.get('APPCFG_SDK_NAME')
 if sdk_name:
 product_tokens.append(sdk_name)
 else:
 version = get_version()
 if version is None:
-release = "unknown"
+release = 'unknown'
 else:
-release = version["release"]
+release = version['release']
-product_tokens.append("appcfg_py/%s" % release)
+product_tokens.append('appcfg_py/%s' % release)
 product_tokens.append(get_platform())
-python_version = ".".join(str(i) for i in sys.version_info)
+python_version = '.'.join(str(i) for i in sys.version_info)
-product_tokens.append("Python/%s" % python_version)
+product_tokens.append('Python/%s' % python_version)
-return " ".join(product_tokens)
+return ' '.join(product_tokens)
 def GetSourceName(get_version=GetVersionObject):
 """Gets the name of this source version."""
 version = get_version()
 if version is None:
-release = "unknown"
+release = 'unknown'
 else:
-release = version["release"]
+release = version['release']
-return "Google-appcfg-%s" % (release,)
+return 'Google-appcfg-%s' % (release,)
 class AppCfgApp(object):
 """Singleton class to wrap AppCfg tool functionality.
 self.options, self.args = self.parser.parse_args(argv[1:])
 if len(self.args) < 1:
 self._PrintHelpAndExit()
 if self.args[0] not in self.actions:
-self.parser.error("Unknown action '%s'\n%s" %
+self.parser.error('Unknown action \'%s\'\n%s' %
 (self.args[0], self.parser.get_description()))
 action_name = self.args.pop(0)
 self.action = self.actions[action_name]
 self.parser, self.options = self._MakeSpecificParser(self.action)
 def Run(self):
 """Executes the requested action.
 Catches any HTTPErrors raised by the action and prints them to stderr.
+Returns:
+1 on error, 0 if successful.
 """
 try:
 self.action(self)
 except urllib2.HTTPError, e:
 body = e.read()
-print >>self.error_fh, ("Error %d: --- begin server output ---\n"
+print >>self.error_fh, ('Error %d: --- begin server output ---\n'
-"%s\n--- end server output ---" %
+'%s\n--- end server output ---' %
-(e.code, body.rstrip("\n")))
+(e.code, body.rstrip('\n')))
 return 1
 except yaml_errors.EventListenerError, e:
-print >>self.error_fh, ("Error parsing yaml file:\n%s" % e)
+print >>self.error_fh, ('Error parsing yaml file:\n%s' % e)
 return 1
 return 0
 def _GetActionDescriptions(self):
 """Returns a formatted string containing the short_descs for all actions."""
 action_names = self.actions.keys()
 action_names.sort()
-desc = ""
+desc = ''
 for action_name in action_names:
-desc += "  %s: %s\n" % (action_name, self.actions[action_name].short_desc)
+desc += '  %s: %s\n' % (action_name, self.actions[action_name].short_desc)
 return desc
 def _GetOptionParser(self):
 """Creates an OptionParser with generic usage and description strings.
 class Formatter(optparse.IndentedHelpFormatter):
 """Custom help formatter that does not reformat the description."""
 def format_description(self, description):
 """Very simple formatter."""
-return description + "\n"
+return description + '\n'
 desc = self._GetActionDescriptions()
-desc = ("Action must be one of:\n%s"
+desc = ('Action must be one of:\n%s'
-"Use 'help <action>' for a detailed description.") % desc
+'Use \'help <action>\' for a detailed description.') % desc
-parser = self.parser_class(usage="%prog [options] <action>",
+parser = self.parser_class(usage='%prog [options] <action>',
 description=desc,
 formatter=Formatter(),
-conflict_handler="resolve")
+conflict_handler='resolve')
-parser.add_option("-h", "--help", action="store_true",
+parser.add_option('-h', '--help', action='store_true',
-dest="help", help="Show the help message and exit.")
+dest='help', help='Show the help message and exit.')
-parser.add_option("-q", "--quiet", action="store_const", const=0,
+parser.add_option('-q', '--quiet', action='store_const', const=0,
-dest="verbose", help="Print errors only.")
+dest='verbose', help='Print errors only.')
-parser.add_option("-v", "--verbose", action="store_const", const=2,
+parser.add_option('-v', '--verbose', action='store_const', const=2,
-dest="verbose", default=1,
+dest='verbose', default=1,
-help="Print info level logs.")
+help='Print info level logs.')
-parser.add_option("--noisy", action="store_const", const=3,
+parser.add_option('--noisy', action='store_const', const=3,
-dest="verbose", help="Print all logs.")
+dest='verbose', help='Print all logs.')
-parser.add_option("-s", "--server", action="store", dest="server",
+parser.add_option('-s', '--server', action='store', dest='server',
-default="appengine.google.com",
+default='appengine.google.com',
-metavar="SERVER", help="The server to connect to.")
+metavar='SERVER', help='The server to connect to.')
-parser.add_option("--secure", action="store_true", dest="secure",
+parser.add_option('--secure', action='store_true', dest='secure',
 default=False,
-help="Use SSL when communicating with the server.")
+help='Use SSL when communicating with the server.')
-parser.add_option("-e", "--email", action="store", dest="email",
+parser.add_option('-e', '--email', action='store', dest='email',
-metavar="EMAIL", default=None,
+metavar='EMAIL', default=None,
-help="The username to use. Will prompt if omitted.")
+help='The username to use. Will prompt if omitted.')
-parser.add_option("-H", "--host", action="store", dest="host",
+parser.add_option('-H', '--host', action='store', dest='host',
-metavar="HOST", default=None,
+metavar='HOST', default=None,
-help="Overrides the Host header sent with all RPCs.")
+help='Overrides the Host header sent with all RPCs.')
-parser.add_option("--no_cookies", action="store_false",
+parser.add_option('--no_cookies', action='store_false',
-dest="save_cookies", default=True,
+dest='save_cookies', default=True,
-help="Do not save authentication cookies to local disk.")
+help='Do not save authentication cookies to local disk.')
-parser.add_option("--passin", action="store_true",
+parser.add_option('--passin', action='store_true',
-dest="passin", default=False,
+dest='passin', default=False,
-help="Read the login password from stdin.")
+help='Read the login password from stdin.')
 return parser
 def _MakeSpecificParser(self, action):
 """Creates a new parser with documentation specific to 'action'.
 parser: An instance of OptionsParser customized to 'action'.
 options: The command line options after re-parsing.
 """
 parser = self._GetOptionParser()
 parser.set_usage(action.usage)
-parser.set_description("%s\n%s" % (action.short_desc, action.long_desc))
+parser.set_description('%s\n%s' % (action.short_desc, action.long_desc))
 action.options(self, parser)
 options, unused_args = parser.parse_args(self.argv[1:])
 return parser, options
 def _PrintHelpAndExit(self, exit_code=2):
 def GetUserCredentials():
 """Prompts the user for a username and password."""
 email = self.options.email
 if email is None:
-email = self.raw_input_fn("Email: ")
+email = self.raw_input_fn('Email: ')
-password_prompt = "Password for %s: " % email
+password_prompt = 'Password for %s: ' % email
 if self.options.passin:
 password = self.raw_input_fn(password_prompt)
 else:
 password = self.password_input_fn(password_prompt)
 return (email, password)
-if self.options.host and self.options.host == "localhost":
+if self.options.host and self.options.host == 'localhost':
 email = self.options.email
 if email is None:
-email = "test@example.com"
+email = 'test@example.com'
-logging.info("Using debug user %s.  Override with --email" % email)
+logging.info('Using debug user %s.  Override with --email' % email)
 server = self.rpc_server_class(
 self.options.server,
-lambda: (email, "password"),
+lambda: (email, 'password'),
 GetUserAgent(),
 GetSourceName(),
 host_override=self.options.host,
 save_cookies=self.options.save_cookies)
 server.authenticated = True
 return self.rpc_server_class(self.options.server, GetUserCredentials,
 GetUserAgent(), GetSourceName(),
 host_override=self.options.host,
 save_cookies=self.options.save_cookies,
 auth_tries=auth_tries,
-account_type="HOSTED_OR_GOOGLE",
+account_type='HOSTED_OR_GOOGLE',
 secure=self.options.secure)
 def _FindYaml(self, basepath, file_name):
 """Find yaml files in application directory.
 Returns:
 Path to located yaml file if one exists, else None.
 """
 if not os.path.isdir(basepath):
-self.parser.error("Not a directory: %s" % basepath)
+self.parser.error('Not a directory: %s' % basepath)
-for yaml_file in (file_name + ".yaml", file_name + ".yml"):
+for yaml_file in (file_name + '.yaml', file_name + '.yml'):
 yaml_path = os.path.join(basepath, yaml_file)
 if os.path.isfile(yaml_path):
 return yaml_path
 return None
 basepath: the directory of the application.
 Returns:
 An AppInfoExternal object.
 """
-appyaml_filename = self._FindYaml(basepath, "app")
+appyaml_filename = self._FindYaml(basepath, 'app')
 if appyaml_filename is None:
-self.parser.error("Directory does not contain an app.yaml "
+self.parser.error('Directory does not contain an app.yaml '
-"configuration file.")
+'configuration file.')
-fh = open(appyaml_filename, "r")
+fh = open(appyaml_filename, 'r')
 try:
 appyaml = appinfo.LoadSingleAppInfo(fh)
 finally:
 fh.close()
 return appyaml
+def _ParseYamlFile(self, basepath, basename, parser):
+"""Parses the a yaml file.
+Args:
+basepath: the directory of the application.
+basename: the base name of the file (with the '.yaml' stripped off).
+parser: the function or method used to parse the file.
+Returns:
+A single parsed yaml file or None if the file does not exist.
+"""
+file_name = self._FindYaml(basepath, basename)
+if file_name is not None:
+fh = open(file_name, 'r')
+try:
+defns = parser(fh)
+finally:
+fh.close()
+return defns
+return None
 def _ParseIndexYaml(self, basepath):
 """Parses the index.yaml file.
 Args:
 basepath: the directory of the application.
 Returns:
 A single parsed yaml file or None if the file does not exist.
 """
-file_name = self._FindYaml(basepath, "index")
+return self._ParseYamlFile(basepath, 'index',
-if file_name is not None:
+datastore_index.ParseIndexDefinitions)
-fh = open(file_name, "r")
-try:
-index_defs = datastore_index.ParseIndexDefinitions(fh)
-finally:
-fh.close()
-return index_defs
-return None
 def _ParseCronYaml(self, basepath):
 """Parses the cron.yaml file.
 Args:
 basepath: the directory of the application.
 Returns:
-A CronInfoExternal object.
+A CronInfoExternal object or None if the file does not exist.
 """
-file_name = self._FindYaml(basepath, "cron")
+return self._ParseYamlFile(basepath, 'cron', croninfo.LoadSingleCron)
-if file_name is not None:
-fh = open(file_name, "r")
+def _ParseQueueYaml(self, basepath):
-try:
+"""Parses the queue.yaml file.
-cron_info = croninfo.LoadSingleCron(fh)
-finally:
+Args:
-fh.close()
+basepath: the directory of the application.
-return cron_info
-return None
+Returns:
+A CronInfoExternal object or None if the file does not exist.
+"""
+return self._ParseYamlFile(basepath, 'queue', queueinfo.LoadSingleQueue)
 def Help(self):
 """Prints help for a specific action.
 Expects self.args[0] to contain the name of the action in question.
 Exits the program after printing the help message.
 """
 if len(self.args) != 1 or self.args[0] not in self.actions:
-self.parser.error("Expected a single action argument. Must be one of:\n" +
+self.parser.error('Expected a single action argument. Must be one of:\n' +
 self._GetActionDescriptions())
 action = self.actions[self.args[0]]
 self.parser, unused_options = self._MakeSpecificParser(action)
 self._PrintHelpAndExit(exit_code=0)
 def Update(self):
 """Updates and deploys a new appversion."""
 if len(self.args) != 1:
-self.parser.error("Expected a single <directory> argument.")
+self.parser.error('Expected a single <directory> argument.')
 basepath = self.args[0]
 appyaml = self._ParseAppYaml(basepath)
 rpc_server = self._GetRpcServer()
 updatecheck = self.update_check_class(rpc_server, appyaml)
 updatecheck.CheckForUpdates()
 appversion = AppVersionUpload(rpc_server, appyaml)
 appversion.DoUpload(FileIterator(basepath), self.options.max_size,
-lambda path: open(os.path.join(basepath, path), "rb"))
+lambda path: open(os.path.join(basepath, path), 'rb'))
 index_defs = self._ParseIndexYaml(basepath)
 if index_defs:
 index_upload = IndexDefinitionUpload(rpc_server, appyaml, index_defs)
 try:
 index_upload.DoUpload()
 except urllib2.HTTPError, e:
-StatusUpdate("Error %d: --- begin server output ---\n"
+StatusUpdate('Error %d: --- begin server output ---\n'
-"%s\n--- end server output ---" %
+'%s\n--- end server output ---' %
-(e.code, e.read().rstrip("\n")))
+(e.code, e.read().rstrip('\n')))
 print >> self.error_fh, (
-"Your app was updated, but there was an error updating your "
+'Your app was updated, but there was an error updating your '
-"indexes. Please retry later with appcfg.py update_indexes.")
+'indexes. Please retry later with appcfg.py update_indexes.')
 cron_entries = self._ParseCronYaml(basepath)
 if cron_entries:
 cron_upload = CronEntryUpload(rpc_server, appyaml, cron_entries)
 cron_upload.DoUpload()
+queue_entries = self._ParseQueueYaml(basepath)
+if queue_entries:
+queue_upload = QueueEntryUpload(rpc_server, appyaml, queue_entries)
+queue_upload.DoUpload()
 def _UpdateOptions(self, parser):
 """Adds update-specific options to 'parser'.
 Args:
 parser: An instance of OptionsParser.
 """
-parser.add_option("-S", "--max_size", type="int", dest="max_size",
+parser.add_option('-S', '--max_size', type='int', dest='max_size',
-default=10485760, metavar="SIZE",
+default=10485760, metavar='SIZE',
-help="Maximum size of a file to upload.")
+help='Maximum size of a file to upload.')
 def VacuumIndexes(self):
 """Deletes unused indexes."""
 if len(self.args) != 1:
-self.parser.error("Expected a single <directory> argument.")
+self.parser.error('Expected a single <directory> argument.')
 basepath = self.args[0]
 config = self._ParseAppYaml(basepath)
 index_defs = self._ParseIndexYaml(basepath)
 """Adds vacuum_indexes-specific options to 'parser'.
 Args:
 parser: An instance of OptionsParser.
 """
-parser.add_option("-f", "--force", action="store_true", dest="force_delete",
+parser.add_option('-f', '--force', action='store_true', dest='force_delete',
 default=False,
-help="Force deletion without being prompted.")
+help='Force deletion without being prompted.')
 def UpdateCron(self):
 """Updates any new or changed cron definitions."""
 if len(self.args) != 1:
-self.parser.error("Expected a single <directory> argument.")
+self.parser.error('Expected a single <directory> argument.')
 basepath = self.args[0]
 appyaml = self._ParseAppYaml(basepath)
 rpc_server = self._GetRpcServer()
 cron_upload.DoUpload()
 def UpdateIndexes(self):
 """Updates indexes."""
 if len(self.args) != 1:
-self.parser.error("Expected a single <directory> argument.")
+self.parser.error('Expected a single <directory> argument.')
 basepath = self.args[0]
 appyaml = self._ParseAppYaml(basepath)
 rpc_server = self._GetRpcServer()
 index_defs = self._ParseIndexYaml(basepath)
 if index_defs:
 index_upload = IndexDefinitionUpload(rpc_server, appyaml, index_defs)
 index_upload.DoUpload()
+def UpdateQueues(self):
+"""Updates any new or changed task queue definitions."""
+if len(self.args) != 1:
+self.parser.error('Expected a single <directory> argument.')
+basepath = self.args[0]
+appyaml = self._ParseAppYaml(basepath)
+rpc_server = self._GetRpcServer()
+queue_entries = self._ParseQueueYaml(basepath)
+if queue_entries:
+queue_upload = QueueEntryUpload(rpc_server, appyaml, queue_entries)
+queue_upload.DoUpload()
 def Rollback(self):
 """Does a rollback of any existing transaction for this app version."""
 if len(self.args) != 1:
-self.parser.error("Expected a single <directory> argument.")
+self.parser.error('Expected a single <directory> argument.')
 basepath = self.args[0]
 appyaml = self._ParseAppYaml(basepath)
 appversion = AppVersionUpload(self._GetRpcServer(), appyaml)
 def RequestLogs(self):
 """Write request logs to a file."""
 if len(self.args) != 2:
 self.parser.error(
-"Expected a <directory> argument and an <output_file> argument.")
+'Expected a <directory> argument and an <output_file> argument.')
 if (self.options.severity is not None and
 not 0 <= self.options.severity <= MAX_LOG_LEVEL):
 self.parser.error(
-"Severity range is 0 (DEBUG) through %s (CRITICAL)." % MAX_LOG_LEVEL)
+'Severity range is 0 (DEBUG) through %s (CRITICAL).' % MAX_LOG_LEVEL)
 if self.options.num_days is None:
 self.options.num_days = int(not self.options.append)
 basepath = self.args[0]
 appyaml = self._ParseAppYaml(basepath)
 logs_requester = LogsRequester(rpc_server, appyaml, self.args[1],
 self.options.num_days,
 self.options.append,
 self.options.severity,
 time.time(),
-self.options.vhost)
+self.options.vhost,
+self.options.include_vhost)
 logs_requester.DownloadLogs()
 def _RequestLogsOptions(self, parser):
 """Adds request_logs-specific options to 'parser'.
 Args:
 parser: An instance of OptionsParser.
 """
-parser.add_option("-n", "--num_days", type="int", dest="num_days",
+parser.add_option('-n', '--num_days', type='int', dest='num_days',
-action="store", default=None,
+action='store', default=None,
-help="Number of days worth of log data to get. "
+help='Number of days worth of log data to get. '
-"The cut-off point is midnight UTC. "
+'The cut-off point is midnight UTC. '
-"Use 0 to get all available logs. "
+'Use 0 to get all available logs. '
-"Default is 1, unless --append is also given; "
+'Default is 1, unless --append is also given; '
-"then the default is 0.")
+'then the default is 0.')
-parser.add_option("-a", "--append", dest="append",
+parser.add_option('-a', '--append', dest='append',
-action="store_true", default=False,
+action='store_true', default=False,
-help="Append to existing file.")
+help='Append to existing file.')
-parser.add_option("--severity", type="int", dest="severity",
+parser.add_option('--severity', type='int', dest='severity',
-action="store", default=None,
+action='store', default=None,
-help="Severity of app-level log messages to get. "
+help='Severity of app-level log messages to get. '
-"The range is 0 (DEBUG) through 4 (CRITICAL). "
+'The range is 0 (DEBUG) through 4 (CRITICAL). '
-"If omitted, only request logs are returned.")
+'If omitted, only request logs are returned.')
-parser.add_option("--vhost", type="string", dest="vhost",
+parser.add_option('--vhost', type='string', dest='vhost',
-action="store", default=None,
+action='store', default=None,
-help="The virtual host of log messages to get. "
+help='The virtual host of log messages to get. '
-"If omitted, all log messages are returned.")
+'If omitted, all log messages are returned.')
+parser.add_option('--include_vhost', dest='include_vhost',
+action='store_true', default=False,
+help='Include virtual host in log messages.')
 def CronInfo(self, now=None, output=sys.stdout):
 """Displays information about cron definitions.
 Args:
 now: used for testing.
 output: Used for testing.
 """
 if len(self.args) != 1:
-self.parser.error("Expected a single <directory> argument.")
+self.parser.error('Expected a single <directory> argument.')
 if now is None:
 now = datetime.datetime.now()
 basepath = self.args[0]
 cron_entries = self._ParseCronYaml(basepath)
 if cron_entries and cron_entries.cron:
 for entry in cron_entries.cron:
 description = entry.description
 if not description:
-description = "<no description>"
+description = '<no description>'
-print >>output, "\n%s:\nURL: %s\nSchedule: %s" % (description,
+print >>output, '\n%s:\nURL: %s\nSchedule: %s' % (description,
 entry.url,
 entry.schedule)
 schedule = groctimespecification.GrocTimeSpecification(entry.schedule)
 matches = schedule.GetMatches(now, self.options.num_runs)
 for match in matches:
-print >>output, "%s, %s from now" % (
+print >>output, '%s, %s from now' % (
-match.strftime("%Y-%m-%d %H:%M:%S"), match - now)
+match.strftime('%Y-%m-%d %H:%M:%S'), match - now)
 def _CronInfoOptions(self, parser):
 """Adds cron_info-specific options to 'parser'.
 Args:
 parser: An instance of OptionsParser.
 """
-parser.add_option("-n", "--num_runs", type="int", dest="num_runs",
+parser.add_option('-n', '--num_runs', type='int', dest='num_runs',
-action="store", default=5,
+action='store', default=5,
-help="Number of runs of each cron job to display"
+help='Number of runs of each cron job to display'
-"Default is 5")
+'Default is 5')
 def _CheckRequiredLoadOptions(self):
 """Checks that upload/download options are present."""
-for option in ["filename", "kind", "config_file"]:
+for option in ['filename', 'kind', 'config_file']:
 if getattr(self.options, option) is None:
-self.parser.error("Option '%s' is required." % option)
+self.parser.error('Option \'%s\' is required.' % option)
 if not self.options.url:
-self.parser.error("You must have google.appengine.ext.remote_api.handler "
+self.parser.error('You must have google.appengine.ext.remote_api.handler '
-"assigned to an endpoint in app.yaml, or provide "
+'assigned to an endpoint in app.yaml, or provide '
-"the url of the handler via the 'url' option.")
+'the url of the handler via the \'url\' option.')
 def InferRemoteApiUrl(self, appyaml):
 """Uses app.yaml to determine the remote_api endpoint.
 Args:
 Returns:
 The url of the remote_api endpoint as a string, or None
 """
 handlers = appyaml.handlers
-handler_suffix = "remote_api/handler.py"
+handler_suffix = 'remote_api/handler.py'
 app_id = appyaml.application
 for handler in handlers:
-if hasattr(handler, "script") and handler.script:
+if hasattr(handler, 'script') and handler.script:
 if handler.script.endswith(handler_suffix):
 server = self.options.server
-if server == "appengine.google.com":
+if server == 'appengine.google.com':
-return "http://%s.appspot.com%s" % (app_id, handler.url)
+return 'http://%s.appspot.com%s' % (app_id, handler.url)
 else:
-return "http://%s%s" % (server, handler.url)
+return 'http://%s%s' % (server, handler.url)
 return None
 def RunBulkloader(self, arg_dict):
 """Invokes the bulkloader with the given keyword arguments.
 arg_dict: Dictionary of arguments to pass to bulkloader.Run().
 """
 try:
 import sqlite3
 except ImportError:
-logging.error("upload_data action requires SQLite3 and the python "
+logging.error('upload_data action requires SQLite3 and the python '
-"sqlite3 module (included in python since 2.5).")
+'sqlite3 module (included in python since 2.5).')
 sys.exit(1)
 sys.exit(bulkloader.Run(arg_dict))
 def _SetupLoad(self):
 """Performs common verification and set up for upload and download."""
 if len(self.args) != 1:
-self.parser.error("Expected <directory> argument.")
+self.parser.error('Expected <directory> argument.')
 basepath = self.args[0]
 appyaml = self._ParseAppYaml(basepath)
 self.options.app_id = appyaml.application
 self.options.url = url
 self._CheckRequiredLoadOptions()
 if self.options.batch_size < 1:
-self.parser.error("batch_size must be 1 or larger.")
+self.parser.error('batch_size must be 1 or larger.')
 if verbosity == 1:
 logging.getLogger().setLevel(logging.INFO)
 self.options.debug = False
 else:
 self.options.debug = True
 def _MakeLoaderArgs(self):
 return dict([(arg_name, getattr(self.options, arg_name, None)) for
 arg_name in (
-"app_id",
+'app_id',
-"url",
+'url',
-"filename",
+'filename',
-"batch_size",
+'batch_size',
-"kind",
+'kind',
-"num_threads",
+'num_threads',
-"bandwidth_limit",
+'bandwidth_limit',
-"rps_limit",
+'rps_limit',
-"http_limit",
+'http_limit',
-"db_filename",
+'db_filename',
-"config_file",
+'config_file',
-"auth_domain",
+'auth_domain',
-"has_header",
+'has_header',
-"loader_opts",
+'loader_opts',
-"log_file",
+'log_file',
-"passin",
+'passin',
-"email",
+'email',
-"debug",
+'debug',
-"exporter_opts",
+'exporter_opts',
-"result_db_filename",
+'result_db_filename',
 )])
 def PerformDownload(self, run_fn=None):
 """Performs a datastore download via the bulkloader.
 Args:
 """
 if run_fn is None:
 run_fn = self.RunBulkloader
 self._SetupLoad()
-StatusUpdate("Downloading data records.")
+StatusUpdate('Downloading data records.')
 args = self._MakeLoaderArgs()
 args['download'] = True
 args['has_header'] = False
 """
 if run_fn is None:
 run_fn = self.RunBulkloader
 self._SetupLoad()
-StatusUpdate("Uploading data records.")
+StatusUpdate('Uploading data records.')
 args = self._MakeLoaderArgs()
 args['download'] = False
 run_fn(args)
 """Adds options common to 'upload_data' and 'download_data'.
 Args:
 parser: An instance of OptionsParser.
 """
-parser.add_option("--filename", type="string", dest="filename",
+parser.add_option('--filename', type='string', dest='filename',
-action="store",
+action='store',
-help="The name of the file containing the input data."
+help='The name of the file containing the input data.'
-" (Required)")
+' (Required)')
-parser.add_option("--config_file", type="string", dest="config_file",
+parser.add_option('--config_file', type='string', dest='config_file',
-action="store",
+action='store',
-help="Name of the configuration file. (Required)")
+help='Name of the configuration file. (Required)')
-parser.add_option("--kind", type="string", dest="kind",
+parser.add_option('--kind', type='string', dest='kind',
-action="store",
+action='store',
-help="The kind of the entities to store. (Required)")
+help='The kind of the entities to store. (Required)')
-parser.add_option("--url", type="string", dest="url",
+parser.add_option('--url', type='string', dest='url',
-action="store",
+action='store',
-help="The location of the remote_api endpoint.")
+help='The location of the remote_api endpoint.')
-parser.add_option("--num_threads", type="int", dest="num_threads",
+parser.add_option('--num_threads', type='int', dest='num_threads',
-action="store", default=10,
+action='store', default=10,
-help="Number of threads to upload records with.")
+help='Number of threads to upload records with.')
-parser.add_option("--batch_size", type="int", dest="batch_size",
+parser.add_option('--batch_size', type='int', dest='batch_size',
-action="store", default=10,
+action='store', default=10,
-help="Number of records to post in each request.")
+help='Number of records to post in each request.')
-parser.add_option("--bandwidth_limit", type="int", dest="bandwidth_limit",
+parser.add_option('--bandwidth_limit', type='int', dest='bandwidth_limit',
-action="store", default=250000,
+action='store', default=250000,
-help="The maximum bytes/second bandwidth for transfers.")
+help='The maximum bytes/second bandwidth for transfers.')
-parser.add_option("--rps_limit", type="int", dest="rps_limit",
+parser.add_option('--rps_limit', type='int', dest='rps_limit',
-action="store", default=20,
+action='store', default=20,
-help="The maximum records/second for transfers.")
+help='The maximum records/second for transfers.')
-parser.add_option("--http_limit", type="int", dest="http_limit",
+parser.add_option('--http_limit', type='int', dest='http_limit',
-action="store", default=8,
+action='store', default=8,
-help="The maximum requests/second for transfers.")
+help='The maximum requests/second for transfers.')
-parser.add_option("--db_filename", type="string", dest="db_filename",
+parser.add_option('--db_filename', type='string', dest='db_filename',
-action="store",
+action='store',
-help="Name of the progress database file.")
+help='Name of the progress database file.')
-parser.add_option("--auth_domain", type="string", dest="auth_domain",
+parser.add_option('--auth_domain', type='string', dest='auth_domain',
-action="store", default="gmail.com",
+action='store', default='gmail.com',
-help="The name of the authorization domain to use.")
+help='The name of the authorization domain to use.')
-parser.add_option("--log_file", type="string", dest="log_file",
+parser.add_option('--log_file', type='string', dest='log_file',
-help="File to write bulkloader logs.  If not supplied "
+help='File to write bulkloader logs.  If not supplied '
-"then a new log file will be created, named: "
+'then a new log file will be created, named: '
-"bulkloader-log-TIMESTAMP.")
+'bulkloader-log-TIMESTAMP.')
 def _PerformUploadOptions(self, parser):
 """Adds 'upload_data' specific options to the 'parser' passed in.
 Args:
 parser: An instance of OptionsParser.
 """
 self._PerformLoadOptions(parser)
-parser.add_option("--has_header", dest="has_header",
+parser.add_option('--has_header', dest='has_header',
-action="store_true", default=False,
+action='store_true', default=False,
-help="Whether the first line of the input file should be"
+help='Whether the first line of the input file should be'
-" skipped")
+' skipped')
-parser.add_option("--loader_opts", type="string", dest="loader_opts",
+parser.add_option('--loader_opts', type='string', dest='loader_opts',
-help="A string to pass to the Loader.initialize method.")
+help='A string to pass to the Loader.initialize method.')
 def _PerformDownloadOptions(self, parser):
 """Adds 'download_data' specific options to the 'parser' passed in.
 Args:
 parser: An instance of OptionsParser.
 """
 self._PerformLoadOptions(parser)
-parser.add_option("--exporter_opts", type="string", dest="exporter_opts",
+parser.add_option('--exporter_opts', type='string', dest='exporter_opts',
-help="A string to pass to the Exporter.initialize method."
+help='A string to pass to the Exporter.initialize method.'
 )
-parser.add_option("--result_db_filename", type="string",
+parser.add_option('--result_db_filename', type='string',
-dest="result_db_filename",
+dest='result_db_filename',
-action="store",
+action='store',
-help="Database to write entities to for download.")
+help='Database to write entities to for download.')
 class Action(object):
 """Contains information about a command line action.
 Attributes:
 formatting will be preserved.
 options: A function that will add extra options to a given OptionParser
 object.
 """
-def __init__(self, function, usage, short_desc, long_desc="",
+def __init__(self, function, usage, short_desc, long_desc='',
 options=lambda obj, parser: None):
 """Initializer for the class attributes."""
 self.function = function
 self.usage = usage
 self.short_desc = short_desc
 def __call__(self, appcfg):
 """Invoke this Action on the specified AppCfg.
 This calls the function of the appropriate name on AppCfg, and
-respects polymophic overrides."""
+respects polymophic overrides.
+Args:
+appcfg: The appcfg to use.
+Returns:
+The result of the function call.
+"""
 method = getattr(appcfg, self.function)
 return method()
 actions = {
-"help": Action(
+'help': Action(
-function="Help",
+function='Help',
-usage="%prog help <action>",
+usage='%prog help <action>',
-short_desc="Print help for a specific action."),
+short_desc='Print help for a specific action.'),
-"update": Action(
+'update': Action(
-function="Update",
+function='Update',
-usage="%prog [options] update <directory>",
+usage='%prog [options] update <directory>',
 options=_UpdateOptions,
-short_desc="Create or update an app version.",
+short_desc='Create or update an app version.',
 long_desc="""
 Specify a directory that contains all of the files required by
 the app, and appcfg.py will create/update the app version referenced
 in the app.yaml file at the top level of that directory.  appcfg.py
 will follow symlinks and recursively upload all files to the server.
 Temporary or source control files (e.g. foo~, .svn/*) will be skipped."""),
-"update_cron": Action(
+'update_cron': Action(
-function="UpdateCron",
+function='UpdateCron',
-usage="%prog [options] update_cron <directory>",
+usage='%prog [options] update_cron <directory>',
-short_desc="Update application cron definitions.",
+short_desc='Update application cron definitions.',
 long_desc="""
 The 'update_cron' command will update any new, removed or changed cron
-definitions from the cron.yaml file."""),
+definitions from the optional cron.yaml file."""),
-"update_indexes": Action(
+'update_indexes': Action(
-function="UpdateIndexes",
+function='UpdateIndexes',
-usage="%prog [options] update_indexes <directory>",
+usage='%prog [options] update_indexes <directory>',
-short_desc="Update application indexes.",
+short_desc='Update application indexes.',
 long_desc="""
 The 'update_indexes' command will add additional indexes which are not currently
 in production as well as restart any indexes that were not completed."""),
-"vacuum_indexes": Action(
+'update_queues': Action(
-function="VacuumIndexes",
+function='UpdateQueues',
-usage="%prog [options] vacuum_indexes <directory>",
+usage='%prog [options] update_queues <directory>',
+short_desc='Update application task queue definitions.',
+long_desc="""
+The 'update_queue' command will update any new, removed or changed task queue
+definitions from the optional queue.yaml file."""),
+'vacuum_indexes': Action(
+function='VacuumIndexes',
+usage='%prog [options] vacuum_indexes <directory>',
 options=_VacuumIndexesOptions,
-short_desc="Delete unused indexes from application.",
+short_desc='Delete unused indexes from application.',
 long_desc="""
 The 'vacuum_indexes' command will help clean up indexes which are no longer
 in use.  It does this by comparing the local index configuration with
 indexes that are actually defined on the server.  If any indexes on the
 server do not exist in the index configuration file, the user is given the
 option to delete them."""),
-"rollback": Action(
+'rollback': Action(
-function="Rollback",
+function='Rollback',
-usage="%prog [options] rollback <directory>",
+usage='%prog [options] rollback <directory>',
-short_desc="Rollback an in-progress update.",
+short_desc='Rollback an in-progress update.',
 long_desc="""
 The 'update' command requires a server-side transaction.  Use 'rollback'
 if you get an error message about another transaction being in progress
 and you are sure that there is no such transaction."""),
-"request_logs": Action(
+'request_logs': Action(
-function="RequestLogs",
+function='RequestLogs',
-usage="%prog [options] request_logs <directory> <output_file>",
+usage='%prog [options] request_logs <directory> <output_file>',
 options=_RequestLogsOptions,
-short_desc="Write request logs in Apache common log format.",
+short_desc='Write request logs in Apache common log format.',
 long_desc="""
 The 'request_logs' command exports the request logs from your application
 to a file.  It will write Apache common log format records ordered
 chronologically.  If output file is '-' stdout will be written."""),
-"cron_info": Action(
+'cron_info': Action(
-function="CronInfo",
+function='CronInfo',
-usage="%prog [options] cron_info <directory>",
+usage='%prog [options] cron_info <directory>',
 options=_CronInfoOptions,
-short_desc="Display information about cron jobs.",
+short_desc='Display information about cron jobs.',
 long_desc="""
 The 'cron_info' command will display the next 'number' runs (default 5) for
 each cron job defined in the cron.yaml file."""),
-"upload_data": Action(
+'upload_data': Action(
-function="PerformUpload",
+function='PerformUpload',
-usage="%prog [options] upload_data <directory>",
+usage='%prog [options] upload_data <directory>',
 options=_PerformUploadOptions,
-short_desc="Upload data records to datastore.",
+short_desc='Upload data records to datastore.',
 long_desc="""
 The 'upload_data' command translates input records into datastore entities and
 uploads them into your application's datastore."""),
-"download_data": Action(
+'download_data': Action(
-function="PerformDownload",
+function='PerformDownload',
-usage="%prog [options] download_data <directory>",
+usage='%prog [options] download_data <directory>',
 options=_PerformDownloadOptions,
-short_desc="Download entities from datastore.",
+short_desc='Download entities from datastore.',
 long_desc="""
 The 'download_data' command downloads datastore entities and writes them to
 file as CSV or developer defined format."""),
 }
 def main(argv):
-logging.basicConfig(format=("%(asctime)s %(levelname)s %(filename)s:"
+logging.basicConfig(format=('%(asctime)s %(levelname)s %(filename)s:'
-"%(lineno)s %(message)s "))
+'%(lineno)s %(message)s '))
 try:
 result = AppCfgApp(argv).Run()
 if result:
 sys.exit(result)
 except KeyboardInterrupt:
-StatusUpdate("Interrupted.")
+StatusUpdate('Interrupted.')
 sys.exit(1)
-if __name__ == "__main__":
+if __name__ == '__main__':
 main(sys.argv)

changeset 2413	d0b7dac5325c
parent 2309	be1b94099f2d
child 2864	2e0b0af889be