+
−#!/usr/bin/env python
+
−#
+
−# Copyright 2007 Google Inc.
+
−#
+
−# Licensed under the Apache License, Version 2.0 (the "License");
+
−# you may not use this file except in compliance with the License.
+
−# You may obtain a copy of the License at
+
−#
+
−#     http://www.apache.org/licenses/LICENSE-2.0
+
−#
+
−# Unless required by applicable law or agreed to in writing, software
+
−# distributed under the License is distributed on an "AS IS" BASIS,
+
−# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
−# See the License for the specific language governing permissions and
+
−# limitations under the License.
+
−#
+
−
+
−"""Tool for deploying apps to an app server.
+
−
+
−Currently, the application only uploads new appversions. To do this, it first
+
−walks the directory tree rooted at the path the user specifies, adding all the
+
−files it finds to a list. It then uploads the application configuration
+
−(app.yaml) to the server using HTTP, followed by uploading each of the files.
+
−It then commits the transaction with another request.
+
−
+
−The bulk of this work is handled by the AppVersionUpload class, which exposes
+
−methods to add to the list of files, fetch a list of modified files, upload
+
−files, and commit or rollback the transaction.
+
−"""
+
−
+
−
+
−import cookielib
+
−import datetime
+
−import getpass
+
−import logging
+
−import mimetypes
+
−import optparse
+
−import os
+
−import re
+
−import sha
+
−import socket
+
−import sys
+
−import tempfile
+
−import time
+
−import urllib
+
−import urllib2
+
−
+
−import google
+
−from google.appengine.api import appinfo
+
−from google.appengine.api import validation
+
−from google.appengine.api import yaml_errors
+
−from google.appengine.api import yaml_object
+
−from google.appengine.datastore import datastore_index
+
−import yaml
+
−
+
−
+
−MAX_FILES_TO_CLONE = 100
+
−LIST_DELIMITER = "\n"
+
−TUPLE_DELIMITER = "|"
+
−
+
−VERSION_FILE = "../VERSION"
+
−
+
−UPDATE_CHECK_TIMEOUT = 3
+
−
+
−NAG_FILE = ".appcfg_nag"
+
−
+
−MAX_LOG_LEVEL = 4
+
−
+
−verbosity = 1
+
−
+
−
+
−def StatusUpdate(msg):
+
−  """Print a status message to stderr.
+
−
+
−  If 'verbosity' is greater than 0, print the message.
+
−
+
−  Args:
+
−    msg: The string to print.
+
−  """
+
−  if verbosity > 0:
+
−    print >>sys.stderr, msg
+
−
+
−
+
−class ClientLoginError(urllib2.HTTPError):
+
−  """Raised to indicate there was an error authenticating with ClientLogin."""
+
−
+
−  def __init__(self, url, code, msg, headers, args):
+
−    urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
+
−    self.args = args
+
−    self.reason = args["Error"]
+
−
+
−
+
−class AbstractRpcServer(object):
+
−  """Provides a common interface for a simple RPC server."""
+
−
+
−  def __init__(self, host, auth_function, host_override=None,
+
−               extra_headers=None, save_cookies=False):
+
−    """Creates a new HttpRpcServer.
+
−
+
−    Args:
+
−      host: The host to send requests to.
+
−      auth_function: A function that takes no arguments and returns an
+
−        (email, password) tuple when called. Will be called if authentication
+
−        is required.
+
−      host_override: The host header to send to the server (defaults to host).
+
−      extra_headers: A dict of extra headers to append to every request. Values
+
−        supplied here will override other default headers that are supplied.
+
−      save_cookies: If True, save the authentication cookies to local disk.
+
−        If False, use an in-memory cookiejar instead.  Subclasses must
+
−        implement this functionality.  Defaults to False.
+
−    """
+
−    self.host = host
+
−    self.host_override = host_override
+
−    self.auth_function = auth_function
+
−    self.authenticated = False
+
−
+
−    self.extra_headers = {
+
−      "User-agent": GetUserAgent()
+
−    }
+
−    if extra_headers:
+
−      self.extra_headers.update(extra_headers)
+
−
+
−    self.save_cookies = save_cookies
+
−    self.cookie_jar = cookielib.MozillaCookieJar()
+
−    self.opener = self._GetOpener()
+
−    if self.host_override:
+
−      logging.info("Server: %s; Host: %s", self.host, self.host_override)
+
−    else:
+
−      logging.info("Server: %s", self.host)
+
−
+
−  def _GetOpener(self):
+
−    """Returns an OpenerDirector for making HTTP requests.
+
−
+
−    Returns:
+
−      A urllib2.OpenerDirector object.
+
−    """
+
−    raise NotImplemented()
+
−
+
−  def _CreateRequest(self, url, data=None):
+
−    """Creates a new urllib request."""
+
−    logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
+
−    req = urllib2.Request(url, data=data)
+
−    if self.host_override:
+
−      req.add_header("Host", self.host_override)
+
−    for key, value in self.extra_headers.iteritems():
+
−      req.add_header(key, value)
+
−    return req
+
−
+
−  def _GetAuthToken(self, email, password):
+
−    """Uses ClientLogin to authenticate the user, returning an auth token.
+
−
+
−    Args:
+
−      email:    The user's email address
+
−      password: The user's password
+
−
+
−    Raises:
+
−      ClientLoginError: If there was an error authenticating with ClientLogin.
+
−      HTTPError: If there was some other form of HTTP error.
+
−
+
−    Returns:
+
−      The authentication token returned by ClientLogin.
+
−    """
+
−    req = self._CreateRequest(
+
−        url="https://www.google.com/accounts/ClientLogin",
+
−        data=urllib.urlencode({
+
−            "Email": email,
+
−            "Passwd": password,
+
−            "service": "ah",
+
−            "source": "Google-appcfg-1.0",
+
−            "accountType": "HOSTED_OR_GOOGLE"
+
−        })
+
−    )
+
−    try:
+
−      response = self.opener.open(req)
+
−      response_body = response.read()
+
−      response_dict = dict(x.split("=")
+
−                           for x in response_body.split("\n") if x)
+
−      return response_dict["Auth"]
+
−    except urllib2.HTTPError, e:
+
−      if e.code == 403:
+
−        body = e.read()
+
−        response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
+
−        raise ClientLoginError(req.get_full_url(), e.code, e.msg,
+
−                               e.headers, response_dict)
+
−      else:
+
−        raise
+
−
+
−  def _GetAuthCookie(self, auth_token):
+
−    """Fetches authentication cookies for an authentication token.
+
−
+
−    Args:
+
−      auth_token: The authentication token returned by ClientLogin.
+
−
+
−    Raises:
+
−      HTTPError: If there was an error fetching the authentication cookies.
+
−    """
+
−    continue_location = "http://localhost/"
+
−    args = {"continue": continue_location, "auth": auth_token}
+
−    login_path = os.environ.get("APPCFG_LOGIN_PATH", "/_ah")
+
−    req = self._CreateRequest("http://%s%s/login?%s" %
+
−                              (self.host, login_path, urllib.urlencode(args)))
+
−    try:
+
−      response = self.opener.open(req)
+
−    except urllib2.HTTPError, e:
+
−      response = e
+
−    if (response.code != 302 or
+
−        response.info()["location"] != continue_location):
+
−      raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
+
−                              response.headers, response.fp)
+
−    self.authenticated = True
+
−
+
−  def _Authenticate(self):
+
−    """Authenticates the user.
+
−
+
−    The authentication process works as follows:
+
−     1) We get a username and password from the user
+
−     2) We use ClientLogin to obtain an AUTH token for the user
+
−        (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
+
−     3) We pass the auth token to /_ah/login on the server to obtain an
+
−        authentication cookie. If login was successful, it tries to redirect
+
−        us to the URL we provided.
+
−
+
−    If we attempt to access the upload API without first obtaining an
+
−    authentication cookie, it returns a 401 response and directs us to
+
−    authenticate ourselves with ClientLogin.
+
−    """
+
−    for i in range(3):
+
−      credentials = self.auth_function()
+
−      try:
+
−        auth_token = self._GetAuthToken(credentials[0], credentials[1])
+
−      except ClientLoginError, e:
+
−        if e.reason == "BadAuthentication":
+
−          print >>sys.stderr, "Invalid username or password."
+
−          continue
+
−        if e.reason == "CaptchaRequired":
+
−          print >>sys.stderr, (
+
−              "Please go to\n"
+
−              "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
+
−              "and verify you are a human.  Then try again.")
+
−          break;
+
−        if e.reason == "NotVerified":
+
−          print >>sys.stderr, "Account not verified."
+
−          break
+
−        if e.reason == "TermsNotAgreed":
+
−          print >>sys.stderr, "User has not agreed to TOS."
+
−          break
+
−        if e.reason == "AccountDeleted":
+
−          print >>sys.stderr, "The user account has been deleted."
+
−          break
+
−        if e.reason == "AccountDisabled":
+
−          print >>sys.stderr, "The user account has been disabled."
+
−          break
+
−        if e.reason == "ServiceDisabled":
+
−          print >>sys.stderr, ("The user's access to the service has been "
+
−                               "disabled.")
+
−          break
+
−        if e.reason == "ServiceUnavailable":
+
−          print >>sys.stderr, "The service is not available; try again later."
+
−          break
+
−        raise
+
−      self._GetAuthCookie(auth_token)
+
−      return
+
−
+
−  def Send(self, request_path, payload="",
+
−           content_type="application/octet-stream",
+
−           timeout=None,
+
−           **kwargs):
+
−    """Sends an RPC and returns the response.
+
−
+
−    Args:
+
−      request_path: The path to send the request to, eg /api/appversion/create.
+
−      payload: The body of the request, or None to send an empty request.
+
−      content_type: The Content-Type header to use.
+
−      timeout: timeout in seconds; default None i.e. no timeout.
+
−        (Note: for large requests on OS X, the timeout doesn't work right.)
+
−      kwargs: Any keyword arguments are converted into query string parameters.
+
−
+
−    Returns:
+
−      The response body, as a string.
+
−    """
+
−    if not self.authenticated:
+
−      self._Authenticate()
+
−
+
−    old_timeout = socket.getdefaulttimeout()
+
−    socket.setdefaulttimeout(timeout)
+
−    try:
+
−      tries = 0
+
−      while True:
+
−        tries += 1
+
−        args = dict(kwargs)
+
−        url = "http://%s%s?%s" % (self.host, request_path,
+
−                                  urllib.urlencode(args))
+
−        req = self._CreateRequest(url=url, data=payload)
+
−        req.add_header("Content-Type", content_type)
+
−        req.add_header("X-appcfg-api-version", "1")
+
−        try:
+
−          f = self.opener.open(req)
+
−          response = f.read()
+
−          f.close()
+
−          return response
+
−        except urllib2.HTTPError, e:
+
−          if tries > 3:
+
−            raise
+
−          elif e.code == 401:
+
−            self._Authenticate()
+
−          elif e.code >= 500 and e.code < 600:
+
−            continue
+
−          else:
+
−            raise
+
−    finally:
+
−      socket.setdefaulttimeout(old_timeout)
+
−
+
−
+
−class HttpRpcServer(AbstractRpcServer):
+
−  """Provides a simplified RPC-style interface for HTTP requests."""
+
−
+
−  DEFAULT_COOKIE_FILE_PATH = "~/.appcfg_cookies"
+
−
+
−  def _Authenticate(self):
+
−    """Save the cookie jar after authentication."""
+
−    super(HttpRpcServer, self)._Authenticate()
+
−    if self.cookie_jar.filename is not None and self.save_cookies:
+
−      StatusUpdate("Saving authentication cookies to %s" %
+
−                   self.cookie_jar.filename)
+
−      self.cookie_jar.save()
+
−
+
−  def _GetOpener(self):
+
−    """Returns an OpenerDirector that supports cookies and ignores redirects.
+
−
+
−    Returns:
+
−      A urllib2.OpenerDirector object.
+
−    """
+
−    opener = urllib2.OpenerDirector()
+
−    opener.add_handler(urllib2.ProxyHandler())
+
−    opener.add_handler(urllib2.UnknownHandler())
+
−    opener.add_handler(urllib2.HTTPHandler())
+
−    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
+
−    opener.add_handler(urllib2.HTTPSHandler())
+
−    opener.add_handler(urllib2.HTTPErrorProcessor())
+
−
+
−    if self.save_cookies:
+
−      self.cookie_jar.filename = os.path.expanduser(HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)
+
−
+
−      if os.path.exists(self.cookie_jar.filename):
+
−        try:
+
−          self.cookie_jar.load()
+
−          self.authenticated = True
+
−          StatusUpdate("Loaded authentication cookies from %s" %
+
−                       self.cookie_jar.filename)
+
−        except (OSError, IOError, cookielib.LoadError), e:
+
−          logging.debug("Could not load authentication cookies; %s: %s",
+
−                        e.__class__.__name__, e)
+
−          self.cookie_jar.filename = None
+
−      else:
+
−        try:
+
−          fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600)
+
−          os.close(fd)
+
−        except (OSError, IOError), e:
+
−          logging.debug("Could not create authentication cookies file; %s: %s",
+
−                        e.__class__.__name__, e)
+
−          self.cookie_jar.filename = None
+
−
+
−    opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
+
−    return opener
+
−
+
−
+
−def GetMimeTypeIfStaticFile(config, filename):
+
−  """Looks up the mime type for 'filename'.
+
−
+
−  Uses the handlers in 'config' to determine if the file should
+
−  be treated as a static file.
+
−
+
−  Args:
+
−    config: The app.yaml object to check the filename against.
+
−    filename: The name of the file.
+
−
+
−  Returns:
+
−    The mime type string.  For example, 'text/plain' or 'image/gif'.
+
−    None if this is not a static file.
+
−  """
+
−  for handler in config.handlers:
+
−    handler_type = handler.GetHandlerType()
+
−    if handler_type in ("static_dir", "static_files"):
+
−      if handler_type == "static_dir":
+
−        regex = os.path.join(re.escape(handler.GetHandler()), ".*")
+
−      else:
+
−        regex = handler.upload
+
−      if re.match(regex, filename):
+
−        if handler.mime_type is not None:
+
−          return handler.mime_type
+
−        else:
+
−          guess = mimetypes.guess_type(filename)[0]
+
−          if guess is None:
+
−            default = "application/octet-stream"
+
−            print >>sys.stderr, ("Could not guess mimetype for %s.  Using %s."
+
−                                 % (filename, default))
+
−            return default
+
−          return guess
+
−  return None
+
−
+
−
+
−def BuildClonePostBody(file_tuples):
+
−  """Build the post body for the /api/clone{files,blobs} urls.
+
−
+
−  Args:
+
−    file_tuples: A list of tuples.  Each tuple should contain the entries
+
−      appropriate for the endpoint in question.
+
−
+
−  Returns:
+
−    A string containing the properly delimited tuples.
+
−  """
+
−  file_list = []
+
−  for tup in file_tuples:
+
−    path = tup[0]
+
−    tup = tup[1:]
+
−    file_list.append(TUPLE_DELIMITER.join([path] + list(tup)))
+
−  return LIST_DELIMITER.join(file_list)
+
−
+
−
+
−class NagFile(validation.Validated):
+
−  """A validated YAML class to represent the user's nag preferences.
+
−
+
−  Attributes:
+
−    timestamp: The timestamp of the last nag.
+
−    opt_in: True if the user wants to check for updates on dev_appserver
+
−      start.  False if not.  May be None if we have not asked the user yet.
+
−  """
+
−
+
−  ATTRIBUTES = {
+
−    "timestamp": validation.TYPE_FLOAT,
+
−    "opt_in": validation.Optional(validation.TYPE_BOOL),
+
−  }
+
−
+
−  @staticmethod
+
−  def Load(nag_file):
+
−    """Load a single NagFile object where one and only one is expected.
+
−
+
−    Args:
+
−      nag_file: A file-like object or string containing the yaml data to parse.
+
−
+
−    Returns:
+
−      A NagFile instance.
+
−    """
+
−    return yaml_object.BuildSingleObject(NagFile, nag_file)
+
−
+
−
+
−def GetVersionObject(isfile=os.path.isfile, open_fn=open):
+
−  """Gets the version of the SDK by parsing the VERSION file.
+
−
+
−  Args:
+
−    isfile, open_fn: Used for testing.
+
−
+
−  Returns:
+
−    A Yaml object or None if the VERSION file does not exist.
+
−  """
+
−  version_filename = os.path.join(os.path.dirname(google.__file__),
+
−                                  VERSION_FILE)
+
−  if not isfile(version_filename):
+
−    logging.error("Could not find version file at %s", version_filename)
+
−    return None
+
−
+
−  version_fh = open_fn(version_filename, "r")
+
−  try:
+
−    version = yaml.safe_load(version_fh)
+
−  finally:
+
−    version_fh.close()
+
−
+
−  return version
+
−
+
−
+
−class UpdateCheck(object):
+
−  """Determines if the local SDK is the latest version.
+
−
+
−  Nags the user when there are updates to the SDK.  As the SDK becomes
+
−  more out of date, the language in the nagging gets stronger.  We
+
−  store a little yaml file in the user's home directory so that we nag
+
−  the user only once a week.
+
−
+
−  The yaml file has the following field:
+
−    'timestamp': Last time we nagged the user in seconds since the epoch.
+
−
+
−  Attributes:
+
−    server: An AbstractRpcServer instance used to check for the latest SDK.
+
−    config: The app's AppInfoExternal.  Needed to determine which api_version
+
−      the app is using.
+
−  """
+
−
+
−  def __init__(self,
+
−               server,
+
−               config,
+
−               isdir=os.path.isdir,
+
−               isfile=os.path.isfile,
+
−               open_fn=open):
+
−    """Create a new UpdateCheck.
+
−
+
−    Args:
+
−      server: The AbstractRpcServer to use.
+
−      config: The yaml object that specifies the configuration of this
+
−        application.
+
−
+
−    Args for testing:
+
−      isdir: Replacement for os.path.isdir.
+
−      isfile: Replacement for os.path.isfile.
+
−      open: Replacement for the open builtin.
+
−    """
+
−    self.server = server
+
−    self.config = config
+
−    self.isdir = isdir
+
−    self.isfile = isfile
+
−    self.open = open_fn
+
−
+
−  @staticmethod
+
−  def MakeNagFilename():
+
−    """Returns the filename for the nag file for this user."""
+
−    user_homedir = os.path.expanduser("~/")
+
−    if not os.path.isdir(user_homedir):
+
−      drive, tail = os.path.splitdrive(os.__file__)
+
−      if drive:
+
−        os.environ["HOMEDRIVE"] = drive
+
−
+
−    return os.path.expanduser("~/" + NAG_FILE)
+
−
+
−  def _ParseVersionFile(self):
+
−    """Parse the local VERSION file.
+
−
+
−    Returns:
+
−      A Yaml object or None if the file does not exist.
+
−    """
+
−    return GetVersionObject(isfile=self.isfile, open_fn=self.open)
+
−
+
−  def CheckSupportedVersion(self):
+
−    """Determines if the app's api_version is supported by the SDK.
+
−
+
−    Uses the api_version field from the AppInfoExternal to determine if
+
−    the SDK supports that api_version.
+
−
+
−    Raises:
+
−      SystemExit if the api_version is not supported.
+
−    """
+
−    version = self._ParseVersionFile()
+
−    if version is None:
+
−      logging.error("Could not determine if the SDK supports the api_version "
+
−                    "requested in app.yaml.")
+
−      return
+
−    if self.config.api_version not in version["api_versions"]:
+
−      logging.critical("The api_version specified in app.yaml (%s) is not "
+
−                       "supported by this release of the SDK.  The supported "
+
−                       "api_versions are %s.",
+
−                       self.config.api_version, version["api_versions"])
+
−      sys.exit(1)
+
−
+
−  def CheckForUpdates(self):
+
−    """Queries the server for updates and nags the user if appropriate.
+
−
+
−    Queries the server for the latest SDK version at the same time reporting
+
−    the local SDK version.  The server will respond with a yaml document
+
−    containing the fields:
+
−      "release": The name of the release (e.g. 1.2).
+
−      "timestamp": The time the release was created (YYYY-MM-DD HH:MM AM/PM TZ).
+
−      "api_versions": A list of api_version strings (e.g. ['1', 'beta']).
+
−
+
−    We will nag the user with increasing severity if:
+
−    - There is a new release.
+
−    - There is a new release with a new api_version.
+
−    - There is a new release that does not support the api_version named in
+
−      self.config.
+
−    """
+
−    version = self._ParseVersionFile()
+
−    if version is None:
+
−      logging.info("Skipping update check")
+
−      return
+
−    logging.info("Checking for updates to the SDK.")
+
−
+
−    try:
+
−      response = self.server.Send("/api/updatecheck",
+
−                                  timeout=UPDATE_CHECK_TIMEOUT,
+
−                                  release=version["release"],
+
−                                  timestamp=version["timestamp"],
+
−                                  api_versions=version["api_versions"])
+
−    except urllib2.URLError, e:
+
−      logging.info("Update check failed: %s", e)
+
−      return
+
−
+
−    latest = yaml.safe_load(response)
+
−    if latest["release"] == version["release"]:
+
−      logging.info("The SDK is up to date.")
+
−      return
+
−
+
−    api_versions = latest["api_versions"]
+
−    if self.config.api_version not in api_versions:
+
−      self._Nag(
+
−          "The api version you are using (%s) is obsolete!  You should\n"
+
−          "upgrade your SDK and test that your code works with the new\n"
+
−          "api version." % self.config.api_version,
+
−          latest, version, force=True)
+
−      return
+
−
+
−    if self.config.api_version != api_versions[len(api_versions) - 1]:
+
−      self._Nag(
+
−          "The api version you are using (%s) is deprecated. You should\n"
+
−          "upgrade your SDK to try the new functionality." %
+
−          self.config.api_version, latest, version)
+
−      return
+
−
+
−    self._Nag("There is a new release of the SDK available.",
+
−              latest, version)
+
−
+
−  def _ParseNagFile(self):
+
−    """Parses the nag file.
+
−
+
−    Returns:
+
−      A NagFile if the file was present else None.
+
−    """
+
−    nag_filename = UpdateCheck.MakeNagFilename()
+
−    if self.isfile(nag_filename):
+
−      fh = self.open(nag_filename, "r")
+
−      try:
+
−        nag = NagFile.Load(fh)
+
−      finally:
+
−        fh.close()
+
−      return nag
+
−    return None
+
−
+
−  def _WriteNagFile(self, nag):
+
−    """Writes the NagFile to the user's nag file.
+
−
+
−    If the destination path does not exist, this method will log an error
+
−    and fail silently.
+
−
+
−    Args:
+
−      nag: The NagFile to write.
+
−    """
+
−    nagfilename = UpdateCheck.MakeNagFilename()
+
−    try:
+
−      fh = self.open(nagfilename, "w")
+
−      try:
+
−        fh.write(nag.ToYAML())
+
−      finally:
+
−        fh.close()
+
−    except (OSError, IOError), e:
+
−      logging.error("Could not write nag file to %s. Error: %s", nagfilename, e)
+
−
+
−  def _Nag(self, msg, latest, version, force=False):
+
−    """Prints a nag message and updates the nag file's timestamp.
+
−
+
−    Because we don't want to nag the user everytime, we store a simple
+
−    yaml document in the user's home directory.  If the timestamp in this
+
−    doc is over a week old, we'll nag the user.  And when we nag the user,
+
−    we update the timestamp in this doc.
+
−
+
−    Args:
+
−      msg: The formatted message to print to the user.
+
−      latest: The yaml document received from the server.
+
−      version: The local yaml version document.
+
−      force: If True, always nag the user, ignoring the nag file.
+
−    """
+
−    nag = self._ParseNagFile()
+
−    if nag and not force:
+
−      last_nag = datetime.datetime.fromtimestamp(nag.timestamp)
+
−      if datetime.datetime.now() - last_nag < datetime.timedelta(weeks=1):
+
−        logging.debug("Skipping nag message")
+
−        return
+
−
+
−    if nag is None:
+
−      nag = NagFile()
+
−    nag.timestamp = time.time()
+
−    self._WriteNagFile(nag)
+
−
+
−    print "****************************************************************"
+
−    print msg
+
−    print "-----------"
+
−    print "Latest SDK:"
+
−    print yaml.dump(latest)
+
−    print "-----------"
+
−    print "Your SDK:"
+
−    print yaml.dump(version)
+
−    print "-----------"
+
−    print "Please visit http://code.google.com/appengine for the latest SDK"
+
−    print "****************************************************************"
+
−
+
−  def AllowedToCheckForUpdates(self, input_fn=raw_input):
+
−    """Determines if the user wants to check for updates.
+
−
+
−    On startup, the dev_appserver wants to check for updates to the SDK.
+
−    Because this action reports usage to Google when the user is not
+
−    otherwise communicating with Google (e.g. pushing a new app version),
+
−    the user must opt in.
+
−
+
−    If the user does not have a nag file, we will query the user and
+
−    save the response in the nag file.  Subsequent calls to this function
+
−    will re-use that response.
+
−
+
−    Returns:
+
−      True if the user wants to check for updates.  False otherwise.
+
−    """
+
−    nag = self._ParseNagFile()
+
−    if nag is None:
+
−      nag = NagFile()
+
−      nag.timestamp = time.time()
+
−
+
−    if nag.opt_in is None:
+
−      answer = input_fn("Allow dev_appserver to check for updates on startup? "
+
−                        "(Y/n): ")
+
−      answer = answer.strip().lower()
+
−      if answer == "n" or answer == "no":
+
−        print ("dev_appserver will not check for updates on startup.  To "
+
−               "change this setting, edit %s" % UpdateCheck.MakeNagFilename())
+
−        nag.opt_in = False
+
−      else:
+
−        print ("dev_appserver will check for updates on startup.  To change "
+
−               "this setting, edit %s" % UpdateCheck.MakeNagFilename())
+
−        nag.opt_in = True
+
−      self._WriteNagFile(nag)
+
−    return nag.opt_in
+
−
+
−
+
−class IndexDefinitionUpload(object):
+
−  """Provides facilities to upload index definitions to the hosting service."""
+
−
+
−  def __init__(self, server, config, definitions):
+
−    """Creates a new DatastoreIndexUpload.
+
−
+
−    Args:
+
−      server: The RPC server to use.  Should be an instance of HttpRpcServer
+
−        or TestRpcServer.
+
−      config: The AppInfoExternal object derived from the app.yaml file.
+
−      definitions: An IndexDefinitions object.
+
−    """
+
−    self.server = server
+
−    self.config = config
+
−    self.definitions = definitions
+
−
+
−  def DoUpload(self):
+
−    StatusUpdate("Uploading index definitions.")
+
−    self.server.Send("/api/datastore/index/add",
+
−                     app_id=self.config.application,
+
−                     version=self.config.version,
+
−                     payload=self.definitions.ToYAML())
+
−
+
−
+
−class IndexOperation(object):
+
−  """Provide facilities for writing Index operation commands."""
+
−
+
−  def __init__(self, server, config):
+
−    """Creates a new IndexOperation.
+
−
+
−    Args:
+
−      server: The RPC server to use.  Should be an instance of HttpRpcServer
+
−        or TestRpcServer.
+
−      config: appinfo.AppInfoExternal configuration object.
+
−    """
+
−    self.server = server
+
−    self.config = config
+
−
+
−  def DoDiff(self, definitions):
+
−    """Retrieve diff file from the server.
+
−
+
−    Args:
+
−      definitions: datastore_index.IndexDefinitions as loaded from users
+
−        index.yaml file.
+
−
+
−    Returns:
+
−      A pair of datastore_index.IndexDefinitions objects.  The first record
+
−      is the set of indexes that are present in the index.yaml file but missing
+
−      from the server.  The second record is the set of indexes that are
+
−      present on the server but missing from the index.yaml file (indicating
+
−      that these indexes should probably be vacuumed).
+
−    """
+
−    StatusUpdate("Fetching index definitions diff.")
+
−    response = self.server.Send("/api/datastore/index/diff",
+
−                                app_id=self.config.application,
+
−                                payload=definitions.ToYAML())
+
−    return datastore_index.ParseMultipleIndexDefinitions(response)
+
−
+
−  def DoDelete(self, definitions):
+
−    """Delete indexes from the server.
+
−
+
−    Args:
+
−      definitions: Index definitions to delete from datastore.
+
−
+
−    Returns:
+
−      A single datstore_index.IndexDefinitions containing indexes that were
+
−      not deleted, probably because they were already removed.  This may
+
−      be normal behavior as there is a potential race condition between fetching
+
−      the index-diff and sending deletion confirmation through.
+
−    """
+
−    StatusUpdate("Deleting selected index definitions.")
+
−    response = self.server.Send("/api/datastore/index/delete",
+
−                                app_id=self.config.application,
+
−                                payload=definitions.ToYAML())
+
−    return datastore_index.ParseIndexDefinitions(response)
+
−
+
−
+
−class VacuumIndexesOperation(IndexOperation):
+
−  """Provide facilities to request the deletion of datastore indexes."""
+
−
+
−  def __init__(self, server, config, force,
+
−               confirmation_fn=raw_input):
+
−    """Creates a new VacuumIndexesOperation.
+
−
+
−    Args:
+
−      server: The RPC server to use.  Should be an instance of HttpRpcServer
+
−        or TestRpcServer.
+
−      config: appinfo.AppInfoExternal configuration object.
+
−      force: True to force deletion of indexes, else False.
+
−      confirmation_fn: Function used for getting input form user.
+
−    """
+
−    super(VacuumIndexesOperation, self).__init__(server, config)
+
−    self.force = force
+
−    self.confirmation_fn = confirmation_fn
+
−
+
−  def GetConfirmation(self, index):
+
−    """Get confirmation from user to delete an index.
+
−
+
−    This method will enter an input loop until the user provides a
+
−    response it is expecting.  Valid input is one of three responses:
+
−
+
−      y: Confirm deletion of index.
+
−      n: Do not delete index.
+
−      a: Delete all indexes without asking for further confirmation.
+
−
+
−    If the user enters nothing at all, the default action is to skip
+
−    that index and do not delete.
+
−
+
−    If the user selects 'a', as a side effect, the 'force' flag is set.
+
−
+
−    Args:
+
−      index: Index to confirm.
+
−
+
−    Returns:
+
−      True if user enters 'y' or 'a'.  False if user enter 'n'.
+
−    """
+
−    while True:
+
−      print "This index is no longer defined in your index.yaml file."
+
−      print
+
−      print index.ToYAML()
+
−      print
+
−
+
−      confirmation = self.confirmation_fn(
+
−          "Are you sure you want to delete this index? (N/y/a): ")
+
−      confirmation = confirmation.strip().lower()
+
−
+
−      if confirmation == 'y':
+
−        return True
+
−      elif confirmation == 'n' or confirmation == '':
+
−        return False
+
−      elif confirmation == 'a':
+
−        self.force = True
+
−        return True
+
−      else:
+
−        print "Did not understand your response."
+
−
+
−  def DoVacuum(self, definitions):
+
−    """Vacuum indexes in datastore.
+
−
+
−    This method will query the server to determine which indexes are not
+
−    being used according to the user's local index.yaml file.  Once it has
+
−    made this determination, it confirms with the user which unused indexes
+
−    should be deleted.  Once confirmation for each index is receives, it
+
−    deletes those indexes.
+
−
+
−    Because another user may in theory delete the same indexes at the same
+
−    time as the user, there is a potential race condition.  In this rare cases,
+
−    some of the indexes previously confirmed for deletion will not be found.
+
−    The user is notified which indexes these were.
+
−
+
−    Args:
+
−      definitions: datastore_index.IndexDefinitions as loaded from users
+
−        index.yaml file.
+
−    """
+
−    new_indexes, unused_indexes = self.DoDiff(definitions)
+
−
+
−    deletions = datastore_index.IndexDefinitions(indexes=[])
+
−    if unused_indexes.indexes is not None:
+
−      for index in unused_indexes.indexes:
+
−        if self.force or self.GetConfirmation(index):
+
−          deletions.indexes.append(index)
+
−
+
−    if len(deletions.indexes) > 0:
+
−      not_deleted = self.DoDelete(deletions)
+
−
+
−      if not_deleted.indexes:
+
−        not_deleted_count = len(not_deleted.indexes)
+
−        if not_deleted_count == 1:
+
−          warning_message = ('An index was not deleted.  Most likely this is '
+
−                             'because it no longer exists.\n\n')
+
−        else:
+
−          warning_message = ('%d indexes were not deleted.  Most likely this '
+
−                             'is because they no longer exist.\n\n'
+
−                             % not_deleted_count)
+
−        for index in not_deleted.indexes:
+
−          warning_message = warning_message + index.ToYAML()
+
−        logging.warning(warning_message)
+
−
+
−
+
−class LogsRequester(object):
+
−  """Provide facilities to export request logs."""
+
−
+
−  def __init__(self, server, config, output_file,
+
−               num_days, append, severity, now):
+
−    """Constructor.
+
−
+
−    Args:
+
−      server: The RPC server to use.  Should be an instance of HttpRpcServer
+
−        or TestRpcServer.
+
−      config: appinfo.AppInfoExternal configuration object.
+
−      output_file: Output file name.
+
−      num_days: Number of days worth of logs to export; 0 for all available.
+
−      append: True if appending to an existing file.
+
−      severity: App log severity to request (0-4); None for no app logs.
+
−      now: POSIX timestamp used for calculating valid dates for num_days.
+
−    """
+
−    self.server = server
+
−    self.config = config
+
−    self.output_file = output_file
+
−    self.append = append
+
−    self.num_days = num_days
+
−    self.severity = severity
+
−    self.version_id = self.config.version + ".1"
+
−    self.sentinel = None
+
−    self.write_mode = "w"
+
−    if self.append:
+
−      self.sentinel = FindSentinel(self.output_file)
+
−      self.write_mode = "a"
+
−    self.valid_dates = None
+
−    if self.num_days:
+
−      patterns = []
+
−      for i in xrange(self.num_days):
+
−        then = time.gmtime(now - 24*3600 * i)
+
−        patterns.append(re.escape(time.strftime("%d/%m/%Y", then)))
+
−        patterns.append(re.escape(time.strftime("%d/%b/%Y", then)))
+
−      self.valid_dates = re.compile(r"[^[]+\[(" + "|".join(patterns) + r"):")
+
−
+
−  def DownloadLogs(self):
+
−    """Download the requested logs.
+
−
+
−    This will write the logs to the file designated by
+
−    self.output_file, or to stdout if the filename is '-'.
+
−    Multiple roundtrips to the server may be made.
+
−    """
+
−    StatusUpdate("Downloading request logs for %s %s." %
+
−                 (self.config.application, self.version_id))
+
−    tf = tempfile.TemporaryFile()
+
−    offset = None
+
−    try:
+
−      while True:
+
−        try:
+
−          offset = self.RequestLogLines(tf, offset)
+
−          if not offset:
+
−            break
+
−        except KeyboardInterrupt:
+
−          StatusUpdate("Keyboard interrupt; saving data downloaded so far.")
+
−          break
+
−      StatusUpdate("Copying request logs to %r." % self.output_file)
+
−      if self.output_file == "-":
+
−        of = sys.stdout
+
−      else:
+
−        try:
+
−          of = open(self.output_file, self.write_mode)
+
−        except IOError, err:
+
−          StatusUpdate("Can't write %r: %s." % (self.output_file, err))
+
−          sys.exit(1)
+
−      try:
+
−        line_count = CopyReversedLines(tf, of)
+
−      finally:
+
−        of.flush()
+
−        if of is not sys.stdout:
+
−          of.close()
+
−    finally:
+
−      tf.close()
+
−    StatusUpdate("Copied %d records." % line_count)
+
−
+
−  def RequestLogLines(self, tf, offset):
+
−    """Make a single roundtrip to the server.
+
−
+
−    Args:
+
−      tf: Writable binary stream to which the log lines returned by
+
−        the server are written, stripped of headers, and excluding
+
−        lines skipped due to self.sentinel or self.valid_dates filtering.
+
−      offset: Offset string for a continued request; None for the first.
+
−
+
−    Returns:
+
−      The offset string to be used for the next request, if another
+
−      request should be issued; or None, if not.
+
−    """
+
−    logging.info("Request with offset %r.", offset)
+
−    kwds = {'app_id': self.config.application,
+
−            'version': self.version_id,
+
−            'limit': 100,
+
−            }
+
−    if offset:
+
−      kwds['offset'] = offset
+
−    if self.severity is not None:
+
−      kwds['severity'] = str(self.severity)
+
−    response = self.server.Send("/api/request_logs", payload=None, **kwds)
+
−    response = response.replace("\r", "\0")
+
−    lines = response.splitlines()
+
−    logging.info("Received %d bytes, %d records.", len(response), len(lines))
+
−    offset = None
+
−    if lines and lines[0].startswith('#'):
+
−      match = re.match(r'^#\s*next_offset=(\S+)\s*$', lines[0])
+
−      del lines[0]
+
−      if match:
+
−        offset = match.group(1)
+
−    if lines and lines[-1].startswith('#'):
+
−      del lines[-1]
+
−    valid_dates = self.valid_dates
+
−    sentinel = self.sentinel
+
−    len_sentinel = None
+
−    if sentinel:
+
−      len_sentinel = len(sentinel)
+
−    for line in lines:
+
−      if ((sentinel and
+
−           line.startswith(sentinel) and
+
−           line[len_sentinel : len_sentinel+1] in ("", "\0")) or
+
−          (valid_dates and not valid_dates.match(line))):
+
−        return None
+
−      tf.write(line + '\n')
+
−    if not lines:
+
−      return None
+
−    return offset
+
−
+
−
+
−def CopyReversedLines(input, output, blocksize=2**16):
+
−  r"""Copy lines from input stream to output stream in reverse order.
+
−
+
−  As a special feature, null bytes in the input are turned into
+
−  newlines followed by tabs in the output, but these "sub-lines"
+
−  separated by null bytes are not reversed.  E.g. If the input is
+
−  "A\0B\nC\0D\n", the output is "C\n\tD\nA\n\tB\n".
+
−
+
−  Args:
+
−    input: A seekable stream open for reading in binary mode.
+
−    output: A stream open for writing; doesn't have to be seekable or binary.
+
−    blocksize: Optional block size for buffering, for unit testing.
+
−
+
−  Returns:
+
−    The number of lines copied.
+
−  """
+
−  line_count = 0
+
−  input.seek(0, 2)
+
−  last_block = input.tell() // blocksize
+
−  spillover = ""
+
−  for iblock in xrange(last_block + 1, -1, -1):
+
−    input.seek(iblock * blocksize)
+
−    data = input.read(blocksize)
+
−    lines = data.splitlines(True)
+
−    lines[-1:] = "".join(lines[-1:] + [spillover]).splitlines(True)
+
−    if lines and not lines[-1].endswith("\n"):
+
−      lines[-1] += "\n"
+
−    lines.reverse()
+
−    if lines and iblock > 0:
+
−      spillover = lines.pop()
+
−    if lines:
+
−      line_count += len(lines)
+
−      data = "".join(lines).replace("\0", "\n\t")
+
−      output.write(data)
+
−  return line_count
+
−
+
−
+
−def FindSentinel(filename, blocksize=2**16):
+
−  """Return the sentinel line from the output file.
+
−
+
−  Args:
+
−    filename: The filename of the output file.  (We'll read this file.)
+
−    blocksize: Optional block size for buffering, for unit testing.
+
−
+
−  Returns:
+
−    The contents of the last line in the file that doesn't start with
+
−    a tab, with its trailing newline stripped; or None if the file
+
−    couldn't be opened or no such line could be found by inspecting
+
−    the last 'blocksize' bytes of the file.
+
−  """
+
−  if filename == "-":
+
−    StatusUpdate("Can't combine --append with output to stdout.")
+
−    sys.exit(2)
+
−  try:
+
−    fp = open(filename, "rb")
+
−  except IOError, err:
+
−    StatusUpdate("Append mode disabled: can't read %r: %s." % (filename, err))
+
−    return None
+
−  try:
+
−    fp.seek(0, 2)
+
−    fp.seek(max(0, fp.tell() - blocksize))
+
−    lines = fp.readlines()
+
−    del lines[:1]
+
−    sentinel = None
+
−    for line in lines:
+
−      if not line.startswith("\t"):
+
−        sentinel = line
+
−    if not sentinel:
+
−      StatusUpdate("Append mode disabled: can't find sentinel in %r." %
+
−                   filename)
+
−      return None
+
−    return sentinel.rstrip("\n")
+
−  finally:
+
−    fp.close()
+
−
+
−
+
−class AppVersionUpload(object):
+
−  """Provides facilities to upload a new appversion to the hosting service.
+
−
+
−  Attributes:
+
−    server: The AbstractRpcServer to use for the upload.
+
−    config: The AppInfoExternal object derived from the app.yaml file.
+
−    app_id: The application string from 'config'.
+
−    version: The version string from 'config'.
+
−    files: A dictionary of files to upload to the server, mapping path to
+
−      hash of the file contents.
+
−    in_transaction: True iff a transaction with the server has started.
+
−      An AppVersionUpload can do only one transaction at a time.
+
−  """
+
−
+
−  def __init__(self, server, config):
+
−    """Creates a new AppVersionUpload.
+
−
+
−    Args:
+
−      server: The RPC server to use. Should be an instance of HttpRpcServer or
+
−        TestRpcServer.
+
−      config: An AppInfoExternal object that specifies the configuration for
+
−        this application.
+
−    """
+
−    self.server = server
+
−    self.config = config
+
−    self.app_id = self.config.application
+
−    self.version = self.config.version
+
−    self.files = {}
+
−    self.in_transaction = False
+
−
+
−  def _Hash(self, content):
+
−    """Compute the hash of the content.
+
−
+
−    Arg:
+
−      content: The data to hash as a string.
+
−
+
−    Returns:
+
−      The string representation of the hash.
+
−    """
+
−    h = sha.new(content).hexdigest()
+
−    return '%s_%s_%s_%s_%s' % (h[0:8], h[8:16], h[16:24], h[24:32], h[32:40])
+
−
+
−  def AddFile(self, path, file_handle):
+
−    """Adds the provided file to the list to be pushed to the server.
+
−
+
−    Args:
+
−      path: The path the file should be uploaded as.
+
−      file_handle: A stream containing data to upload.
+
−    """
+
−    assert not self.in_transaction, "Already in a transaction."
+
−    assert file_handle is not None
+
−
+
−    reason = appinfo.ValidFilename(path)
+
−    if reason != '':
+
−      logging.error(reason)
+
−      return
+
−
+
−    pos = file_handle.tell()
+
−    content_hash = self._Hash(file_handle.read())
+
−    file_handle.seek(pos, 0)
+
−
+
−    self.files[path] = content_hash
+
−
+
−  def Begin(self):
+
−    """Begins the transaction, returning a list of files that need uploading.
+
−
+
−    All calls to AddFile must be made before calling Begin().
+
−
+
−    Returns:
+
−      A list of pathnames for files that should be uploaded using UploadFile()
+
−      before Commit() can be called.
+
−    """
+
−    assert not self.in_transaction, "Already in a transaction."
+
−
+
−    StatusUpdate("Initiating update.")
+
−    self.server.Send("/api/appversion/create", app_id=self.app_id,
+
−                     version=self.version, payload=self.config.ToYAML())
+
−    self.in_transaction = True
+
−
+
−    files_to_clone = []
+
−    blobs_to_clone = []
+
−    for path, content_hash in self.files.iteritems():
+
−      mime_type = GetMimeTypeIfStaticFile(self.config, path)
+
−      if mime_type is not None:
+
−        blobs_to_clone.append((path, content_hash, mime_type))
+
−      else:
+
−        files_to_clone.append((path, content_hash))
+
−
+
−    files_to_upload = {}
+
−
+
−    def CloneFiles(url, files, file_type):
+
−      if len(files) == 0:
+
−        return
+
−
+
−      StatusUpdate("Cloning %d %s file%s." %
+
−                   (len(files), file_type, len(files) != 1 and "s" or ""))
+
−      for i in xrange(0, len(files), MAX_FILES_TO_CLONE):
+
−        if i > 0 and i % MAX_FILES_TO_CLONE == 0:
+
−          StatusUpdate("Cloned %d files." % i)
+
−
+
−        chunk = files[i:min(len(files), i + MAX_FILES_TO_CLONE)]
+
−        result = self.server.Send(url,
+
−                                  app_id=self.app_id, version=self.version,
+
−                                  payload=BuildClonePostBody(chunk))
+
−        if result:
+
−          files_to_upload.update(dict(
+
−              (f, self.files[f]) for f in result.split(LIST_DELIMITER)))
+
−
+
−    CloneFiles("/api/appversion/cloneblobs", blobs_to_clone, "static")
+
−    CloneFiles("/api/appversion/clonefiles", files_to_clone, "application")
+
−
+
−    logging.info('Files to upload: ' + str(files_to_upload))
+
−
+
−    self.files = files_to_upload
+
−    return sorted(files_to_upload.iterkeys())
+
−
+
−  def UploadFile(self, path, file_handle):
+
−    """Uploads a file to the hosting service.
+
−
+
−    Must only be called after Begin().
+
−    The path provided must be one of those that were returned by Begin().
+
−
+
−    Args:
+
−      path: The path the file is being uploaded as.
+
−      file_handle: A file-like object containing the data to upload.
+
−
+
−    Raises:
+
−      KeyError: The provided file is not amongst those to be uploaded.
+
−    """
+
−    assert self.in_transaction, "Begin() must be called before UploadFile()."
+
−    if path not in self.files:
+
−      raise KeyError("File '%s' is not in the list of files to be uploaded."
+
−                     % path)
+
−
+
−    del self.files[path]
+
−    mime_type = GetMimeTypeIfStaticFile(self.config, path)
+
−    if mime_type is not None:
+
−      self.server.Send("/api/appversion/addblob", app_id=self.app_id,
+
−                       version=self.version, path=path, content_type=mime_type,
+
−                       payload=file_handle.read())
+
−    else:
+
−      self.server.Send("/api/appversion/addfile", app_id=self.app_id,
+
−                       version=self.version, path=path,
+
−                       payload=file_handle.read())
+
−
+
−  def Commit(self):
+
−    """Commits the transaction, making the new app version available.
+
−
+
−    All the files returned by Begin() must have been uploaded with UploadFile()
+
−    before Commit() can be called.
+
−
+
−    Raises:
+
−      Exception: Some required files were not uploaded.
+
−    """
+
−    assert self.in_transaction, "Begin() must be called before Commit()."
+
−    if self.files:
+
−      raise Exception("Not all required files have been uploaded.")
+
−
+
−    StatusUpdate("Closing update.")
+
−    self.server.Send("/api/appversion/commit", app_id=self.app_id,
+
−                     version=self.version)
+
−    self.in_transaction = False
+
−
+
−  def Rollback(self):
+
−    """Rolls back the transaction if one is in progress."""
+
−    if not self.in_transaction:
+
−      return
+
−    StatusUpdate("Rolling back the update.")
+
−    self.server.Send("/api/appversion/rollback", app_id=self.app_id,
+
−                     version=self.version)
+
−    self.in_transaction = False
+
−    self.files = {}
+
−
+
−  def DoUpload(self, paths, max_size, openfunc):
+
−    """Uploads a new appversion with the given config and files to the server.
+
−
+
−    Args:
+
−      paths: An iterator that yields the relative paths of the files to upload.
+
−      max_size: The maximum size file to upload.
+
−      openfunc: A function that takes a path and returns a file-like object.
+
−    """
+
−    logging.info("Reading app configuration.")
+
−
+
−    try:
+
−      StatusUpdate("Scanning files on local disk.")
+
−      num_files = 0
+
−      for path in paths:
+
−        file_handle = openfunc(path)
+
−        try:
+
−          if self.config.skip_files.match(path):
+
−            logging.info("Ignoring file '%s': File matches ignore regex.",
+
−                         path)
+
−          else:
+
−            file_length = GetFileLength(file_handle)
+
−            if file_length > max_size:
+
−              logging.error("Ignoring file '%s': Too long "
+
−                            "(max %d bytes, file is %d bytes)",
+
−                            path, max_size, file_length)
+
−            else:
+
−              logging.info("Processing file '%s'", path)
+
−              self.AddFile(path, file_handle)
+
−        finally:
+
−          file_handle.close()
+
−        num_files += 1
+
−        if num_files % 500 == 0:
+
−          StatusUpdate("Scanned %d files." % num_files)
+
−    except KeyboardInterrupt:
+
−      logging.info("User interrupted. Aborting.")
+
−      raise
+
−    except EnvironmentError, e:
+
−      logging.error("An error occurred processing file '%s': %s. Aborting.",
+
−                    path, e)
+
−      raise
+
−
+
−    try:
+
−      missing_files = self.Begin()
+
−      if len(missing_files) > 0:
+
−        StatusUpdate("Uploading %d files." % len(missing_files))
+
−        num_files = 0
+
−        for missing_file in missing_files:
+
−          logging.info("Uploading file '%s'" % missing_file)
+
−          file_handle = openfunc(missing_file)
+
−          try:
+
−            self.UploadFile(missing_file, file_handle)
+
−          finally:
+
−            file_handle.close()
+
−          num_files += 1
+
−          if num_files % 500 == 0:
+
−            StatusUpdate("Uploaded %d files." % num_files)
+
−
+
−      self.Commit()
+
−    except KeyboardInterrupt:
+
−      logging.info("User interrupted. Aborting.")
+
−      self.Rollback()
+
−      raise
+
−    except:
+
−      logging.error("An unexpected error occurred. Aborting.")
+
−      self.Rollback()
+
−      raise
+
−
+
−    logging.info("Done!")
+
−
+
−
+
−def FileIterator(base, separator=os.path.sep):
+
−  """Walks a directory tree, returning all the files. Follows symlinks.
+
−
+
−  Args:
+
−    base: The base path to search for files under.
+
−    separator: Path separator used by the running system's platform.
+
−
+
−  Yields:
+
−    Paths of files found, relative to base.
+
−  """
+
−  dirs = [""]
+
−  while dirs:
+
−    current_dir = dirs.pop()
+
−    for entry in os.listdir(os.path.join(base, current_dir)):
+
−      name = os.path.join(current_dir, entry)
+
−      fullname = os.path.join(base, name)
+
−      if os.path.isfile(fullname):
+
−        if separator == "\\":
+
−          name = name.replace("\\", "/")
+
−        yield name
+
−      elif os.path.isdir(fullname):
+
−        dirs.append(name)
+
−
+
−
+
−def GetFileLength(fh):
+
−  """Returns the length of the file represented by fh.
+
−
+
−  This function is capable of finding the length of any seekable stream,
+
−  unlike os.fstat, which only works on file streams.
+
−
+
−  Args:
+
−    fh: The stream to get the length of.
+
−
+
−  Returns:
+
−    The length of the stream.
+
−  """
+
−  pos = fh.tell()
+
−  fh.seek(0, 2)
+
−  length = fh.tell()
+
−  fh.seek(pos, 0)
+
−  return length
+
−
+
−
+
−def GetPlatformToken(os_module=os, sys_module=sys, platform=sys.platform):
+
−  """Returns a 'User-agent' token for the host system platform.
+
−
+
−  Args:
+
−    os_module, sys_module, platform: Used for testing.
+
−
+
−  Returns:
+
−    String containing the platform token for the host system.
+
−  """
+
−  if hasattr(sys_module, "getwindowsversion"):
+
−    windows_version = sys_module.getwindowsversion()
+
−    version_info = ".".join(str(i) for i in windows_version[:4])
+
−    return platform + "/" + version_info
+
−  elif hasattr(os_module, "uname"):
+
−    uname = os_module.uname()
+
−    return "%s/%s" % (uname[0], uname[2])
+
−  else:
+
−    return "unknown"
+
−
+
−
+
−def GetUserAgent(get_version=GetVersionObject, get_platform=GetPlatformToken):
+
−  """Determines the value of the 'User-agent' header to use for HTTP requests.
+
−
+
−  If the 'APPCFG_SDK_NAME' environment variable is present, that will be
+
−  used as the first product token in the user-agent.
+
−
+
−  Args:
+
−    get_version, get_platform: Used for testing.
+
−
+
−  Returns:
+
−    String containing the 'user-agent' header value, which includes the SDK
+
−    version, the platform information, and the version of Python;
+
−    e.g., "appcfg_py/1.0.1 Darwin/9.2.0 Python/2.5.2".
+
−  """
+
−  product_tokens = []
+
−
+
−  sdk_name = os.environ.get("APPCFG_SDK_NAME")
+
−  if sdk_name:
+
−    product_tokens.append(sdk_name)
+
−  else:
+
−    version = get_version()
+
−    if version is None:
+
−      release = "unknown"
+
−    else:
+
−      release = version["release"]
+
−
+
−    product_tokens.append("appcfg_py/%s" % release)
+
−
+
−  product_tokens.append(get_platform())
+
−
+
−  python_version = ".".join(str(i) for i in sys.version_info)
+
−  product_tokens.append("Python/%s" % python_version)
+
−
+
−  return " ".join(product_tokens)
+
−
+
−
+
−class AppCfgApp(object):
+
−  """Singleton class to wrap AppCfg tool functionality.
+
−
+
−  This class is responsible for parsing the command line and executing
+
−  the desired action on behalf of the user.  Processing files and
+
−  communicating with the server is handled by other classes.
+
−
+
−  Attributes:
+
−    actions: A dictionary mapping action names to Action objects.
+
−    action: The Action specified on the command line.
+
−    parser: An instance of optparse.OptionParser.
+
−    options: The command line options parsed by 'parser'.
+
−    argv: The original command line as a list.
+
−    args: The positional command line args left over after parsing the options.
+
−    raw_input_fn: Function used for getting raw user input, like email.
+
−    password_input_fn: Function used for getting user password.
+
−    error_fh: Unexpected HTTPErrors are printed to this file handle.
+
−
+
−  Attributes for testing:
+
−    parser_class: The class to use for parsing the command line.  Because
+
−      OptionsParser will exit the program when there is a parse failure, it
+
−      is nice to subclass OptionsParser and catch the error before exiting.
+
−  """
+
−
+
−  def __init__(self, argv, parser_class=optparse.OptionParser,
+
−               rpc_server_class=HttpRpcServer,
+
−               raw_input_fn=raw_input,
+
−               password_input_fn=getpass.getpass,
+
−               error_fh=sys.stderr):
+
−    """Initializer.  Parses the cmdline and selects the Action to use.
+
−
+
−    Initializes all of the attributes described in the class docstring.
+
−    Prints help or error messages if there is an error parsing the cmdline.
+
−
+
−    Args:
+
−      argv: The list of arguments passed to this program.
+
−      parser_class: Options parser to use for this application.
+
−      rpc_server_class: RPC server class to use for this application.
+
−      raw_input_fn: Function used for getting user email.
+
−      password_input_fn: Function used for getting user password.
+
−      error_fh: Unexpected HTTPErrors are printed to this file handle.
+
−    """
+
−    self.parser_class = parser_class
+
−    self.argv = argv
+
−    self.rpc_server_class = rpc_server_class
+
−    self.raw_input_fn = raw_input_fn
+
−    self.password_input_fn = password_input_fn
+
−    self.error_fh = error_fh
+
−
+
−    self.parser = self._GetOptionParser()
+
−    for action in self.actions.itervalues():
+
−      action.options(self, self.parser)
+
−
+
−    self.options, self.args = self.parser.parse_args(argv[1:])
+
−
+
−    if len(self.args) < 1:
+
−      self._PrintHelpAndExit()
+
−    if self.args[0] not in self.actions:
+
−      self.parser.error("Unknown action '%s'\n%s" %
+
−                        (self.args[0], self.parser.get_description()))
+
−    action_name = self.args.pop(0)
+
−    self.action = self.actions[action_name]
+
−
+
−    self.parser, self.options = self._MakeSpecificParser(self.action)
+
−
+
−    if self.options.help:
+
−      self._PrintHelpAndExit()
+
−
+
−    if self.options.verbose == 2:
+
−      logging.getLogger().setLevel(logging.INFO)
+
−    elif self.options.verbose == 3:
+
−      logging.getLogger().setLevel(logging.DEBUG)
+
−
+
−    global verbosity
+
−    verbosity = self.options.verbose
+
−
+
−  def Run(self):
+
−    """Executes the requested action.
+
−
+
−    Catches any HTTPErrors raised by the action and prints them to stderr.
+
−    """
+
−    try:
+
−      self.action.function(self)
+
−    except urllib2.HTTPError, e:
+
−      body = e.read()
+
−      print >>self.error_fh, ("Error %d: --- begin server output ---\n"
+
−                              "%s\n--- end server output ---" %
+
−                              (e.code, body.rstrip("\n")))
+
−    except yaml_errors.EventListenerError, e:
+
−      print >>self.error_fh, ("Error parsing yaml file:\n%s" % e)
+
−
+
−  def _GetActionDescriptions(self):
+
−    """Returns a formatted string containing the short_descs for all actions."""
+
−    action_names = self.actions.keys()
+
−    action_names.sort()
+
−    desc = ""
+
−    for action_name in action_names:
+
−      desc += "  %s: %s\n" % (action_name, self.actions[action_name].short_desc)
+
−    return desc
+
−
+
−  def _GetOptionParser(self):
+
−    """Creates an OptionParser with generic usage and description strings.
+
−
+
−    Returns:
+
−      An OptionParser instance.
+
−    """
+
−
+
−    class Formatter(optparse.IndentedHelpFormatter):
+
−      """Custom help formatter that does not reformat the description."""
+
−      def format_description(self, description):
+
−        return description + "\n"
+
−
+
−    desc = self._GetActionDescriptions()
+
−    desc = ("Action must be one of:\n%s"
+
−            "Use 'help <action>' for a detailed description.") % desc
+
−
+
−    parser = self.parser_class(usage="%prog [options] <action>",
+
−                               description=desc,
+
−                               formatter=Formatter(),
+
−                               conflict_handler="resolve")
+
−    parser.add_option("-h", "--help", action="store_true",
+
−                      dest="help", help="Show the help message and exit.")
+
−    parser.add_option("-q", "--quiet", action="store_const", const=0,
+
−                      dest="verbose", help="Print errors only.")
+
−    parser.add_option("-v", "--verbose", action="store_const", const=2,
+
−                      dest="verbose", default=1,
+
−                      help="Print info level logs.")
+
−    parser.add_option("--noisy", action="store_const", const=3,
+
−                      dest="verbose", help="Print all logs.")
+
−    parser.add_option("-s", "--server", action="store", dest="server",
+
−                      default="appengine.google.com",
+
−                      metavar="SERVER", help="The server to connect to.")
+
−    parser.add_option("-e", "--email", action="store", dest="email",
+
−                      metavar="EMAIL", default=None,
+
−                      help="The username to use. Will prompt if omitted.")
+
−    parser.add_option("-H", "--host", action="store", dest="host",
+
−                      metavar="HOST", default=None,
+
−                      help="Overrides the Host header sent with all RPCs.")
+
−    parser.add_option("--no_cookies", action="store_false",
+
−                      dest="save_cookies", default=True,
+
−                      help="Do not save authentication cookies to local disk.")
+
−    parser.add_option("--passin", action="store_true",
+
−                      dest="passin", default=False,
+
−                      help="Read the login password from stdin.")
+
−    return parser
+
−
+
−  def _MakeSpecificParser(self, action):
+
−    """Creates a new parser with documentation specific to 'action'.
+
−
+
−    Args:
+
−      action: An Action instance to be used when initializing the new parser.
+
−
+
−    Returns:
+
−      A tuple containing:
+
−      parser: An instance of OptionsParser customized to 'action'.
+
−      options: The command line options after re-parsing.
+
−    """
+
−    parser = self._GetOptionParser()
+
−    parser.set_usage(action.usage)
+
−    parser.set_description("%s\n%s" % (action.short_desc, action.long_desc))
+
−    action.options(self, parser)
+
−    options, args = parser.parse_args(self.argv[1:])
+
−    return parser, options
+
−
+
−  def _PrintHelpAndExit(self, exit_code=2):
+
−    """Prints the parser's help message and exits the program.
+
−
+
−    Args:
+
−      exit_code: The integer code to pass to sys.exit().
+
−    """
+
−    self.parser.print_help()
+
−    sys.exit(exit_code)
+
−
+
−  def _GetRpcServer(self):
+
−    """Returns an instance of an AbstractRpcServer.
+
−
+
−    Returns:
+
−      A new AbstractRpcServer, on which RPC calls can be made.
+
−    """
+
−
+
−    def GetUserCredentials():
+
−      """Prompts the user for a username and password."""
+
−      email = self.options.email
+
−      if email is None:
+
−        email = self.raw_input_fn("Email: ")
+
−
+
−      password_prompt = "Password for %s: " % email
+
−      if self.options.passin:
+
−        password = self.raw_input_fn(password_prompt)
+
−      else:
+
−        password = self.password_input_fn(password_prompt)
+
−
+
−      return (email, password)
+
−
+
−    if self.options.host and self.options.host == "localhost":
+
−      email = self.options.email
+
−      if email is None:
+
−        email = "test@example.com"
+
−        logging.info("Using debug user %s.  Override with --email" % email)
+
−      server = self.rpc_server_class(
+
−          self.options.server,
+
−          lambda: (email, "password"),
+
−          host_override=self.options.host,
+
−          extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email},
+
−          save_cookies=self.options.save_cookies)
+
−      server.authenticated = True
+
−      return server
+
−
+
−    return self.rpc_server_class(self.options.server, GetUserCredentials,
+
−                                 host_override=self.options.host,
+
−                                 save_cookies=self.options.save_cookies)
+
−
+
−  def _FindYaml(self, basepath, file_name):
+
−    """Find yaml files in application directory.
+
−
+
−    Args:
+
−      basepath: Base application directory.
+
−      file_name: Filename without extension to search for.
+
−
+
−    Returns:
+
−      Path to located yaml file if one exists, else None.
+
−    """
+
−    if not os.path.isdir(basepath):
+
−      self.parser.error("Not a directory: %s" % basepath)
+
−
+
−    for yaml_file in (file_name + '.yaml', file_name + '.yml'):
+
−      yaml_path = os.path.join(basepath, yaml_file)
+
−      if os.path.isfile(yaml_path):
+
−        return yaml_path
+
−
+
−    return None
+
−
+
−  def _ParseAppYaml(self, basepath):
+
−    """Parses the app.yaml file.
+
−
+
−    Returns:
+
−      An AppInfoExternal object.
+
−    """
+
−    appyaml_filename = self._FindYaml(basepath, "app")
+
−    if appyaml_filename is None:
+
−      self.parser.error("Directory does not contain an app.yaml "
+
−                        "configuration file.")
+
−
+
−    fh = open(appyaml_filename, "r")
+
−    try:
+
−      appyaml = appinfo.LoadSingleAppInfo(fh)
+
−    finally:
+
−      fh.close()
+
−    return appyaml
+
−
+
−  def _ParseIndexYaml(self, basepath):
+
−    """Parses the index.yaml file.
+
−
+
−    Returns:
+
−      A single parsed yaml file or None if the file does not exist.
+
−    """
+
−    file_name = self._FindYaml(basepath, "index")
+
−    if file_name is not None:
+
−      fh = open(file_name, "r")
+
−      try:
+
−        index_defs = datastore_index.ParseIndexDefinitions(fh)
+
−      finally:
+
−        fh.close()
+
−      return index_defs
+
−    return None
+
−
+
−  def Help(self):
+
−    """Prints help for a specific action.
+
−
+
−    Expects self.args[0] to contain the name of the action in question.
+
−    Exits the program after printing the help message.
+
−    """
+
−    if len(self.args) != 1 or self.args[0] not in self.actions:
+
−      self.parser.error("Expected a single action argument. Must be one of:\n" +
+
−                        self._GetActionDescriptions())
+
−
+
−    action = self.actions[self.args[0]]
+
−    self.parser, options = self._MakeSpecificParser(action)
+
−    self._PrintHelpAndExit(exit_code=0)
+
−
+
−  def Update(self):
+
−    """Updates and deploys a new appversion."""
+
−    if len(self.args) != 1:
+
−      self.parser.error("Expected a single <directory> argument.")
+
−
+
−    basepath = self.args[0]
+
−    appyaml = self._ParseAppYaml(basepath)
+
−    rpc_server = self._GetRpcServer()
+
−
+
−    updatecheck = UpdateCheck(rpc_server, appyaml)
+
−    updatecheck.CheckForUpdates()
+
−
+
−    appversion = AppVersionUpload(rpc_server, appyaml)
+
−    appversion.DoUpload(FileIterator(basepath), self.options.max_size,
+
−                        lambda path: open(os.path.join(basepath, path), "rb"))
+
−
+
−    index_defs = self._ParseIndexYaml(basepath)
+
−    if index_defs:
+
−      index_upload = IndexDefinitionUpload(rpc_server, appyaml, index_defs)
+
−      try:
+
−        index_upload.DoUpload()
+
−      except urllib2.HTTPError, e:
+
−        StatusUpdate("Error %d: --- begin server output ---\n"
+
−                     "%s\n--- end server output ---" %
+
−                     (e.code, e.read().rstrip("\n")))
+
−        print >> self.error_fh, (
+
−          "Your app was updated, but there was an error updating your indexes. "
+
−          "Please retry later with appcfg.py update_indexes.")
+
−
+
−  def _UpdateOptions(self, parser):
+
−    """Adds update-specific options to 'parser'.
+
−
+
−    Args:
+
−      parser: An instance of OptionsParser.
+
−    """
+
−    parser.add_option("-S", "--max_size", type="int", dest="max_size",
+
−                      default=1048576, metavar="SIZE",
+
−                      help="Maximum size of a file to upload.")
+
−
+
−  def VacuumIndexes(self):
+
−    """Deletes unused indexes."""
+
−    if len(self.args) != 1:
+
−      self.parser.error("Expected a single <directory> argument.")
+
−
+
−    basepath = self.args[0]
+
−    config = self._ParseAppYaml(basepath)
+
−
+
−    index_defs = self._ParseIndexYaml(basepath)
+
−    if index_defs is None:
+
−      index_defs = datastore_index.IndexDefinitions()
+
−
+
−    rpc_server = self._GetRpcServer()
+
−    vacuum = VacuumIndexesOperation(rpc_server,
+
−                                    config,
+
−                                    self.options.force_delete)
+
−    vacuum.DoVacuum(index_defs)
+
−
+
−  def _VacuumIndexesOptions(self, parser):
+
−    """Adds vacuum_indexes-specific options to 'parser'.
+
−
+
−    Args:
+
−      parser: An instance of OptionsParser.
+
−    """
+
−    parser.add_option("-f", "--force", action="store_true", dest="force_delete",
+
−                      default=False,
+
−                      help="Force deletion without being prompted.")
+
−
+
−  def UpdateIndexes(self):
+
−    """Updates indexes."""
+
−    if len(self.args) != 1:
+
−      self.parser.error("Expected a single <directory> argument.")
+
−
+
−    basepath = self.args[0]
+
−    appyaml = self._ParseAppYaml(basepath)
+
−    rpc_server = self._GetRpcServer()
+
−
+
−    index_defs = self._ParseIndexYaml(basepath)
+
−    if index_defs:
+
−      index_upload = IndexDefinitionUpload(rpc_server, appyaml, index_defs)
+
−      index_upload.DoUpload()
+
−
+
−  def Rollback(self):
+
−    """Does a rollback of any existing transaction for this app version."""
+
−    if len(self.args) != 1:
+
−      self.parser.error("Expected a single <directory> argument.")
+
−
+
−    basepath = self.args[0]
+
−    appyaml = self._ParseAppYaml(basepath)
+
−
+
−    appversion = AppVersionUpload(self._GetRpcServer(), appyaml)
+
−    appversion.in_transaction = True
+
−    appversion.Rollback()
+
−
+
−  def RequestLogs(self):
+
−    """Write request logs to a file."""
+
−    if len(self.args) != 2:
+
−      self.parser.error(
+
−          "Expected a <directory> argument and an <output_file> argument.")
+
−    if (self.options.severity is not None and
+
−        not 0 <= self.options.severity <= MAX_LOG_LEVEL):
+
−      self.parser.error(
+
−          "Severity range is 0 (DEBUG) through %s (CRITICAL)." % MAX_LOG_LEVEL)
+
−
+
−    if self.options.num_days is None:
+
−      self.options.num_days = int(not self.options.append)
+
−    basepath = self.args[0]
+
−    appyaml = self._ParseAppYaml(basepath)
+
−    rpc_server = self._GetRpcServer()
+
−    logs_requester = LogsRequester(rpc_server, appyaml, self.args[1],
+
−                                   self.options.num_days,
+
−                                   self.options.append,
+
−                                   self.options.severity,
+
−                                   time.time())
+
−    logs_requester.DownloadLogs()
+
−
+
−  def _RequestLogsOptions(self, parser):
+
−    """Ads request_logs-specific options to 'parser'.
+
−
+
−    Args:
+
−      parser: An instance of OptionsParser.
+
−    """
+
−    parser.add_option("-n", "--num_days", type="int", dest="num_days",
+
−                      action="store", default=None,
+
−                      help="Number of days worth of log data to get. "
+
−                           "The cut-off point is midnight UTC. "
+
−                           "Use 0 to get all available logs. "
+
−                           "Default is 1, unless --append is also given; "
+
−                           "then the default is 0.")
+
−    parser.add_option("-a", "--append", dest="append",
+
−                       action="store_true", default=False,
+
−                      help="Append to existing file.")
+
−    parser.add_option("--severity", type="int", dest="severity",
+
−                      action="store", default=None,
+
−                      help="Severity of app-level log messages to get. "
+
−                           "The range is 0 (DEBUG) through 4 (CRITICAL). "
+
−                           "If omitted, only request logs are returned.")
+
−
+
−  class Action(object):
+
−    """Contains information about a command line action.
+
−
+
−    Attributes:
+
−      function: An AppCfgApp function that will perform the appropriate
+
−        action.
+
−      usage: A command line usage string.
+
−      short_desc: A one-line description of the action.
+
−      long_desc: A detailed description of the action.  Whitespace and
+
−        formatting will be preserved.
+
−      options: A function that will add extra options to a given OptionParser
+
−        object.
+
−    """
+
−
+
−    def __init__(self, function, usage, short_desc, long_desc="",
+
−                 options=lambda obj, parser: None):
+
−      """Initializer for the class attributes."""
+
−      self.function = function
+
−      self.usage = usage
+
−      self.short_desc = short_desc
+
−      self.long_desc = long_desc
+
−      self.options = options
+
−
+
−  actions = {
+
−
+
−      "help": Action(
+
−        function=Help,
+
−        usage="%prog help <action>",
+
−        short_desc="Print help for a specific action."),
+
−
+
−      "update": Action(
+
−        function=Update,
+
−        usage="%prog [options] update <directory>",
+
−        options=_UpdateOptions,
+
−        short_desc="Create or update an app version.",
+
−        long_desc="""
+
−Specify a directory that contains all of the files required by
+
−the app, and appcfg.py will create/update the app version referenced
+
−in the app.yaml file at the top level of that directory.  appcfg.py
+
−will follow symlinks and recursively upload all files to the server.
+
−Temporary or source control files (e.g. foo~, .svn/*) will be skipped."""),
+
−
+
−      "update_indexes": Action(
+
−        function=UpdateIndexes,
+
−        usage="%prog [options] update_indexes <directory>",
+
−        short_desc="Update application indexes.",
+
−        long_desc="""
+
−The 'update_indexes' command will add additional indexes which are not currently
+
−in production as well as restart any indexes that were not completed."""),
+
−
+
−      "vacuum_indexes": Action(
+
−        function=VacuumIndexes,
+
−        usage="%prog [options] vacuum_indexes <directory>",
+
−        options=_VacuumIndexesOptions,
+
−        short_desc="Delete unused indexes from application.",
+
−        long_desc="""
+
−The 'vacuum_indexes' command will help clean up indexes which are no longer
+
−in use.  It does this by comparing the local index configuration with
+
−indexes that are actually defined on the server.  If any indexes on the
+
−server do not exist in the index configuration file, the user is given the
+
−option to delete them."""),
+
−
+
−      "rollback": Action(
+
−        function=Rollback,
+
−        usage="%prog [options] rollback <directory>",
+
−        short_desc="Rollback an in-progress update.",
+
−        long_desc="""
+
−The 'update' command requires a server-side transaction.  Use 'rollback'
+
−if you get an error message about another transaction being in progress
+
−and you are sure that there is no such transaction."""),
+
−
+
−      "request_logs": Action(
+
−        function=RequestLogs,
+
−        usage="%prog [options] request_logs <directory> <output_file>",
+
−        options=_RequestLogsOptions,
+
−        short_desc="Write request logs in Apache common log format.",
+
−        long_desc="""
+
−The 'request_logs' command exports the request logs from your application
+
−to a file.  It will write Apache common log format records ordered
+
−chronologically.  If output file is '-' stdout will be written."""),
+
−
+
−  }
+
−
+
−
+
−def main(argv):
+
−  logging.basicConfig(format=("%(asctime)s %(levelname)s %(filename)s:"
+
−                              "%(lineno)s %(message)s "))
+
−  try:
+
−    AppCfgApp(argv).Run()
+
−  except KeyboardInterrupt:
+
−    StatusUpdate("Interrupted.")
+
−    sys.exit(1)
+
−
+
−
+
−if __name__ == "__main__":
+
−  main(sys.argv)