#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Tool for deploying apps to an app server.

Currently, the application only uploads new appversions. To do this, it first
walks the directory tree rooted at the path the user specifies, adding all the
files it finds to a list. It then uploads the application configuration
(app.yaml) to the server using HTTP, followed by uploading each of the files.
It then commits the transaction with another request.

The bulk of this work is handled by the AppVersionUpload class, which exposes
methods to add to the list of files, fetch a list of modified files, upload
files, and commit or rollback the transaction.
"""


import cookielib
import datetime
import getpass
import logging
import mimetypes
import optparse
import os
import re
import sha
import socket
import sys
import tempfile
import time
import urllib
import urllib2

import google
from google.appengine.api import appinfo
from google.appengine.api import validation
from google.appengine.api import yaml_errors
from google.appengine.api import yaml_object
from google.appengine.datastore import datastore_index
import yaml


MAX_FILES_TO_CLONE = 100
LIST_DELIMITER = "\n"
TUPLE_DELIMITER = "|"

VERSION_FILE = "../VERSION"

UPDATE_CHECK_TIMEOUT = 3

NAG_FILE = ".appcfg_nag"

MAX_LOG_LEVEL = 4

verbosity = 1


def StatusUpdate(msg):
  """Print a status message to stderr.

  If 'verbosity' is greater than 0, print the message.

  Args:
    msg: The string to print.
  """
  if verbosity > 0:
    print >>sys.stderr, msg


class ClientLoginError(urllib2.HTTPError):
  """Raised to indicate there was an error authenticating with ClientLogin."""

  def __init__(self, url, code, msg, headers, args):
    urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
    self.args = args
    self.reason = args["Error"]


class AbstractRpcServer(object):
  """Provides a common interface for a simple RPC server."""

  def __init__(self, host, auth_function, host_override=None,
               extra_headers=None, save_cookies=False):
    """Creates a new HttpRpcServer.

    Args:
      host: The host to send requests to.
      auth_function: A function that takes no arguments and returns an
        (email, password) tuple when called. Will be called if authentication
        is required.
      host_override: The host header to send to the server (defaults to host).
      extra_headers: A dict of extra headers to append to every request. Values
        supplied here will override other default headers that are supplied.
      save_cookies: If True, save the authentication cookies to local disk.
        If False, use an in-memory cookiejar instead.  Subclasses must
        implement this functionality.  Defaults to False.
    """
    self.host = host
    self.host_override = host_override
    self.auth_function = auth_function
    self.authenticated = False

    self.extra_headers = {
      "User-agent": GetUserAgent()
    }
    if extra_headers:
      self.extra_headers.update(extra_headers)

    self.save_cookies = save_cookies
    self.cookie_jar = cookielib.MozillaCookieJar()
    self.opener = self._GetOpener()
    if self.host_override:
      logging.info("Server: %s; Host: %s", self.host, self.host_override)
    else:
      logging.info("Server: %s", self.host)

  def _GetOpener(self):
    """Returns an OpenerDirector for making HTTP requests.

    Returns:
      A urllib2.OpenerDirector object.
    """
    raise NotImplemented()

  def _CreateRequest(self, url, data=None):
    """Creates a new urllib request."""
    logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
    req = urllib2.Request(url, data=data)
    if self.host_override:
      req.add_header("Host", self.host_override)
    for key, value in self.extra_headers.iteritems():
      req.add_header(key, value)
    return req

  def _GetAuthToken(self, email, password):
    """Uses ClientLogin to authenticate the user, returning an auth token.

    Args:
      email:    The user's email address
      password: The user's password

    Raises:
      ClientLoginError: If there was an error authenticating with ClientLogin.
      HTTPError: If there was some other form of HTTP error.

    Returns:
      The authentication token returned by ClientLogin.
    """
    req = self._CreateRequest(
        url="https://www.google.com/accounts/ClientLogin",
        data=urllib.urlencode({
            "Email": email,
            "Passwd": password,
            "service": "ah",
            "source": "Google-appcfg-1.0",
            "accountType": "HOSTED_OR_GOOGLE"
        })
    )
    try:
      response = self.opener.open(req)
      response_body = response.read()
      response_dict = dict(x.split("=")
                           for x in response_body.split("\n") if x)
      return response_dict["Auth"]
    except urllib2.HTTPError, e:
      if e.code == 403:
        body = e.read()
        response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
        raise ClientLoginError(req.get_full_url(), e.code, e.msg,
                               e.headers, response_dict)
      else:
        raise

  def _GetAuthCookie(self, auth_token):
    """Fetches authentication cookies for an authentication token.

    Args:
      auth_token: The authentication token returned by ClientLogin.

    Raises:
      HTTPError: If there was an error fetching the authentication cookies.
    """
    continue_location = "http://localhost/"
    args = {"continue": continue_location, "auth": auth_token}
    login_path = os.environ.get("APPCFG_LOGIN_PATH", "/_ah")
    req = self._CreateRequest("http://%s%s/login?%s" %
                              (self.host, login_path, urllib.urlencode(args)))
    try:
      response = self.opener.open(req)
    except urllib2.HTTPError, e:
      response = e
    if (response.code != 302 or
        response.info()["location"] != continue_location):
      raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
                              response.headers, response.fp)
    self.authenticated = True

  def _Authenticate(self):
    """Authenticates the user.

    The authentication process works as follows:
     1) We get a username and password from the user
     2) We use ClientLogin to obtain an AUTH token for the user
        (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
     3) We pass the auth token to /_ah/login on the server to obtain an
        authentication cookie. If login was successful, it tries to redirect
        us to the URL we provided.

    If we attempt to access the upload API without first obtaining an
    authentication cookie, it returns a 401 response and directs us to
    authenticate ourselves with ClientLogin.
    """
    for i in range(3):
      credentials = self.auth_function()
      try:
        auth_token = self._GetAuthToken(credentials[0], credentials[1])
      except ClientLoginError, e:
        if e.reason == "BadAuthentication":
          print >>sys.stderr, "Invalid username or password."
          continue
        if e.reason == "CaptchaRequired":
          print >>sys.stderr, (
              "Please go to\n"
              "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
              "and verify you are a human.  Then try again.")
          break;
        if e.reason == "NotVerified":
          print >>sys.stderr, "Account not verified."
          break
        if e.reason == "TermsNotAgreed":
          print >>sys.stderr, "User has not agreed to TOS."
          break
        if e.reason == "AccountDeleted":
          print >>sys.stderr, "The user account has been deleted."
          break
        if e.reason == "AccountDisabled":
          print >>sys.stderr, "The user account has been disabled."
          break
        if e.reason == "ServiceDisabled":
          print >>sys.stderr, ("The user's access to the service has been "
                               "disabled.")
          break
        if e.reason == "ServiceUnavailable":
          print >>sys.stderr, "The service is not available; try again later."
          break
        raise
      self._GetAuthCookie(auth_token)
      return

  def Send(self, request_path, payload="",
           content_type="application/octet-stream",
           timeout=None,
           **kwargs):
    """Sends an RPC and returns the response.

    Args:
      request_path: The path to send the request to, eg /api/appversion/create.
      payload: The body of the request, or None to send an empty request.
      content_type: The Content-Type header to use.
      timeout: timeout in seconds; default None i.e. no timeout.
        (Note: for large requests on OS X, the timeout doesn't work right.)
      kwargs: Any keyword arguments are converted into query string parameters.

    Returns:
      The response body, as a string.
    """
    if not self.authenticated:
      self._Authenticate()

    old_timeout = socket.getdefaulttimeout()
    socket.setdefaulttimeout(timeout)
    try:
      tries = 0
      while True:
        tries += 1
        args = dict(kwargs)
        url = "http://%s%s?%s" % (self.host, request_path,
                                  urllib.urlencode(args))
        req = self._CreateRequest(url=url, data=payload)
        req.add_header("Content-Type", content_type)
        req.add_header("X-appcfg-api-version", "1")
        try:
          f = self.opener.open(req)
          response = f.read()
          f.close()
          return response
        except urllib2.HTTPError, e:
          if tries > 3:
            raise
          elif e.code == 401:
            self._Authenticate()
          elif e.code >= 500 and e.code < 600:
            continue
          else:
            raise
    finally:
      socket.setdefaulttimeout(old_timeout)


class HttpRpcServer(AbstractRpcServer):
  """Provides a simplified RPC-style interface for HTTP requests."""

  DEFAULT_COOKIE_FILE_PATH = "~/.appcfg_cookies"

  def _Authenticate(self):
    """Save the cookie jar after authentication."""
    super(HttpRpcServer, self)._Authenticate()
    if self.cookie_jar.filename is not None and self.save_cookies:
      StatusUpdate("Saving authentication cookies to %s" %
                   self.cookie_jar.filename)
      self.cookie_jar.save()

  def _GetOpener(self):
    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:
      A urllib2.OpenerDirector object.
    """
    opener = urllib2.OpenerDirector()
    opener.add_handler(urllib2.ProxyHandler())
    opener.add_handler(urllib2.UnknownHandler())
    opener.add_handler(urllib2.HTTPHandler())
    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
    opener.add_handler(urllib2.HTTPSHandler())
    opener.add_handler(urllib2.HTTPErrorProcessor())

    if self.save_cookies:
      self.cookie_jar.filename = os.path.expanduser(HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)

      if os.path.exists(self.cookie_jar.filename):
        try:
          self.cookie_jar.load()
          self.authenticated = True
          StatusUpdate("Loaded authentication cookies from %s" %
                       self.cookie_jar.filename)
        except (OSError, IOError, cookielib.LoadError), e:
          logging.debug("Could not load authentication cookies; %s: %s",
                        e.__class__.__name__, e)
          self.cookie_jar.filename = None
      else:
        try:
          fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600)
          os.close(fd)
        except (OSError, IOError), e:
          logging.debug("Could not create authentication cookies file; %s: %s",
                        e.__class__.__name__, e)
          self.cookie_jar.filename = None

    opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
    return opener


def GetMimeTypeIfStaticFile(config, filename):
  """Looks up the mime type for 'filename'.

  Uses the handlers in 'config' to determine if the file should
  be treated as a static file.

  Args:
    config: The app.yaml object to check the filename against.
    filename: The name of the file.

  Returns:
    The mime type string.  For example, 'text/plain' or 'image/gif'.
    None if this is not a static file.
  """
  for handler in config.handlers:
    handler_type = handler.GetHandlerType()
    if handler_type in ("static_dir", "static_files"):
      if handler_type == "static_dir":
        regex = os.path.join(re.escape(handler.GetHandler()), ".*")
      else:
        regex = handler.upload
      if re.match(regex, filename):
        if handler.mime_type is not None:
          return handler.mime_type
        else:
          guess = mimetypes.guess_type(filename)[0]
          if guess is None:
            default = "application/octet-stream"
            print >>sys.stderr, ("Could not guess mimetype for %s.  Using %s."
                                 % (filename, default))
            return default
          return guess
  return None


def BuildClonePostBody(file_tuples):
  """Build the post body for the /api/clone{files,blobs} urls.

  Args:
    file_tuples: A list of tuples.  Each tuple should contain the entries
      appropriate for the endpoint in question.

  Returns:
    A string containing the properly delimited tuples.
  """
  file_list = []
  for tup in file_tuples:
    path = tup[0]
    tup = tup[1:]
    file_list.append(TUPLE_DELIMITER.join([path] + list(tup)))
  return LIST_DELIMITER.join(file_list)


class NagFile(validation.Validated):
  """A validated YAML class to represent the user's nag preferences.

  Attributes:
    timestamp: The timestamp of the last nag.
    opt_in: True if the user wants to check for updates on dev_appserver
      start.  False if not.  May be None if we have not asked the user yet.
  """

  ATTRIBUTES = {
    "timestamp": validation.TYPE_FLOAT,
    "opt_in": validation.Optional(validation.TYPE_BOOL),
  }

  @staticmethod
  def Load(nag_file):
    """Load a single NagFile object where one and only one is expected.

    Args:
      nag_file: A file-like object or string containing the yaml data to parse.

    Returns:
      A NagFile instance.
    """
    return yaml_object.BuildSingleObject(NagFile, nag_file)


def GetVersionObject(isfile=os.path.isfile, open_fn=open):
  """Gets the version of the SDK by parsing the VERSION file.

  Args:
    isfile, open_fn: Used for testing.

  Returns:
    A Yaml object or None if the VERSION file does not exist.
  """
  version_filename = os.path.join(os.path.dirname(google.__file__),
                                  VERSION_FILE)
  if not isfile(version_filename):
    logging.error("Could not find version file at %s", version_filename)
    return None

  version_fh = open_fn(version_filename, "r")
  try:
    version = yaml.safe_load(version_fh)
  finally:
    version_fh.close()

  return version


class UpdateCheck(object):
  """Determines if the local SDK is the latest version.

  Nags the user when there are updates to the SDK.  As the SDK becomes
  more out of date, the language in the nagging gets stronger.  We
  store a little yaml file in the user's home directory so that we nag
  the user only once a week.

  The yaml file has the following field:
    'timestamp': Last time we nagged the user in seconds since the epoch.

  Attributes:
    server: An AbstractRpcServer instance used to check for the latest SDK.
    config: The app's AppInfoExternal.  Needed to determine which api_version
      the app is using.
  """

  def __init__(self,
               server,
               config,
               isdir=os.path.isdir,
               isfile=os.path.isfile,
               open_fn=open):
    """Create a new UpdateCheck.

    Args:
      server: The AbstractRpcServer to use.
      config: The yaml object that specifies the configuration of this
        application.

    Args for testing:
      isdir: Replacement for os.path.isdir.
      isfile: Replacement for os.path.isfile.
      open: Replacement for the open builtin.
    """
    self.server = server
    self.config = config
    self.isdir = isdir
    self.isfile = isfile
    self.open = open_fn

  @staticmethod
  def MakeNagFilename():
    """Returns the filename for the nag file for this user."""
    user_homedir = os.path.expanduser("~/")
    if not os.path.isdir(user_homedir):
      drive, tail = os.path.splitdrive(os.__file__)
      if drive:
        os.environ["HOMEDRIVE"] = drive

    return os.path.expanduser("~/" + NAG_FILE)

  def _ParseVersionFile(self):
    """Parse the local VERSION file.

    Returns:
      A Yaml object or None if the file does not exist.
    """
    return GetVersionObject(isfile=self.isfile, open_fn=self.open)

  def CheckSupportedVersion(self):
    """Determines if the app's api_version is supported by the SDK.

    Uses the api_version field from the AppInfoExternal to determine if
    the SDK supports that api_version.

    Raises:
      SystemExit if the api_version is not supported.
    """
    version = self._ParseVersionFile()
    if version is None:
      logging.error("Could not determine if the SDK supports the api_version "
                    "requested in app.yaml.")
      return
    if self.config.api_version not in version["api_versions"]:
      logging.critical("The api_version specified in app.yaml (%s) is not "
                       "supported by this release of the SDK.  The supported "
                       "api_versions are %s.",
                       self.config.api_version, version["api_versions"])
      sys.exit(1)

  def CheckForUpdates(self):
    """Queries the server for updates and nags the user if appropriate.

    Queries the server for the latest SDK version at the same time reporting
    the local SDK version.  The server will respond with a yaml document
    containing the fields:
      "release": The name of the release (e.g. 1.2).
      "timestamp": The time the release was created (YYYY-MM-DD HH:MM AM/PM TZ).
      "api_versions": A list of api_version strings (e.g. ['1', 'beta']).

    We will nag the user with increasing severity if:
    - There is a new release.
    - There is a new release with a new api_version.
    - There is a new release that does not support the api_version named in
      self.config.
    """
    version = self._ParseVersionFile()
    if version is None:
      logging.info("Skipping update check")
      return
    logging.info("Checking for updates to the SDK.")

    try:
      response = self.server.Send("/api/updatecheck",
                                  timeout=UPDATE_CHECK_TIMEOUT,
                                  release=version["release"],
                                  timestamp=version["timestamp"],
                                  api_versions=version["api_versions"])
    except urllib2.URLError, e:
      logging.info("Update check failed: %s", e)
      return

    latest = yaml.safe_load(response)
    if latest["release"] == version["release"]:
      logging.info("The SDK is up to date.")
      return

    api_versions = latest["api_versions"]
    if self.config.api_version not in api_versions:
      self._Nag(
          "The api version you are using (%s) is obsolete!  You should\n"
          "upgrade your SDK and test that your code works with the new\n"
          "api version." % self.config.api_version,
          latest, version, force=True)
      return

    if self.config.api_version != api_versions[len(api_versions) - 1]:
      self._Nag(
          "The api version you are using (%s) is deprecated. You should\n"
          "upgrade your SDK to try the new functionality." %
          self.config.api_version, latest, version)
      return

    self._Nag("There is a new release of the SDK available.",
              latest, version)

  def _ParseNagFile(self):
    """Parses the nag file.

    Returns:
      A NagFile if the file was present else None.
    """
    nag_filename = UpdateCheck.MakeNagFilename()
    if self.isfile(nag_filename):
      fh = self.open(nag_filename, "r")
      try:
        nag = NagFile.Load(fh)
      finally:
        fh.close()
      return nag
    return None

  def _WriteNagFile(self, nag):
    """Writes the NagFile to the user's nag file.

    If the destination path does not exist, this method will log an error
    and fail silently.

    Args:
      nag: The NagFile to write.
    """
    nagfilename = UpdateCheck.MakeNagFilename()
    try:
      fh = self.open(nagfilename, "w")
      try:
        fh.write(nag.ToYAML())
      finally:
        fh.close()
    except (OSError, IOError), e:
      logging.error("Could not write nag file to %s. Error: %s", nagfilename, e)

  def _Nag(self, msg, latest, version, force=False):
    """Prints a nag message and updates the nag file's timestamp.

    Because we don't want to nag the user everytime, we store a simple
    yaml document in the user's home directory.  If the timestamp in this
    doc is over a week old, we'll nag the user.  And when we nag the user,
    we update the timestamp in this doc.

    Args:
      msg: The formatted message to print to the user.
      latest: The yaml document received from the server.
      version: The local yaml version document.
      force: If True, always nag the user, ignoring the nag file.
    """
    nag = self._ParseNagFile()
    if nag and not force:
      last_nag = datetime.datetime.fromtimestamp(nag.timestamp)
      if datetime.datetime.now() - last_nag < datetime.timedelta(weeks=1):
        logging.debug("Skipping nag message")
        return

    if nag is None:
      nag = NagFile()
    nag.timestamp = time.time()
    self._WriteNagFile(nag)

    print "****************************************************************"
    print msg
    print "-----------"
    print "Latest SDK:"
    print yaml.dump(latest)
    print "-----------"
    print "Your SDK:"
    print yaml.dump(version)
    print "-----------"
    print "Please visit http://code.google.com/appengine for the latest SDK"
    print "****************************************************************"

  def AllowedToCheckForUpdates(self, input_fn=raw_input):
    """Determines if the user wants to check for updates.

    On startup, the dev_appserver wants to check for updates to the SDK.
    Because this action reports usage to Google when the user is not
    otherwise communicating with Google (e.g. pushing a new app version),
    the user must opt in.

    If the user does not have a nag file, we will query the user and
    save the response in the nag file.  Subsequent calls to this function
    will re-use that response.

    Returns:
      True if the user wants to check for updates.  False otherwise.
    """
    nag = self._ParseNagFile()
    if nag is None:
      nag = NagFile()
      nag.timestamp = time.time()

    if nag.opt_in is None:
      answer = input_fn("Allow dev_appserver to check for updates on startup? "
                        "(Y/n): ")
      answer = answer.strip().lower()
      if answer == "n" or answer == "no":
        print ("dev_appserver will not check for updates on startup.  To "
               "change this setting, edit %s" % UpdateCheck.MakeNagFilename())
        nag.opt_in = False
      else:
        print ("dev_appserver will check for updates on startup.  To change "
               "this setting, edit %s" % UpdateCheck.MakeNagFilename())
        nag.opt_in = True
      self._WriteNagFile(nag)
    return nag.opt_in


class IndexDefinitionUpload(object):
  """Provides facilities to upload index definitions to the hosting service."""

  def __init__(self, server, config, definitions):
    """Creates a new DatastoreIndexUpload.

    Args:
      server: The RPC server to use.  Should be an instance of HttpRpcServer
        or TestRpcServer.
      config: The AppInfoExternal object derived from the app.yaml file.
      definitions: An IndexDefinitions object.
    """
    self.server = server
    self.config = config
    self.definitions = definitions

  def DoUpload(self):
    StatusUpdate("Uploading index definitions.")
    self.server.Send("/api/datastore/index/add",
                     app_id=self.config.application,
                     version=self.config.version,
                     payload=self.definitions.ToYAML())


class IndexOperation(object):
  """Provide facilities for writing Index operation commands."""

  def __init__(self, server, config):
    """Creates a new IndexOperation.

    Args:
      server: The RPC server to use.  Should be an instance of HttpRpcServer
        or TestRpcServer.
      config: appinfo.AppInfoExternal configuration object.
    """
    self.server = server
    self.config = config

  def DoDiff(self, definitions):
    """Retrieve diff file from the server.

    Args:
      definitions: datastore_index.IndexDefinitions as loaded from users
        index.yaml file.

    Returns:
      A pair of datastore_index.IndexDefinitions objects.  The first record
      is the set of indexes that are present in the index.yaml file but missing
      from the server.  The second record is the set of indexes that are
      present on the server but missing from the index.yaml file (indicating
      that these indexes should probably be vacuumed).
    """
    StatusUpdate("Fetching index definitions diff.")
    response = self.server.Send("/api/datastore/index/diff",
                                app_id=self.config.application,
                                payload=definitions.ToYAML())
    return datastore_index.ParseMultipleIndexDefinitions(response)

  def DoDelete(self, definitions):
    """Delete indexes from the server.

    Args:
      definitions: Index definitions to delete from datastore.

    Returns:
      A single datstore_index.IndexDefinitions containing indexes that were
      not deleted, probably because they were already removed.  This may
      be normal behavior as there is a potential race condition between fetching
      the index-diff and sending deletion confirmation through.
    """
    StatusUpdate("Deleting selected index definitions.")
    response = self.server.Send("/api/datastore/index/delete",
                                app_id=self.config.application,
                                payload=definitions.ToYAML())
    return datastore_index.ParseIndexDefinitions(response)


class VacuumIndexesOperation(IndexOperation):
  """Provide facilities to request the deletion of datastore indexes."""

  def __init__(self, server, config, force,
               confirmation_fn=raw_input):
    """Creates a new VacuumIndexesOperation.

    Args:
      server: The RPC server to use.  Should be an instance of HttpRpcServer
        or TestRpcServer.
      config: appinfo.AppInfoExternal configuration object.
      force: True to force deletion of indexes, else False.
      confirmation_fn: Function used for getting input form user.
    """
    super(VacuumIndexesOperation, self).__init__(server, config)
    self.force = force
    self.confirmation_fn = confirmation_fn

  def GetConfirmation(self, index):
    """Get confirmation from user to delete an index.

    This method will enter an input loop until the user provides a
    response it is expecting.  Valid input is one of three responses:

      y: Confirm deletion of index.
      n: Do not delete index.
      a: Delete all indexes without asking for further confirmation.

    If the user enters nothing at all, the default action is to skip
    that index and do not delete.

    If the user selects 'a', as a side effect, the 'force' flag is set.

    Args:
      index: Index to confirm.

    Returns:
      True if user enters 'y' or 'a'.  False if user enter 'n'.
    """
    while True:
      print "This index is no longer defined in your index.yaml file."
      print
      print index.ToYAML()
      print

      confirmation = self.confirmation_fn(
          "Are you sure you want to delete this index? (N/y/a): ")
      confirmation = confirmation.strip().lower()

      if confirmation == 'y':
        return True
      elif confirmation == 'n' or confirmation == '':
        return False
      elif confirmation == 'a':
        self.force = True
        return True
      else:
        print "Did not understand your response."

  def DoVacuum(self, definitions):
    """Vacuum indexes in datastore.

    This method will query the server to determine which indexes are not
    being used according to the user's local index.yaml file.  Once it has
    made this determination, it confirms with the user which unused indexes
    should be deleted.  Once confirmation for each index is receives, it
    deletes those indexes.

    Because another user may in theory delete the same indexes at the same
    time as the user, there is a potential race condition.  In this rare cases,
    some of the indexes previously confirmed for deletion will not be found.
    The user is notified which indexes these were.

    Args:
      definitions: datastore_index.IndexDefinitions as loaded from users
        index.yaml file.
    """
    new_indexes, unused_indexes = self.DoDiff(definitions)

    deletions = datastore_index.IndexDefinitions(indexes=[])
    if unused_indexes.indexes is not None:
      for index in unused_indexes.indexes:
        if self.force or self.GetConfirmation(index):
          deletions.indexes.append(index)

    if len(deletions.indexes) > 0:
      not_deleted = self.DoDelete(deletions)

      if not_deleted.indexes:
        not_deleted_count = len(not_deleted.indexes)
        if not_deleted_count == 1:
          warning_message = ('An index was not deleted.  Most likely this is '
                             'because it no longer exists.\n\n')
        else:
          warning_message = ('%d indexes were not deleted.  Most likely this '
                             'is because they no longer exist.\n\n'
                             % not_deleted_count)
        for index in not_deleted.indexes:
          warning_message = warning_message + index.ToYAML()
        logging.warning(warning_message)


class LogsRequester(object):
  """Provide facilities to export request logs."""

  def __init__(self, server, config, output_file,
               num_days, append, severity, now):
    """Constructor.

    Args:
      server: The RPC server to use.  Should be an instance of HttpRpcServer
        or TestRpcServer.
      config: appinfo.AppInfoExternal configuration object.
      output_file: Output file name.
      num_days: Number of days worth of logs to export; 0 for all available.
      append: True if appending to an existing file.
      severity: App log severity to request (0-4); None for no app logs.
      now: POSIX timestamp used for calculating valid dates for num_days.
    """
    self.server = server
    self.config = config
    self.output_file = output_file
    self.append = append
    self.num_days = num_days
    self.severity = severity
    self.version_id = self.config.version + ".1"
    self.sentinel = None
    self.write_mode = "w"
    if self.append:
      self.sentinel = FindSentinel(self.output_file)
      self.write_mode = "a"
    self.valid_dates = None
    if self.num_days:
      patterns = []
      for i in xrange(self.num_days):
        then = time.gmtime(now - 24*3600 * i)
        patterns.append(re.escape(time.strftime("%d/%m/%Y", then)))
        patterns.append(re.escape(time.strftime("%d/%b/%Y", then)))
      self.valid_dates = re.compile(r"[^[]+\[(" + "|".join(patterns) + r"):")

  def DownloadLogs(self):
    """Download the requested logs.

    This will write the logs to the file designated by
    self.output_file, or to stdout if the filename is '-'.
    Multiple roundtrips to the server may be made.
    """
    StatusUpdate("Downloading request logs for %s %s." %
                 (self.config.application, self.version_id))
    tf = tempfile.TemporaryFile()
    offset = None
    try:
      while True:
        try:
          offset = self.RequestLogLines(tf, offset)
          if not offset:
            break
        except KeyboardInterrupt:
          StatusUpdate("Keyboard interrupt; saving data downloaded so far.")
          break
      StatusUpdate("Copying request logs to %r." % self.output_file)
      if self.output_file == "-":
        of = sys.stdout
      else:
        try:
          of = open(self.output_file, self.write_mode)
        except IOError, err:
          StatusUpdate("Can't write %r: %s." % (self.output_file, err))
          sys.exit(1)
      try:
        line_count = CopyReversedLines(tf, of)
      finally:
        of.flush()
        if of is not sys.stdout:
          of.close()
    finally:
      tf.close()
    StatusUpdate("Copied %d records." % line_count)

  def RequestLogLines(self, tf, offset):
    """Make a single roundtrip to the server.

    Args:
      tf: Writable binary stream to which the log lines returned by
        the server are written, stripped of headers, and excluding
        lines skipped due to self.sentinel or self.valid_dates filtering.
      offset: Offset string for a continued request; None for the first.

    Returns:
      The offset string to be used for the next request, if another
      request should be issued; or None, if not.
    """
    logging.info("Request with offset %r.", offset)
    kwds = {'app_id': self.config.application,
            'version': self.version_id,
            'limit': 100,
            }
    if offset:
      kwds['offset'] = offset
    if self.severity is not None:
      kwds['severity'] = str(self.severity)
    response = self.server.Send("/api/request_logs", payload=None, **kwds)
    response = response.replace("\r", "\0")
    lines = response.splitlines()
    logging.info("Received %d bytes, %d records.", len(response), len(lines))
    offset = None
    if lines and lines[0].startswith('#'):
      match = re.match(r'^#\s*next_offset=(\S+)\s*$', lines[0])
      del lines[0]
      if match:
        offset = match.group(1)
    if lines and lines[-1].startswith('#'):
      del lines[-1]
    valid_dates = self.valid_dates
    sentinel = self.sentinel
    len_sentinel = None
    if sentinel:
      len_sentinel = len(sentinel)
    for line in lines:
      if ((sentinel and
           line.startswith(sentinel) and
           line[len_sentinel : len_sentinel+1] in ("", "\0")) or
          (valid_dates and not valid_dates.match(line))):
        return None
      tf.write(line + '\n')
    if not lines:
      return None
    return offset


def CopyReversedLines(input, output, blocksize=2**16):
  r"""Copy lines from input stream to output stream in reverse order.

  As a special feature, null bytes in the input are turned into
  newlines followed by tabs in the output, but these "sub-lines"
  separated by null bytes are not reversed.  E.g. If the input is
  "A\0B\nC\0D\n", the output is "C\n\tD\nA\n\tB\n".

  Args:
    input: A seekable stream open for reading in binary mode.
    output: A stream open for writing; doesn't have to be seekable or binary.
    blocksize: Optional block size for buffering, for unit testing.

  Returns:
    The number of lines copied.
  """
  line_count = 0
  input.seek(0, 2)
  last_block = input.tell() // blocksize
  spillover = ""
  for iblock in xrange(last_block + 1, -1, -1):
    input.seek(iblock * blocksize)
    data = input.read(blocksize)
    lines = data.splitlines(True)
    lines[-1:] = "".join(lines[-1:] + [spillover]).splitlines(True)
    if lines and not lines[-1].endswith("\n"):
      lines[-1] += "\n"
    lines.reverse()
    if lines and iblock > 0:
      spillover = lines.pop()
    if lines:
      line_count += len(lines)
      data = "".join(lines).replace("\0", "\n\t")
      output.write(data)
  return line_count


def FindSentinel(filename, blocksize=2**16):
  """Return the sentinel line from the output file.

  Args:
    filename: The filename of the output file.  (We'll read this file.)
    blocksize: Optional block size for buffering, for unit testing.

  Returns:
    The contents of the last line in the file that doesn't start with
    a tab, with its trailing newline stripped; or None if the file
    couldn't be opened or no such line could be found by inspecting
    the last 'blocksize' bytes of the file.
  """
  if filename == "-":
    StatusUpdate("Can't combine --append with output to stdout.")
    sys.exit(2)
  try:
    fp = open(filename, "rb")
  except IOError, err:
    StatusUpdate("Append mode disabled: can't read %r: %s." % (filename, err))
    return None
  try:
    fp.seek(0, 2)
    fp.seek(max(0, fp.tell() - blocksize))
    lines = fp.readlines()
    del lines[:1]
    sentinel = None
    for line in lines:
      if not line.startswith("\t"):
        sentinel = line
    if not sentinel:
      StatusUpdate("Append mode disabled: can't find sentinel in %r." %
                   filename)
      return None
    return sentinel.rstrip("\n")
  finally:
    fp.close()


class AppVersionUpload(object):
  """Provides facilities to upload a new appversion to the hosting service.

  Attributes:
    server: The AbstractRpcServer to use for the upload.
    config: The AppInfoExternal object derived from the app.yaml file.
    app_id: The application string from 'config'.
    version: The version string from 'config'.
    files: A dictionary of files to upload to the server, mapping path to
      hash of the file contents.
    in_transaction: True iff a transaction with the server has started.
      An AppVersionUpload can do only one transaction at a time.
  """

  def __init__(self, server, config):
    """Creates a new AppVersionUpload.

    Args:
      server: The RPC server to use. Should be an instance of HttpRpcServer or
        TestRpcServer.
      config: An AppInfoExternal object that specifies the configuration for
        this application.
    """
    self.server = server
    self.config = config
    self.app_id = self.config.application
    self.version = self.config.version
    self.files = {}
    self.in_transaction = False

  def _Hash(self, content):
    """Compute the hash of the content.

    Arg:
      content: The data to hash as a string.

    Returns:
      The string representation of the hash.
    """
    h = sha.new(content).hexdigest()
    return '%s_%s_%s_%s_%s' % (h[0:8], h[8:16], h[16:24], h[24:32], h[32:40])

  def AddFile(self, path, file_handle):
    """Adds the provided file to the list to be pushed to the server.

    Args:
      path: The path the file should be uploaded as.
      file_handle: A stream containing data to upload.
    """
    assert not self.in_transaction, "Already in a transaction."
    assert file_handle is not None

    reason = appinfo.ValidFilename(path)
    if reason != '':
      logging.error(reason)
      return

    pos = file_handle.tell()
    content_hash = self._Hash(file_handle.read())
    file_handle.seek(pos, 0)

    self.files[path] = content_hash

  def Begin(self):
    """Begins the transaction, returning a list of files that need uploading.

    All calls to AddFile must be made before calling Begin().

    Returns:
      A list of pathnames for files that should be uploaded using UploadFile()
      before Commit() can be called.
    """
    assert not self.in_transaction, "Already in a transaction."

    StatusUpdate("Initiating update.")
    self.server.Send("/api/appversion/create", app_id=self.app_id,
                     version=self.version, payload=self.config.ToYAML())
    self.in_transaction = True

    files_to_clone = []
    blobs_to_clone = []
    for path, content_hash in self.files.iteritems():
      mime_type = GetMimeTypeIfStaticFile(self.config, path)
      if mime_type is not None:
        blobs_to_clone.append((path, content_hash, mime_type))
      else:
        files_to_clone.append((path, content_hash))

    files_to_upload = {}

    def CloneFiles(url, files, file_type):
      if len(files) == 0:
        return

      StatusUpdate("Cloning %d %s file%s." %
                   (len(files), file_type, len(files) != 1 and "s" or ""))
      for i in xrange(0, len(files), MAX_FILES_TO_CLONE):
        if i > 0 and i % MAX_FILES_TO_CLONE == 0:
          StatusUpdate("Cloned %d files." % i)

        chunk = files[i:min(len(files), i + MAX_FILES_TO_CLONE)]
        result = self.server.Send(url,
                                  app_id=self.app_id, version=self.version,
                                  payload=BuildClonePostBody(chunk))
        if result:
          files_to_upload.update(dict(
              (f, self.files[f]) for f in result.split(LIST_DELIMITER)))

    CloneFiles("/api/appversion/cloneblobs", blobs_to_clone, "static")
    CloneFiles("/api/appversion/clonefiles", files_to_clone, "application")

    logging.info('Files to upload: ' + str(files_to_upload))

    self.files = files_to_upload
    return sorted(files_to_upload.iterkeys())

  def UploadFile(self, path, file_handle):
    """Uploads a file to the hosting service.

    Must only be called after Begin().
    The path provided must be one of those that were returned by Begin().

    Args:
      path: The path the file is being uploaded as.
      file_handle: A file-like object containing the data to upload.

    Raises:
      KeyError: The provided file is not amongst those to be uploaded.
    """
    assert self.in_transaction, "Begin() must be called before UploadFile()."
    if path not in self.files:
      raise KeyError("File '%s' is not in the list of files to be uploaded."
                     % path)

    del self.files[path]
    mime_type = GetMimeTypeIfStaticFile(self.config, path)
    if mime_type is not None:
      self.server.Send("/api/appversion/addblob", app_id=self.app_id,
                       version=self.version, path=path, content_type=mime_type,
                       payload=file_handle.read())
    else:
      self.server.Send("/api/appversion/addfile", app_id=self.app_id,
                       version=self.version, path=path,
                       payload=file_handle.read())

  def Commit(self):
    """Commits the transaction, making the new app version available.

    All the files returned by Begin() must have been uploaded with UploadFile()
    before Commit() can be called.

    Raises:
      Exception: Some required files were not uploaded.
    """
    assert self.in_transaction, "Begin() must be called before Commit()."
    if self.files:
      raise Exception("Not all required files have been uploaded.")

    StatusUpdate("Closing update.")
    self.server.Send("/api/appversion/commit", app_id=self.app_id,
                     version=self.version)
    self.in_transaction = False

  def Rollback(self):
    """Rolls back the transaction if one is in progress."""
    if not self.in_transaction:
      return
    StatusUpdate("Rolling back the update.")
    self.server.Send("/api/appversion/rollback", app_id=self.app_id,
                     version=self.version)
    self.in_transaction = False
    self.files = {}

  def DoUpload(self, paths, max_size, openfunc):
    """Uploads a new appversion with the given config and files to the server.

    Args:
      paths: An iterator that yields the relative paths of the files to upload.
      max_size: The maximum size file to upload.
      openfunc: A function that takes a path and returns a file-like object.
    """
    logging.info("Reading app configuration.")

    try:
      StatusUpdate("Scanning files on local disk.")
      num_files = 0
      for path in paths:
        file_handle = openfunc(path)
        try:
          if self.config.skip_files.match(path):
            logging.info("Ignoring file '%s': File matches ignore regex.",
                         path)
          else:
            file_length = GetFileLength(file_handle)
            if file_length > max_size:
              logging.error("Ignoring file '%s': Too long "
                            "(max %d bytes, file is %d bytes)",
                            path, max_size, file_length)
            else:
              logging.info("Processing file '%s'", path)
              self.AddFile(path, file_handle)
        finally:
          file_handle.close()
        num_files += 1
        if num_files % 500 == 0:
          StatusUpdate("Scanned %d files." % num_files)
    except KeyboardInterrupt:
      logging.info("User interrupted. Aborting.")
      raise
    except EnvironmentError, e:
      logging.error("An error occurred processing file '%s': %s. Aborting.",
                    path, e)
      raise

    try:
      missing_files = self.Begin()
      if len(missing_files) > 0:
        StatusUpdate("Uploading %d files." % len(missing_files))
        num_files = 0
        for missing_file in missing_files:
          logging.info("Uploading file '%s'" % missing_file)
          file_handle = openfunc(missing_file)
          try:
            self.UploadFile(missing_file, file_handle)
          finally:
            file_handle.close()
          num_files += 1
          if num_files % 500 == 0:
            StatusUpdate("Uploaded %d files." % num_files)

      self.Commit()
    except KeyboardInterrupt:
      logging.info("User interrupted. Aborting.")
      self.Rollback()
      raise
    except:
      logging.error("An unexpected error occurred. Aborting.")
      self.Rollback()
      raise

    logging.info("Done!")


def FileIterator(base, separator=os.path.sep):
  """Walks a directory tree, returning all the files. Follows symlinks.

  Args:
    base: The base path to search for files under.
    separator: Path separator used by the running system's platform.

  Yields:
    Paths of files found, relative to base.
  """
  dirs = [""]
  while dirs:
    current_dir = dirs.pop()
    for entry in os.listdir(os.path.join(base, current_dir)):
      name = os.path.join(current_dir, entry)
      fullname = os.path.join(base, name)
      if os.path.isfile(fullname):
        if separator == "\\":
          name = name.replace("\\", "/")
        yield name
      elif os.path.isdir(fullname):
        dirs.append(name)


def GetFileLength(fh):
  """Returns the length of the file represented by fh.

  This function is capable of finding the length of any seekable stream,
  unlike os.fstat, which only works on file streams.

  Args:
    fh: The stream to get the length of.

  Returns:
    The length of the stream.
  """
  pos = fh.tell()
  fh.seek(0, 2)
  length = fh.tell()
  fh.seek(pos, 0)
  return length


def GetPlatformToken(os_module=os, sys_module=sys, platform=sys.platform):
  """Returns a 'User-agent' token for the host system platform.

  Args:
    os_module, sys_module, platform: Used for testing.

  Returns:
    String containing the platform token for the host system.
  """
  if hasattr(sys_module, "getwindowsversion"):
    windows_version = sys_module.getwindowsversion()
    version_info = ".".join(str(i) for i in windows_version[:4])
    return platform + "/" + version_info
  elif hasattr(os_module, "uname"):
    uname = os_module.uname()
    return "%s/%s" % (uname[0], uname[2])
  else:
    return "unknown"


def GetUserAgent(get_version=GetVersionObject, get_platform=GetPlatformToken):
  """Determines the value of the 'User-agent' header to use for HTTP requests.

  If the 'APPCFG_SDK_NAME' environment variable is present, that will be
  used as the first product token in the user-agent.

  Args:
    get_version, get_platform: Used for testing.

  Returns:
    String containing the 'user-agent' header value, which includes the SDK
    version, the platform information, and the version of Python;
    e.g., "appcfg_py/1.0.1 Darwin/9.2.0 Python/2.5.2".
  """
  product_tokens = []

  sdk_name = os.environ.get("APPCFG_SDK_NAME")
  if sdk_name:
    product_tokens.append(sdk_name)
  else:
    version = get_version()
    if version is None:
      release = "unknown"
    else:
      release = version["release"]

    product_tokens.append("appcfg_py/%s" % release)

  product_tokens.append(get_platform())

  python_version = ".".join(str(i) for i in sys.version_info)
  product_tokens.append("Python/%s" % python_version)

  return " ".join(product_tokens)


class AppCfgApp(object):
  """Singleton class to wrap AppCfg tool functionality.

  This class is responsible for parsing the command line and executing
  the desired action on behalf of the user.  Processing files and
  communicating with the server is handled by other classes.

  Attributes:
    actions: A dictionary mapping action names to Action objects.
    action: The Action specified on the command line.
    parser: An instance of optparse.OptionParser.
    options: The command line options parsed by 'parser'.
    argv: The original command line as a list.
    args: The positional command line args left over after parsing the options.
    raw_input_fn: Function used for getting raw user input, like email.
    password_input_fn: Function used for getting user password.
    error_fh: Unexpected HTTPErrors are printed to this file handle.

  Attributes for testing:
    parser_class: The class to use for parsing the command line.  Because
      OptionsParser will exit the program when there is a parse failure, it
      is nice to subclass OptionsParser and catch the error before exiting.
  """

  def __init__(self, argv, parser_class=optparse.OptionParser,
               rpc_server_class=HttpRpcServer,
               raw_input_fn=raw_input,
               password_input_fn=getpass.getpass,
               error_fh=sys.stderr):
    """Initializer.  Parses the cmdline and selects the Action to use.

    Initializes all of the attributes described in the class docstring.
    Prints help or error messages if there is an error parsing the cmdline.

    Args:
      argv: The list of arguments passed to this program.
      parser_class: Options parser to use for this application.
      rpc_server_class: RPC server class to use for this application.
      raw_input_fn: Function used for getting user email.
      password_input_fn: Function used for getting user password.
      error_fh: Unexpected HTTPErrors are printed to this file handle.
    """
    self.parser_class = parser_class
    self.argv = argv
    self.rpc_server_class = rpc_server_class
    self.raw_input_fn = raw_input_fn
    self.password_input_fn = password_input_fn
    self.error_fh = error_fh

    self.parser = self._GetOptionParser()
    for action in self.actions.itervalues():
      action.options(self, self.parser)

    self.options, self.args = self.parser.parse_args(argv[1:])

    if len(self.args) < 1:
      self._PrintHelpAndExit()
    if self.args[0] not in self.actions:
      self.parser.error("Unknown action '%s'\n%s" %
                        (self.args[0], self.parser.get_description()))
    action_name = self.args.pop(0)
    self.action = self.actions[action_name]

    self.parser, self.options = self._MakeSpecificParser(self.action)

    if self.options.help:
      self._PrintHelpAndExit()

    if self.options.verbose == 2:
      logging.getLogger().setLevel(logging.INFO)
    elif self.options.verbose == 3:
      logging.getLogger().setLevel(logging.DEBUG)

    global verbosity
    verbosity = self.options.verbose

  def Run(self):
    """Executes the requested action.

    Catches any HTTPErrors raised by the action and prints them to stderr.
    """
    try:
      self.action.function(self)
    except urllib2.HTTPError, e:
      body = e.read()
      print >>self.error_fh, ("Error %d: --- begin server output ---\n"
                              "%s\n--- end server output ---" %
                              (e.code, body.rstrip("\n")))
    except yaml_errors.EventListenerError, e:
      print >>self.error_fh, ("Error parsing yaml file:\n%s" % e)

  def _GetActionDescriptions(self):
    """Returns a formatted string containing the short_descs for all actions."""
    action_names = self.actions.keys()
    action_names.sort()
    desc = ""
    for action_name in action_names:
      desc += "  %s: %s\n" % (action_name, self.actions[action_name].short_desc)
    return desc

  def _GetOptionParser(self):
    """Creates an OptionParser with generic usage and description strings.

    Returns:
      An OptionParser instance.
    """

    class Formatter(optparse.IndentedHelpFormatter):
      """Custom help formatter that does not reformat the description."""
      def format_description(self, description):
        return description + "\n"

    desc = self._GetActionDescriptions()
    desc = ("Action must be one of:\n%s"
            "Use 'help <action>' for a detailed description.") % desc

    parser = self.parser_class(usage="%prog [options] <action>",
                               description=desc,
                               formatter=Formatter(),
                               conflict_handler="resolve")
    parser.add_option("-h", "--help", action="store_true",
                      dest="help", help="Show the help message and exit.")
    parser.add_option("-q", "--quiet", action="store_const", const=0,
                      dest="verbose", help="Print errors only.")
    parser.add_option("-v", "--verbose", action="store_const", const=2,
                      dest="verbose", default=1,
                      help="Print info level logs.")
    parser.add_option("--noisy", action="store_const", const=3,
                      dest="verbose", help="Print all logs.")
    parser.add_option("-s", "--server", action="store", dest="server",
                      default="appengine.google.com",
                      metavar="SERVER", help="The server to connect to.")
    parser.add_option("-e", "--email", action="store", dest="email",
                      metavar="EMAIL", default=None,
                      help="The username to use. Will prompt if omitted.")
    parser.add_option("-H", "--host", action="store", dest="host",
                      metavar="HOST", default=None,
                      help="Overrides the Host header sent with all RPCs.")
    parser.add_option("--no_cookies", action="store_false",
                      dest="save_cookies", default=True,
                      help="Do not save authentication cookies to local disk.")
    parser.add_option("--passin", action="store_true",
                      dest="passin", default=False,
                      help="Read the login password from stdin.")
    return parser

  def _MakeSpecificParser(self, action):
    """Creates a new parser with documentation specific to 'action'.

    Args:
      action: An Action instance to be used when initializing the new parser.

    Returns:
      A tuple containing:
      parser: An instance of OptionsParser customized to 'action'.
      options: The command line options after re-parsing.
    """
    parser = self._GetOptionParser()
    parser.set_usage(action.usage)
    parser.set_description("%s\n%s" % (action.short_desc, action.long_desc))
    action.options(self, parser)
    options, args = parser.parse_args(self.argv[1:])
    return parser, options

  def _PrintHelpAndExit(self, exit_code=2):
    """Prints the parser's help message and exits the program.

    Args:
      exit_code: The integer code to pass to sys.exit().
    """
    self.parser.print_help()
    sys.exit(exit_code)

  def _GetRpcServer(self):
    """Returns an instance of an AbstractRpcServer.

    Returns:
      A new AbstractRpcServer, on which RPC calls can be made.
    """

    def GetUserCredentials():
      """Prompts the user for a username and password."""
      email = self.options.email
      if email is None:
        email = self.raw_input_fn("Email: ")

      password_prompt = "Password for %s: " % email
      if self.options.passin:
        password = self.raw_input_fn(password_prompt)
      else:
        password = self.password_input_fn(password_prompt)

      return (email, password)

    if self.options.host and self.options.host == "localhost":
      email = self.options.email
      if email is None:
        email = "test@example.com"
        logging.info("Using debug user %s.  Override with --email" % email)
      server = self.rpc_server_class(
          self.options.server,
          lambda: (email, "password"),
          host_override=self.options.host,
          extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email},
          save_cookies=self.options.save_cookies)
      server.authenticated = True
      return server

    return self.rpc_server_class(self.options.server, GetUserCredentials,
                                 host_override=self.options.host,
                                 save_cookies=self.options.save_cookies)

  def _FindYaml(self, basepath, file_name):
    """Find yaml files in application directory.

    Args:
      basepath: Base application directory.
      file_name: Filename without extension to search for.

    Returns:
      Path to located yaml file if one exists, else None.
    """
    if not os.path.isdir(basepath):
      self.parser.error("Not a directory: %s" % basepath)

    for yaml_file in (file_name + '.yaml', file_name + '.yml'):
      yaml_path = os.path.join(basepath, yaml_file)
      if os.path.isfile(yaml_path):
        return yaml_path

    return None

  def _ParseAppYaml(self, basepath):
    """Parses the app.yaml file.

    Returns:
      An AppInfoExternal object.
    """
    appyaml_filename = self._FindYaml(basepath, "app")
    if appyaml_filename is None:
      self.parser.error("Directory does not contain an app.yaml "
                        "configuration file.")

    fh = open(appyaml_filename, "r")
    try:
      appyaml = appinfo.LoadSingleAppInfo(fh)
    finally:
      fh.close()
    return appyaml

  def _ParseIndexYaml(self, basepath):
    """Parses the index.yaml file.

    Returns:
      A single parsed yaml file or None if the file does not exist.
    """
    file_name = self._FindYaml(basepath, "index")
    if file_name is not None:
      fh = open(file_name, "r")
      try:
        index_defs = datastore_index.ParseIndexDefinitions(fh)
      finally:
        fh.close()
      return index_defs
    return None

  def Help(self):
    """Prints help for a specific action.

    Expects self.args[0] to contain the name of the action in question.
    Exits the program after printing the help message.
    """
    if len(self.args) != 1 or self.args[0] not in self.actions:
      self.parser.error("Expected a single action argument. Must be one of:\n" +
                        self._GetActionDescriptions())

    action = self.actions[self.args[0]]
    self.parser, options = self._MakeSpecificParser(action)
    self._PrintHelpAndExit(exit_code=0)

  def Update(self):
    """Updates and deploys a new appversion."""
    if len(self.args) != 1:
      self.parser.error("Expected a single <directory> argument.")

    basepath = self.args[0]
    appyaml = self._ParseAppYaml(basepath)
    rpc_server = self._GetRpcServer()

    updatecheck = UpdateCheck(rpc_server, appyaml)
    updatecheck.CheckForUpdates()

    appversion = AppVersionUpload(rpc_server, appyaml)
    appversion.DoUpload(FileIterator(basepath), self.options.max_size,
                        lambda path: open(os.path.join(basepath, path), "rb"))

    index_defs = self._ParseIndexYaml(basepath)
    if index_defs:
      index_upload = IndexDefinitionUpload(rpc_server, appyaml, index_defs)
      try:
        index_upload.DoUpload()
      except urllib2.HTTPError, e:
        StatusUpdate("Error %d: --- begin server output ---\n"
                     "%s\n--- end server output ---" %
                     (e.code, e.read().rstrip("\n")))
        print >> self.error_fh, (
          "Your app was updated, but there was an error updating your indexes. "
          "Please retry later with appcfg.py update_indexes.")

  def _UpdateOptions(self, parser):
    """Adds update-specific options to 'parser'.

    Args:
      parser: An instance of OptionsParser.
    """
    parser.add_option("-S", "--max_size", type="int", dest="max_size",
                      default=1048576, metavar="SIZE",
                      help="Maximum size of a file to upload.")

  def VacuumIndexes(self):
    """Deletes unused indexes."""
    if len(self.args) != 1:
      self.parser.error("Expected a single <directory> argument.")

    basepath = self.args[0]
    config = self._ParseAppYaml(basepath)

    index_defs = self._ParseIndexYaml(basepath)
    if index_defs is None:
      index_defs = datastore_index.IndexDefinitions()

    rpc_server = self._GetRpcServer()
    vacuum = VacuumIndexesOperation(rpc_server,
                                    config,
                                    self.options.force_delete)
    vacuum.DoVacuum(index_defs)

  def _VacuumIndexesOptions(self, parser):
    """Adds vacuum_indexes-specific options to 'parser'.

    Args:
      parser: An instance of OptionsParser.
    """
    parser.add_option("-f", "--force", action="store_true", dest="force_delete",
                      default=False,
                      help="Force deletion without being prompted.")

  def UpdateIndexes(self):
    """Updates indexes."""
    if len(self.args) != 1:
      self.parser.error("Expected a single <directory> argument.")

    basepath = self.args[0]
    appyaml = self._ParseAppYaml(basepath)
    rpc_server = self._GetRpcServer()

    index_defs = self._ParseIndexYaml(basepath)
    if index_defs:
      index_upload = IndexDefinitionUpload(rpc_server, appyaml, index_defs)
      index_upload.DoUpload()

  def Rollback(self):
    """Does a rollback of any existing transaction for this app version."""
    if len(self.args) != 1:
      self.parser.error("Expected a single <directory> argument.")

    basepath = self.args[0]
    appyaml = self._ParseAppYaml(basepath)

    appversion = AppVersionUpload(self._GetRpcServer(), appyaml)
    appversion.in_transaction = True
    appversion.Rollback()

  def RequestLogs(self):
    """Write request logs to a file."""
    if len(self.args) != 2:
      self.parser.error(
          "Expected a <directory> argument and an <output_file> argument.")
    if (self.options.severity is not None and
        not 0 <= self.options.severity <= MAX_LOG_LEVEL):
      self.parser.error(
          "Severity range is 0 (DEBUG) through %s (CRITICAL)." % MAX_LOG_LEVEL)

    if self.options.num_days is None:
      self.options.num_days = int(not self.options.append)
    basepath = self.args[0]
    appyaml = self._ParseAppYaml(basepath)
    rpc_server = self._GetRpcServer()
    logs_requester = LogsRequester(rpc_server, appyaml, self.args[1],
                                   self.options.num_days,
                                   self.options.append,
                                   self.options.severity,
                                   time.time())
    logs_requester.DownloadLogs()

  def _RequestLogsOptions(self, parser):
    """Ads request_logs-specific options to 'parser'.

    Args:
      parser: An instance of OptionsParser.
    """
    parser.add_option("-n", "--num_days", type="int", dest="num_days",
                      action="store", default=None,
                      help="Number of days worth of log data to get. "
                           "The cut-off point is midnight UTC. "
                           "Use 0 to get all available logs. "
                           "Default is 1, unless --append is also given; "
                           "then the default is 0.")
    parser.add_option("-a", "--append", dest="append",
                       action="store_true", default=False,
                      help="Append to existing file.")
    parser.add_option("--severity", type="int", dest="severity",
                      action="store", default=None,
                      help="Severity of app-level log messages to get. "
                           "The range is 0 (DEBUG) through 4 (CRITICAL). "
                           "If omitted, only request logs are returned.")

  class Action(object):
    """Contains information about a command line action.

    Attributes:
      function: An AppCfgApp function that will perform the appropriate
        action.
      usage: A command line usage string.
      short_desc: A one-line description of the action.
      long_desc: A detailed description of the action.  Whitespace and
        formatting will be preserved.
      options: A function that will add extra options to a given OptionParser
        object.
    """

    def __init__(self, function, usage, short_desc, long_desc="",
                 options=lambda obj, parser: None):
      """Initializer for the class attributes."""
      self.function = function
      self.usage = usage
      self.short_desc = short_desc
      self.long_desc = long_desc
      self.options = options

  actions = {

      "help": Action(
        function=Help,
        usage="%prog help <action>",
        short_desc="Print help for a specific action."),

      "update": Action(
        function=Update,
        usage="%prog [options] update <directory>",
        options=_UpdateOptions,
        short_desc="Create or update an app version.",
        long_desc="""
Specify a directory that contains all of the files required by
the app, and appcfg.py will create/update the app version referenced
in the app.yaml file at the top level of that directory.  appcfg.py
will follow symlinks and recursively upload all files to the server.
Temporary or source control files (e.g. foo~, .svn/*) will be skipped."""),

      "update_indexes": Action(
        function=UpdateIndexes,
        usage="%prog [options] update_indexes <directory>",
        short_desc="Update application indexes.",
        long_desc="""
The 'update_indexes' command will add additional indexes which are not currently
in production as well as restart any indexes that were not completed."""),

      "vacuum_indexes": Action(
        function=VacuumIndexes,
        usage="%prog [options] vacuum_indexes <directory>",
        options=_VacuumIndexesOptions,
        short_desc="Delete unused indexes from application.",
        long_desc="""
The 'vacuum_indexes' command will help clean up indexes which are no longer
in use.  It does this by comparing the local index configuration with
indexes that are actually defined on the server.  If any indexes on the
server do not exist in the index configuration file, the user is given the
option to delete them."""),

      "rollback": Action(
        function=Rollback,
        usage="%prog [options] rollback <directory>",
        short_desc="Rollback an in-progress update.",
        long_desc="""
The 'update' command requires a server-side transaction.  Use 'rollback'
if you get an error message about another transaction being in progress
and you are sure that there is no such transaction."""),

      "request_logs": Action(
        function=RequestLogs,
        usage="%prog [options] request_logs <directory> <output_file>",
        options=_RequestLogsOptions,
        short_desc="Write request logs in Apache common log format.",
        long_desc="""
The 'request_logs' command exports the request logs from your application
to a file.  It will write Apache common log format records ordered
chronologically.  If output file is '-' stdout will be written."""),

  }


def main(argv):
  logging.basicConfig(format=("%(asctime)s %(levelname)s %(filename)s:"
                              "%(lineno)s %(message)s "))
  try:
    AppCfgApp(argv).Run()
  except KeyboardInterrupt:
    StatusUpdate("Interrupted.")
    sys.exit(1)


if __name__ == "__main__":
  main(sys.argv)
