#!/usr/bin/env python

#

# Copyright 2007 Google Inc.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

"""Tool for deploying apps to an app server.

Currently, the application only uploads new appversions. To do this, it first

walks the directory tree rooted at the path the user specifies, adding all the

files it finds to a list. It then uploads the application configuration

(app.yaml) to the server using HTTP, followed by uploading each of the files.

It then commits the transaction with another request.

The bulk of this work is handled by the AppVersionUpload class, which exposes

methods to add to the list of files, fetch a list of modified files, upload

files, and commit or rollback the transaction.

"""

import cookielib

import datetime

import getpass

import logging

import mimetypes

import optparse

import os

import re

import sha

import socket

import sys

import tempfile

import time

import urllib

import urllib2

import google

from google.appengine.api import appinfo

from google.appengine.api import validation

from google.appengine.api import yaml_errors

from google.appengine.api import yaml_object

from google.appengine.datastore import datastore_index

import yaml

MAX_FILES_TO_CLONE = 100

LIST_DELIMITER = "\n"

TUPLE_DELIMITER = "|"

VERSION_FILE = "../VERSION"

UPDATE_CHECK_TIMEOUT = 3

NAG_FILE = ".appcfg_nag"

MAX_LOG_LEVEL = 4

verbosity = 1

def StatusUpdate(msg):

  """Print a status message to stderr.

  If 'verbosity' is greater than 0, print the message.

  Args:

    msg: The string to print.

  """

  if verbosity > 0:

    print >>sys.stderr, msg

class ClientLoginError(urllib2.HTTPError):

  """Raised to indicate there was an error authenticating with ClientLogin."""

  def __init__(self, url, code, msg, headers, args):

    urllib2.HTTPError.__init__(self, url, code, msg, headers, None)

    self.args = args

    self.reason = args["Error"]

class AbstractRpcServer(object):

  """Provides a common interface for a simple RPC server."""

  def __init__(self, host, auth_function, host_override=None,

               extra_headers=None, save_cookies=False):

    """Creates a new HttpRpcServer.

    Args:

      host: The host to send requests to.

      auth_function: A function that takes no arguments and returns an

        (email, password) tuple when called. Will be called if authentication

        is required.

      host_override: The host header to send to the server (defaults to host).

      extra_headers: A dict of extra headers to append to every request. Values

        supplied here will override other default headers that are supplied.

      save_cookies: If True, save the authentication cookies to local disk.

        If False, use an in-memory cookiejar instead.  Subclasses must

        implement this functionality.  Defaults to False.

    """

    self.host = host

    self.host_override = host_override

    self.auth_function = auth_function

    self.authenticated = False

    self.extra_headers = {

      "User-agent": GetUserAgent()

    }

    if extra_headers:

      self.extra_headers.update(extra_headers)

    self.save_cookies = save_cookies

    self.cookie_jar = cookielib.MozillaCookieJar()

    self.opener = self._GetOpener()

    if self.host_override:

      logging.info("Server: %s; Host: %s", self.host, self.host_override)

    else:

      logging.info("Server: %s", self.host)

  def _GetOpener(self):

    """Returns an OpenerDirector for making HTTP requests.

    Returns:

      A urllib2.OpenerDirector object.

    """

    raise NotImplemented()

  def _CreateRequest(self, url, data=None):

    """Creates a new urllib request."""

    logging.debug("Creating request for: '%s' with payload:\n%s", url, data)

    req = urllib2.Request(url, data=data)

    if self.host_override:

      req.add_header("Host", self.host_override)

    for key, value in self.extra_headers.iteritems():

      req.add_header(key, value)

    return req

  def _GetAuthToken(self, email, password):

    """Uses ClientLogin to authenticate the user, returning an auth token.

    Args:

      email:    The user's email address

      password: The user's password

    Raises:

      ClientLoginError: If there was an error authenticating with ClientLogin.

      HTTPError: If there was some other form of HTTP error.

    Returns:

      The authentication token returned by ClientLogin.

    """

    req = self._CreateRequest(

        url="https://www.google.com/accounts/ClientLogin",

        data=urllib.urlencode({

            "Email": email,

            "Passwd": password,

            "service": "ah",

            "source": "Google-appcfg-1.0",

            "accountType": "HOSTED_OR_GOOGLE"

        })

    )

    try:

      response = self.opener.open(req)

      response_body = response.read()

      response_dict = dict(x.split("=")

                           for x in response_body.split("\n") if x)

      return response_dict["Auth"]

    except urllib2.HTTPError, e:

      if e.code == 403:

        body = e.read()

        response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)

        raise ClientLoginError(req.get_full_url(), e.code, e.msg,

                               e.headers, response_dict)

      else:

        raise

  def _GetAuthCookie(self, auth_token):

    """Fetches authentication cookies for an authentication token.

    Args:

      auth_token: The authentication token returned by ClientLogin.

    Raises:

      HTTPError: If there was an error fetching the authentication cookies.

    """

    continue_location = "http://localhost/"

    args = {"continue": continue_location, "auth": auth_token}

    login_path = os.environ.get("APPCFG_LOGIN_PATH", "/_ah")

    req = self._CreateRequest("http://%s%s/login?%s" %

                              (self.host, login_path, urllib.urlencode(args)))

    try:

      response = self.opener.open(req)

    except urllib2.HTTPError, e:

      response = e

    if (response.code != 302 or

        response.info()["location"] != continue_location):

      raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,

                              response.headers, response.fp)

    self.authenticated = True

  def _Authenticate(self):

    """Authenticates the user.

    The authentication process works as follows:

     1) We get a username and password from the user

     2) We use ClientLogin to obtain an AUTH token for the user

        (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).

     3) We pass the auth token to /_ah/login on the server to obtain an

        authentication cookie. If login was successful, it tries to redirect

        us to the URL we provided.

    If we attempt to access the upload API without first obtaining an

    authentication cookie, it returns a 401 response and directs us to

    authenticate ourselves with ClientLogin.

    """

    for i in range(3):

      credentials = self.auth_function()

      try:

        auth_token = self._GetAuthToken(credentials[0], credentials[1])

      except ClientLoginError, e:

        if e.reason == "BadAuthentication":

          print >>sys.stderr, "Invalid username or password."

          continue

        if e.reason == "CaptchaRequired":

          print >>sys.stderr, (

              "Please go to\n"

              "https://www.google.com/accounts/DisplayUnlockCaptcha\n"

              "and verify you are a human.  Then try again.")

          break;

        if e.reason == "NotVerified":

          print >>sys.stderr, "Account not verified."

          break

        if e.reason == "TermsNotAgreed":

          print >>sys.stderr, "User has not agreed to TOS."

          break

        if e.reason == "AccountDeleted":

          print >>sys.stderr, "The user account has been deleted."

          break

        if e.reason == "AccountDisabled":

          print >>sys.stderr, "The user account has been disabled."

          break

        if e.reason == "ServiceDisabled":

          print >>sys.stderr, ("The user's access to the service has been "

                               "disabled.")

          break

        if e.reason == "ServiceUnavailable":

          print >>sys.stderr, "The service is not available; try again later."

          break

        raise

      self._GetAuthCookie(auth_token)

      return

  def Send(self, request_path, payload="",

           content_type="application/octet-stream",

           timeout=None,

           **kwargs):

    """Sends an RPC and returns the response.

    Args:

      request_path: The path to send the request to, eg /api/appversion/create.

      payload: The body of the request, or None to send an empty request.

      content_type: The Content-Type header to use.

      timeout: timeout in seconds; default None i.e. no timeout.

        (Note: for large requests on OS X, the timeout doesn't work right.)

      kwargs: Any keyword arguments are converted into query string parameters.

    Returns:

      The response body, as a string.

    """

    if not self.authenticated:

      self._Authenticate()

    old_timeout = socket.getdefaulttimeout()

    socket.setdefaulttimeout(timeout)

    try:

      tries = 0

      while True:

        tries += 1

        args = dict(kwargs)

        url = "http://%s%s?%s" % (self.host, request_path,

                                  urllib.urlencode(args))

        req = self._CreateRequest(url=url, data=payload)

        req.add_header("Content-Type", content_type)

        req.add_header("X-appcfg-api-version", "1")

        try:

          f = self.opener.open(req)

          response = f.read()

          f.close()

          return response

        except urllib2.HTTPError, e:

          if tries > 3:

            raise

          elif e.code == 401:

            self._Authenticate()

          elif e.code >= 500 and e.code < 600:

            continue

          else:

            raise

    finally:

      socket.setdefaulttimeout(old_timeout)

class HttpRpcServer(AbstractRpcServer):

  """Provides a simplified RPC-style interface for HTTP requests."""

  DEFAULT_COOKIE_FILE_PATH = "~/.appcfg_cookies"

  def _Authenticate(self):

    """Save the cookie jar after authentication."""

    super(HttpRpcServer, self)._Authenticate()

    if self.cookie_jar.filename is not None and self.save_cookies:

      StatusUpdate("Saving authentication cookies to %s" %

                   self.cookie_jar.filename)

      self.cookie_jar.save()

  def _GetOpener(self):

    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:

      A urllib2.OpenerDirector object.

    """

    opener = urllib2.OpenerDirector()

    opener.add_handler(urllib2.ProxyHandler())

    opener.add_handler(urllib2.UnknownHandler())

    opener.add_handler(urllib2.HTTPHandler())

    opener.add_handler(urllib2.HTTPDefaultErrorHandler())

    opener.add_handler(urllib2.HTTPSHandler())

    opener.add_handler(urllib2.HTTPErrorProcessor())

    if self.save_cookies:

      self.cookie_jar.filename = os.path.expanduser(HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)

      if os.path.exists(self.cookie_jar.filename):

        try:

          self.cookie_jar.load()

          self.authenticated = True

          StatusUpdate("Loaded authentication cookies from %s" %

                       self.cookie_jar.filename)

        except (OSError, IOError, cookielib.LoadError), e:

          logging.debug("Could not load authentication cookies; %s: %s",

                        e.__class__.__name__, e)

          self.cookie_jar.filename = None

      else:

        try:

          fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600)

          os.close(fd)

        except (OSError, IOError), e:

          logging.debug("Could not create authentication cookies file; %s: %s",

                        e.__class__.__name__, e)

          self.cookie_jar.filename = None

    opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))

    return opener

def GetMimeTypeIfStaticFile(config, filename):

  """Looks up the mime type for 'filename'.

  Uses the handlers in 'config' to determine if the file should

  be treated as a static file.

  Args:

    config: The app.yaml object to check the filename against.

    filename: The name of the file.

  Returns:

    The mime type string.  For example, 'text/plain' or 'image/gif'.

    None if this is not a static file.

  """

  for handler in config.handlers:

    handler_type = handler.GetHandlerType()

    if handler_type in ("static_dir", "static_files"):

      if handler_type == "static_dir":

        regex = os.path.join(re.escape(handler.GetHandler()), ".*")

      else:

        regex = handler.upload

      if re.match(regex, filename):

        if handler.mime_type is not None:

          return handler.mime_type

        else:

          guess = mimetypes.guess_type(filename)[0]

          if guess is None:

            default = "application/octet-stream"

            print >>sys.stderr, ("Could not guess mimetype for %s.  Using %s."

                                 % (filename, default))

            return default

          return guess

  return None

def BuildClonePostBody(file_tuples):

  """Build the post body for the /api/clone{files,blobs} urls.

  Args:

    file_tuples: A list of tuples.  Each tuple should contain the entries

      appropriate for the endpoint in question.

  Returns:

    A string containing the properly delimited tuples.

  """

  file_list = []

  for tup in file_tuples:

    path = tup[0]

    tup = tup[1:]

    file_list.append(TUPLE_DELIMITER.join([path] + list(tup)))

  return LIST_DELIMITER.join(file_list)

class NagFile(validation.Validated):

  """A validated YAML class to represent the user's nag preferences.

  Attributes:

    timestamp: The timestamp of the last nag.

    opt_in: True if the user wants to check for updates on dev_appserver

      start.  False if not.  May be None if we have not asked the user yet.

  """

  ATTRIBUTES = {

    "timestamp": validation.TYPE_FLOAT,

    "opt_in": validation.Optional(validation.TYPE_BOOL),

  }

  @staticmethod

  def Load(nag_file):

    """Load a single NagFile object where one and only one is expected.

    Args:

      nag_file: A file-like object or string containing the yaml data to parse.

    Returns:

      A NagFile instance.

    """

    return yaml_object.BuildSingleObject(NagFile, nag_file)

def GetVersionObject(isfile=os.path.isfile, open_fn=open):

  """Gets the version of the SDK by parsing the VERSION file.

  Args:

    isfile, open_fn: Used for testing.

  Returns:

    A Yaml object or None if the VERSION file does not exist.

  """

  version_filename = os.path.join(os.path.dirname(google.__file__),

                                  VERSION_FILE)

  if not isfile(version_filename):

    logging.error("Could not find version file at %s", version_filename)

    return None

  version_fh = open_fn(version_filename, "r")

  try:

    version = yaml.safe_load(version_fh)

  finally:

    version_fh.close()

  return version

class UpdateCheck(object):

  """Determines if the local SDK is the latest version.

  Nags the user when there are updates to the SDK.  As the SDK becomes

  more out of date, the language in the nagging gets stronger.  We

  store a little yaml file in the user's home directory so that we nag

  the user only once a week.

  The yaml file has the following field:

    'timestamp': Last time we nagged the user in seconds since the epoch.

  Attributes:

    server: An AbstractRpcServer instance used to check for the latest SDK.

    config: The app's AppInfoExternal.  Needed to determine which api_version

      the app is using.

  """

  def __init__(self,

               server,

               config,

               isdir=os.path.isdir,

               isfile=os.path.isfile,

               open_fn=open):

    """Create a new UpdateCheck.

    Args:

      server: The AbstractRpcServer to use.

      config: The yaml object that specifies the configuration of this

        application.

    Args for testing:

      isdir: Replacement for os.path.isdir.

      isfile: Replacement for os.path.isfile.

      open: Replacement for the open builtin.

    """

    self.server = server

    self.config = config

    self.isdir = isdir

    self.isfile = isfile

    self.open = open_fn

  @staticmethod

  def MakeNagFilename():

    """Returns the filename for the nag file for this user."""

    user_homedir = os.path.expanduser("~/")

    if not os.path.isdir(user_homedir):

      drive, tail = os.path.splitdrive(os.__file__)

      if drive:

        os.environ["HOMEDRIVE"] = drive

    return os.path.expanduser("~/" + NAG_FILE)

  def _ParseVersionFile(self):

    """Parse the local VERSION file.

    Returns:

      A Yaml object or None if the file does not exist.

    """