#!/usr/bin/env python

#

# Copyright 2007 Google Inc.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

"""Tool for deploying apps to an app server.

Currently, the application only uploads new appversions. To do this, it first

walks the directory tree rooted at the path the user specifies, adding all the

files it finds to a list. It then uploads the application configuration

(app.yaml) to the server using HTTP, followed by uploading each of the files.

It then commits the transaction with another request.

The bulk of this work is handled by the AppVersionUpload class, which exposes

methods to add to the list of files, fetch a list of modified files, upload

files, and commit or rollback the transaction.

"""

import calendar

import datetime

import getpass

import logging

import mimetypes

import optparse

import os

import random

import re

import sha

import sys

import tempfile

import time

import urllib

import urllib2

import google

import yaml

from google.appengine.cron import groctimespecification

from google.appengine.api import appinfo

from google.appengine.api import croninfo

from google.appengine.api import queueinfo

from google.appengine.api import validation

from google.appengine.api import yaml_errors

from google.appengine.api import yaml_object

from google.appengine.datastore import datastore_index

from google.appengine.tools import appengine_rpc

from google.appengine.tools import bulkloader

MAX_FILES_TO_CLONE = 100

LIST_DELIMITER = '\n'

TUPLE_DELIMITER = '|'

VERSION_FILE = '../VERSION'

UPDATE_CHECK_TIMEOUT = 3

NAG_FILE = '.appcfg_nag'

MAX_LOG_LEVEL = 4

MAX_BATCH_SIZE = 1000000

MAX_BATCH_COUNT = 100

MAX_BATCH_FILE_SIZE = 200000

BATCH_OVERHEAD = 500

verbosity = 1

appinfo.AppInfoExternal.ATTRIBUTES[appinfo.RUNTIME] = 'python'

_api_versions = os.environ.get('GOOGLE_TEST_API_VERSIONS', '1')

_options = validation.Options(*_api_versions.split(','))

appinfo.AppInfoExternal.ATTRIBUTES[appinfo.API_VERSION] = _options

del _api_versions, _options

def StatusUpdate(msg):

  """Print a status message to stderr.

  If 'verbosity' is greater than 0, print the message.

  Args:

    msg: The string to print.

  """

  if verbosity > 0:

    print >>sys.stderr, msg

def GetMimeTypeIfStaticFile(config, filename):

  """Looks up the mime type for 'filename'.

  Uses the handlers in 'config' to determine if the file should

  be treated as a static file.

  Args:

    config: The app.yaml object to check the filename against.

    filename: The name of the file.

  Returns:

    The mime type string.  For example, 'text/plain' or 'image/gif'.

    None if this is not a static file.

  """

  for handler in config.handlers:

    handler_type = handler.GetHandlerType()

    if handler_type in ('static_dir', 'static_files'):

      if handler_type == 'static_dir':

        regex = os.path.join(re.escape(handler.GetHandler()), '.*')

      else:

        regex = handler.upload

      if re.match(regex, filename):

        if handler.mime_type is not None:

          return handler.mime_type

        else:

          guess = mimetypes.guess_type(filename)[0]

          if guess is None:

            default = 'application/octet-stream'

            print >>sys.stderr, ('Could not guess mimetype for %s.  Using %s.'

                                 % (filename, default))

            return default

          return guess

  return None

def BuildClonePostBody(file_tuples):

  """Build the post body for the /api/clone{files,blobs} urls.

  Args:

    file_tuples: A list of tuples.  Each tuple should contain the entries

      appropriate for the endpoint in question.

  Returns:

    A string containing the properly delimited tuples.

  """

  file_list = []

  for tup in file_tuples:

    path = tup[0]

    tup = tup[1:]

    file_list.append(TUPLE_DELIMITER.join([path] + list(tup)))

  return LIST_DELIMITER.join(file_list)

class NagFile(validation.Validated):

  """A validated YAML class to represent the user's nag preferences.

  Attributes:

    timestamp: The timestamp of the last nag.

    opt_in: True if the user wants to check for updates on dev_appserver

      start.  False if not.  May be None if we have not asked the user yet.

  """

  ATTRIBUTES = {

      'timestamp': validation.TYPE_FLOAT,

      'opt_in': validation.Optional(validation.TYPE_BOOL),

  }

  @staticmethod

  def Load(nag_file):

    """Load a single NagFile object where one and only one is expected.

    Args:

      nag_file: A file-like object or string containing the yaml data to parse.

    Returns:

      A NagFile instance.

    """

    return yaml_object.BuildSingleObject(NagFile, nag_file)

def GetVersionObject(isfile=os.path.isfile, open_fn=open):

  """Gets the version of the SDK by parsing the VERSION file.

  Args:

    isfile: used for testing.

    open_fn: Used for testing.

  Returns:

    A Yaml object or None if the VERSION file does not exist.

  """

  version_filename = os.path.join(os.path.dirname(google.__file__),

                                  VERSION_FILE)

  if not isfile(version_filename):

    logging.error('Could not find version file at %s', version_filename)

    return None

  version_fh = open_fn(version_filename, 'r')

  try:

    version = yaml.safe_load(version_fh)

  finally:

    version_fh.close()

  return version

def RetryWithBackoff(initial_delay, backoff_factor, max_tries, callable_func):

  """Calls a function multiple times, backing off more and more each time.

  Args:

    initial_delay: Initial delay after first try, in seconds.

    backoff_factor: Delay will be multiplied by this factor after each try.

    max_tries: Maximum number of tries.

    callable_func: The method to call, will pass no arguments.

  Returns:

    True if the function succeded in one of its tries.

  Raises:

    Whatever the function raises--an exception will immediately stop retries.

  """

  delay = initial_delay

  while not callable_func() and max_tries > 0:

    StatusUpdate('Will check again in %s seconds.' % delay)

    time.sleep(delay)

    delay *= backoff_factor

    max_tries -= 1

  return max_tries > 0

def _VersionList(release):

  """Parse a version string into a list of ints.

  Args:

    release: The 'release' version, e.g. '1.2.4'.

        (Due to YAML parsing this may also be an int or float.)

  Returns:

    A list of ints corresponding to the parts of the version string

    between periods.  Example:

      '1.2.4' -> [1, 2, 4]

      '1.2.3.4' -> [1, 2, 3, 4]

  Raises:

    ValueError if not all the parts are valid integers.

  """

  return [int(part) for part in str(release).split('.')]

class UpdateCheck(object):

  """Determines if the local SDK is the latest version.

  Nags the user when there are updates to the SDK.  As the SDK becomes

  more out of date, the language in the nagging gets stronger.  We

  store a little yaml file in the user's home directory so that we nag

  the user only once a week.

  The yaml file has the following field:

    'timestamp': Last time we nagged the user in seconds since the epoch.

  Attributes:

    server: An AbstractRpcServer instance used to check for the latest SDK.

    config: The app's AppInfoExternal.  Needed to determine which api_version

      the app is using.

  """

  def __init__(self,

               server,

               config,

               isdir=os.path.isdir,

               isfile=os.path.isfile,

               open_fn=open):

    """Create a new UpdateCheck.

    Args:

      server: The AbstractRpcServer to use.

      config: The yaml object that specifies the configuration of this

        application.

      isdir: Replacement for os.path.isdir (for testing).

      isfile: Replacement for os.path.isfile (for testing).

      open_fn: Replacement for the open builtin (for testing).

    """

    self.server = server

    self.config = config

    self.isdir = isdir

    self.isfile = isfile

    self.open = open_fn

  @staticmethod

  def MakeNagFilename():

    """Returns the filename for the nag file for this user."""

    user_homedir = os.path.expanduser('~/')

    if not os.path.isdir(user_homedir):

      drive, unused_tail = os.path.splitdrive(os.__file__)

      if drive:

        os.environ['HOMEDRIVE'] = drive

    return os.path.expanduser('~/' + NAG_FILE)

  def _ParseVersionFile(self):

    """Parse the local VERSION file.

    Returns:

      A Yaml object or None if the file does not exist.

    """

    return GetVersionObject(isfile=self.isfile, open_fn=self.open)

  def CheckSupportedVersion(self):

    """Determines if the app's api_version is supported by the SDK.

    Uses the api_version field from the AppInfoExternal to determine if

    the SDK supports that api_version.

    Raises:

      SystemExit if the api_version is not supported.

    """

    version = self._ParseVersionFile()

    if version is None:

      logging.error('Could not determine if the SDK supports the api_version '

                    'requested in app.yaml.')

      return

    if self.config.api_version not in version['api_versions']:

      logging.critical('The api_version specified in app.yaml (%s) is not '

                       'supported by this release of the SDK.  The supported '

                       'api_versions are %s.',

                       self.config.api_version, version['api_versions'])

      sys.exit(1)

  def CheckForUpdates(self):

    """Queries the server for updates and nags the user if appropriate.

    Queries the server for the latest SDK version at the same time reporting

    the local SDK version.  The server will respond with a yaml document

    containing the fields:

      'release': The name of the release (e.g. 1.2).

      'timestamp': The time the release was created (YYYY-MM-DD HH:MM AM/PM TZ).

      'api_versions': A list of api_version strings (e.g. ['1', 'beta']).

    We will nag the user with increasing severity if:

    - There is a new release.

    - There is a new release with a new api_version.

    - There is a new release that does not support the api_version named in

      self.config.

    """

    version = self._ParseVersionFile()

    if version is None:

      logging.info('Skipping update check')

      return

    logging.info('Checking for updates to the SDK.')

    try:

      response = self.server.Send('/api/updatecheck',

                                  timeout=UPDATE_CHECK_TIMEOUT,

                                  release=version['release'],

                                  timestamp=version['timestamp'],

                                  api_versions=version['api_versions'])

    except urllib2.URLError, e:

      logging.info('Update check failed: %s', e)

      return

    latest = yaml.safe_load(response)

    if version['release'] == latest['release']:

      logging.info('The SDK is up to date.')

      return

    try:

      this_release = _VersionList(version['release'])

    except ValueError:

      logging.warn('Could not parse this release version (%r)',

                   version['release'])

    else:

      try:

        advertised_release = _VersionList(latest['release'])

      except ValueError:

        logging.warn('Could not parse advertised release version (%r)',

                     latest['release'])

      else:

        if this_release > advertised_release:

          logging.info('This SDK release is newer than the advertised release.')

          return

    api_versions = latest['api_versions']

    if self.config.api_version not in api_versions:

      self._Nag(

          'The api version you are using (%s) is obsolete!  You should\n'

          'upgrade your SDK and test that your code works with the new\n'

          'api version.' % self.config.api_version,

          latest, version, force=True)

      return

    if self.config.api_version != api_versions[len(api_versions) - 1]:

      self._Nag(

          'The api version you are using (%s) is deprecated. You should\n'

          'upgrade your SDK to try the new functionality.' %

          self.config.api_version, latest, version)

      return

    self._Nag('There is a new release of the SDK available.',

              latest, version)

  def _ParseNagFile(self):

    """Parses the nag file.

    Returns:

      A NagFile if the file was present else None.

    """

    nag_filename = UpdateCheck.MakeNagFilename()

    if self.isfile(nag_filename):

      fh = self.open(nag_filename, 'r')

      try:

        nag = NagFile.Load(fh)

      finally:

        fh.close()

      return nag

    return None

  def _WriteNagFile(self, nag):

    """Writes the NagFile to the user's nag file.

    If the destination path does not exist, this method will log an error

    and fail silently.

    Args:

      nag: The NagFile to write.

    """

    nagfilename = UpdateCheck.MakeNagFilename()

    try:

      fh = self.open(nagfilename, 'w')

      try:

        fh.write(nag.ToYAML())

      finally:

        fh.close()

    except (OSError, IOError), e:

      logging.error('Could not write nag file to %s. Error: %s', nagfilename, e)

  def _Nag(self, msg, latest, version, force=False):

    """Prints a nag message and updates the nag file's timestamp.

    Because we don't want to nag the user everytime, we store a simple

    yaml document in the user's home directory.  If the timestamp in this

    doc is over a week old, we'll nag the user.  And when we nag the user,

    we update the timestamp in this doc.

    Args:

      msg: The formatted message to print to the user.

      latest: The yaml document received from the server.

      version: The local yaml version document.

      force: If True, always nag the user, ignoring the nag file.

    """

    nag = self._ParseNagFile()

    if nag and not force:

      last_nag = datetime.datetime.fromtimestamp(nag.timestamp)

      if datetime.datetime.now() - last_nag < datetime.timedelta(weeks=1):

        logging.debug('Skipping nag message')

        return

    if nag is None:

      nag = NagFile()

    nag.timestamp = time.time()

    self._WriteNagFile(nag)

    print '****************************************************************'

    print msg

    print '-----------'

    print 'Latest SDK:'

    print yaml.dump(latest)

    print '-----------'

    print 'Your SDK:'

    print yaml.dump(version)

    print '-----------'

    print 'Please visit http://code.google.com/appengine for the latest SDK'

    print '****************************************************************'

  def AllowedToCheckForUpdates(self, input_fn=raw_input):

    """Determines if the user wants to check for updates.

    On startup, the dev_appserver wants to check for updates to the SDK.

    Because this action reports usage to Google when the user is not

    otherwise communicating with Google (e.g. pushing a new app version),

    the user must opt in.

    If the user does not have a nag file, we will query the user and

    save the response in the nag file.  Subsequent calls to this function

    will re-use that response.

    Args:

      input_fn: used to collect user input. This is for testing only.

    Returns:

      True if the user wants to check for updates.  False otherwise.

    """

    nag = self._ParseNagFile()

    if nag is None:

      nag = NagFile()

      nag.timestamp = time.time()

    if nag.opt_in is None:

      answer = input_fn('Allow dev_appserver to check for updates on startup? '

                        '(Y/n): ')

      answer = answer.strip().lower()

      if answer == 'n' or answer == 'no':

        print ('dev_appserver will not check for updates on startup.  To '

               'change this setting, edit %s' % UpdateCheck.MakeNagFilename())

        nag.opt_in = False

      else:

        print ('dev_appserver will check for updates on startup.  To change '

               'this setting, edit %s' % UpdateCheck.MakeNagFilename())

        nag.opt_in = True

      self._WriteNagFile(nag)

    return nag.opt_in