#!/usr/bin/env python

#

# Copyright 2007 Google Inc.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

"""Pure-Python application server for testing applications locally.

Given a port and the paths to a valid application directory (with an 'app.yaml'

file), the external library directory, and a relative URL to use for logins,

creates an HTTP server that can be used to test an application locally. Uses

stubs instead of actual APIs when SetupStubs() is called first.

Example:

  root_path = '/path/to/application/directory'

  login_url = '/login'

  port = 8080

  template_dir = '/path/to/appserver/templates'

  server = dev_appserver.CreateServer(root_path, login_url, port, template_dir)

  server.serve_forever()

"""

from google.appengine.tools import os_compat

import __builtin__

import BaseHTTPServer

import Cookie

import base64

import cStringIO

import cgi

import cgitb

try:

  import distutils.util

except ImportError:

  pass

import dummy_thread

import email.Utils

import errno

import heapq

import httplib

import imp

import inspect

import itertools

import locale

import logging

import mimetools

import mimetypes

import os

import pickle

import pprint

import random

import select

import re

import sre_compile

import sre_constants

import sre_parse

import socket

import sys

import time

import traceback

import types

import urlparse

import urllib

import google

from google.pyglib import gexcept

from google.appengine.api import apiproxy_stub_map

from google.appengine.api import appinfo

from google.appengine.api import croninfo

from google.appengine.api import datastore_admin

from google.appengine.api import datastore_file_stub

from google.appengine.api import mail_stub

from google.appengine.api import urlfetch_stub

from google.appengine.api import user_service_stub

from google.appengine.api import yaml_errors

from google.appengine.api.capabilities import capability_stub

from google.appengine.api.labs.taskqueue import taskqueue_stub

from google.appengine.api.memcache import memcache_stub

from google.appengine.api.xmpp import xmpp_service_stub

from google.appengine import dist

from google.appengine.tools import dev_appserver_index

from google.appengine.tools import dev_appserver_login

PYTHON_LIB_VAR = '$PYTHON_LIB'

DEVEL_CONSOLE_PATH = PYTHON_LIB_VAR + '/google/appengine/ext/admin'

FILE_MISSING_EXCEPTIONS = frozenset([errno.ENOENT, errno.ENOTDIR])

MAX_URL_LENGTH = 2047

HEADER_TEMPLATE = 'logging_console_header.html'

SCRIPT_TEMPLATE = 'logging_console.js'

MIDDLE_TEMPLATE = 'logging_console_middle.html'

FOOTER_TEMPLATE = 'logging_console_footer.html'

DEFAULT_ENV = {

    'GATEWAY_INTERFACE': 'CGI/1.1',

    'AUTH_DOMAIN': 'gmail.com',

    'TZ': 'UTC',

}

DEFAULT_SELECT_DELAY = 30.0

for ext, mime_type in (('.asc', 'text/plain'),

                       ('.diff', 'text/plain'),

                       ('.csv', 'text/comma-separated-values'),

                       ('.rss', 'application/rss+xml'),

                       ('.text', 'text/plain'),

                       ('.wbmp', 'image/vnd.wap.wbmp')):

  mimetypes.add_type(mime_type, ext)

MAX_RUNTIME_RESPONSE_SIZE = 10 << 20

MAX_REQUEST_SIZE = 10 * 1024 * 1024

API_VERSION = '1'

SITE_PACKAGES = os.path.normcase(os.path.join(os.path.dirname(os.__file__),

                                              'site-packages'))

class Error(Exception):

  """Base-class for exceptions in this module."""

class InvalidAppConfigError(Error):

  """The supplied application configuration file is invalid."""

class AppConfigNotFoundError(Error):

  """Application configuration file not found."""

class TemplatesNotLoadedError(Error):

  """Templates for the debugging console were not loaded."""

def SplitURL(relative_url):

  """Splits a relative URL into its path and query-string components.

  Args:

    relative_url: String containing the relative URL (often starting with '/')

      to split. Should be properly escaped as www-form-urlencoded data.

  Returns:

    Tuple (script_name, query_string) where:

      script_name: Relative URL of the script that was accessed.

      query_string: String containing everything after the '?' character.

  """

  (unused_scheme, unused_netloc, path, query,

   unused_fragment) = urlparse.urlsplit(relative_url)

  return path, query

def GetFullURL(server_name, server_port, relative_url):

  """Returns the full, original URL used to access the relative URL.

  Args:

    server_name: Name of the local host, or the value of the 'host' header

      from the request.

    server_port: Port on which the request was served (string or int).

    relative_url: Relative URL that was accessed, including query string.

  Returns:

    String containing the original URL.

  """

  if str(server_port) != '80':

    netloc = '%s:%s' % (server_name, server_port)

  else:

    netloc = server_name

  return 'http://%s%s' % (netloc, relative_url)

class URLDispatcher(object):

  """Base-class for handling HTTP requests."""

  def Dispatch(self,

               relative_url,

               path,

               headers,

               infile,

               outfile,

               base_env_dict=None):

    """Dispatch and handle an HTTP request.

    base_env_dict should contain at least these CGI variables:

      REQUEST_METHOD, REMOTE_ADDR, SERVER_SOFTWARE, SERVER_NAME,

      SERVER_PROTOCOL, SERVER_PORT

    Args:

      relative_url: String containing the URL accessed.

      path: Local path of the resource that was matched; back-references will be

        replaced by values matched in the relative_url. Path may be relative

        or absolute, depending on the resource being served (e.g., static files

        will have an absolute path; scripts will be relative).

      headers: Instance of mimetools.Message with headers from the request.

      infile: File-like object with input data from the request.

      outfile: File-like object where output data should be written.

      base_env_dict: Dictionary of CGI environment parameters if available.

        Defaults to None.

    Returns:

      None if request handling is complete.

      Tuple (path, headers, input_file) for an internal redirect:

        path: Path of URL to redirect to.

        headers: Headers to send to other dispatcher.

        input_file: New input to send to new dispatcher.

    """

    raise NotImplementedError

  def EndRedirect(self, dispatched_output, original_output):

    """Process the end of an internal redirect.

    This method is called after all subsequent dispatch requests have finished.

    By default the output from the dispatched process is copied to the original.

    This will not be called on dispatchers that do not return an internal

    redirect.

    Args:

      dispatched_output: StringIO buffer containing the results from the

       dispatched

      original_output: The original output file.

    """

    original_output.write(dispatched_output.read())

class URLMatcher(object):

  """Matches an arbitrary URL using a list of URL patterns from an application.

  Each URL pattern has an associated URLDispatcher instance and path to the

  resource's location on disk. See AddURL for more details. The first pattern

  that matches an inputted URL will have its associated values returned by

  Match().

  """

  def __init__(self):

    """Initializer."""

    self._url_patterns = []

  def AddURL(self, regex, dispatcher, path, requires_login, admin_only):

    """Adds a URL pattern to the list of patterns.

    If the supplied regex starts with a '^' or ends with a '$' an

    InvalidAppConfigError exception will be raised. Start and end symbols

    and implicitly added to all regexes, meaning we assume that all regexes

    consume all input from a URL.

    Args:

      regex: String containing the regular expression pattern.

      dispatcher: Instance of URLDispatcher that should handle requests that

        match this regex.

      path: Path on disk for the resource. May contain back-references like

        r'\1', r'\2', etc, which will be replaced by the corresponding groups

        matched by the regex if present.

      requires_login: True if the user must be logged-in before accessing this

        URL; False if anyone can access this URL.

      admin_only: True if the user must be a logged-in administrator to

        access the URL; False if anyone can access the URL.

    Raises:

      TypeError: if dispatcher is not a URLDispatcher sub-class instance.

      InvalidAppConfigError: if regex isn't valid.

    """

    if not isinstance(dispatcher, URLDispatcher):

      raise TypeError('dispatcher must be a URLDispatcher sub-class')

    if regex.startswith('^') or regex.endswith('$'):

      raise InvalidAppConfigError('regex starts with "^" or ends with "$"')

    adjusted_regex = '^%s$' % regex

    try:

      url_re = re.compile(adjusted_regex)

    except re.error, e:

      raise InvalidAppConfigError('regex invalid: %s' % e)

    match_tuple = (url_re, dispatcher, path, requires_login, admin_only)

    self._url_patterns.append(match_tuple)

  def Match(self,

            relative_url,

            split_url=SplitURL):

    """Matches a URL from a request against the list of URL patterns.

    The supplied relative_url may include the query string (i.e., the '?'

    character and everything following).

    Args:

      relative_url: Relative URL being accessed in a request.

      split_url: Used for dependency injection.

    Returns:

      Tuple (dispatcher, matched_path, requires_login, admin_only), which are

      the corresponding values passed to AddURL when the matching URL pattern

      was added to this matcher. The matched_path will have back-references

      replaced using values matched by the URL pattern. If no match was found,

      dispatcher will be None.

    """

    adjusted_url, unused_query_string = split_url(relative_url)

    for url_tuple in self._url_patterns:

      url_re, dispatcher, path, requires_login, admin_only = url_tuple

      the_match = url_re.match(adjusted_url)

      if the_match:

        adjusted_path = the_match.expand(path)

        return dispatcher, adjusted_path, requires_login, admin_only

    return None, None, None, None

  def GetDispatchers(self):

    """Retrieves the URLDispatcher objects that could be matched.

    Should only be used in tests.

    Returns:

      A set of URLDispatcher objects.

    """

    return set([url_tuple[1] for url_tuple in self._url_patterns])

class MatcherDispatcher(URLDispatcher):

  """Dispatcher across multiple URLMatcher instances."""

  def __init__(self,

               login_url,

               url_matchers,

               get_user_info=dev_appserver_login.GetUserInfo,

               login_redirect=dev_appserver_login.LoginRedirect):

    """Initializer.

    Args:

      login_url: Relative URL which should be used for handling user logins.

      url_matchers: Sequence of URLMatcher objects.

      get_user_info: Used for dependency injection.

      login_redirect: Used for dependency injection.

    """

    self._login_url = login_url

    self._url_matchers = tuple(url_matchers)

    self._get_user_info = get_user_info

    self._login_redirect = login_redirect

  def Dispatch(self,

               relative_url,

               path,

               headers,

               infile,

               outfile,

               base_env_dict=None):

    """Dispatches a request to the first matching dispatcher.

    Matchers are checked in the order they were supplied to the constructor.

    If no matcher matches, a 404 error will be written to the outfile. The

    path variable supplied to this method is ignored.

    """

    cookies = ', '.join(headers.getheaders('cookie'))

    email_addr, admin, user_id = self._get_user_info(cookies)

    for matcher in self._url_matchers:

      dispatcher, matched_path, requires_login, admin_only = matcher.Match(

          relative_url)

      if dispatcher is None:

        continue

      logging.debug('Matched "%s" to %s with path %s',

                    relative_url, dispatcher, matched_path)

      if (requires_login or admin_only) and not email_addr:

        logging.debug('Login required, redirecting user')

        self._login_redirect(self._login_url,

                             base_env_dict['SERVER_NAME'],

                             base_env_dict['SERVER_PORT'],

                             relative_url,

                             outfile)

      elif admin_only and not admin:

        outfile.write('Status: %d Not authorized\r\n'

                      '\r\n'

                      'Current logged in user %s is not '

                      'authorized to view this page.'

                      % (httplib.FORBIDDEN, email_addr))

      else:

        forward = dispatcher.Dispatch(relative_url,

                                      matched_path,

                                      headers,

                                      infile,

                                      outfile,

                                      base_env_dict=base_env_dict)

        if forward:

          new_path, new_headers, new_input = forward

          logging.info('Internal redirection to %s', new_path)

          new_outfile = cStringIO.StringIO()

          self.Dispatch(new_path,

                        None,

                        new_headers,

                        new_input,

                        new_outfile,

                        dict(base_env_dict))

          new_outfile.seek(0)

          dispatcher.EndRedirect(new_outfile, outfile)

      return

    outfile.write('Status: %d URL did not match\r\n'

                  '\r\n'

                  'Not found error: %s did not match any patterns '

                  'in application configuration.'

                  % (httplib.NOT_FOUND, relative_url))

class ApplicationLoggingHandler(logging.Handler):

  """Python Logging handler that displays the debugging console to users."""

  _COOKIE_NAME = '_ah_severity'

  _TEMPLATES_INITIALIZED = False

  _HEADER = None

  _SCRIPT = None

  _MIDDLE = None

  _FOOTER = None

  @staticmethod

  def InitializeTemplates(header, script, middle, footer):

    """Initializes the templates used to render the debugging console.

    This method must be called before any ApplicationLoggingHandler instances

    are created.

    Args:

      header: The header template that is printed first.

      script: The script template that is printed after the logging messages.

      middle: The middle element that's printed before the footer.

      footer; The last element that's printed at the end of the document.

    """

    ApplicationLoggingHandler._HEADER = header

    ApplicationLoggingHandler._SCRIPT = script

    ApplicationLoggingHandler._MIDDLE = middle

    ApplicationLoggingHandler._FOOTER = footer

    ApplicationLoggingHandler._TEMPLATES_INITIALIZED = True

  @staticmethod

  def AreTemplatesInitialized():

    """Returns True if InitializeTemplates has been called, False otherwise."""

    return ApplicationLoggingHandler._TEMPLATES_INITIALIZED

  def __init__(self, *args, **kwargs):

    """Initializer.

    Args:

      args, kwargs: See logging.Handler.

    Raises:

      TemplatesNotLoadedError exception if the InitializeTemplates method was

      not called before creating this instance.

    """

    if not self._TEMPLATES_INITIALIZED:

      raise TemplatesNotLoadedError

    logging.Handler.__init__(self, *args, **kwargs)

    self._record_list = []

    self._start_time = time.time()

  def emit(self, record):

    """Called by the logging module each time the application logs a message.

    Args:

      record: logging.LogRecord instance corresponding to the newly logged

        message.

    """

    self._record_list.append(record)

  def AddDebuggingConsole(self, relative_url, env, outfile):

    """Prints an HTML debugging console to an output stream, if requested.

    Args:

      relative_url: Relative URL that was accessed, including the query string.

        Used to determine if the parameter 'debug' was supplied, in which case

        the console will be shown.

      env: Dictionary containing CGI environment variables. Checks for the

        HTTP_COOKIE entry to see if the accessing user has any logging-related

        cookies set.

      outfile: Output stream to which the console should be written if either

        a debug parameter was supplied or a logging cookie is present.

    """

    unused_script_name, query_string = SplitURL(relative_url)

    param_dict = cgi.parse_qs(query_string, True)

    cookie_dict = Cookie.SimpleCookie(env.get('HTTP_COOKIE', ''))

    if 'debug' not in param_dict and self._COOKIE_NAME not in cookie_dict:

      return