py_tasks_melange: comparison thirdparty/google_appengine/google/appengine/tools/dev

equal deleted inserted replaced

-:5c931bd3dc1e
+:a7766286a7be
 from google.appengine.api import user_service_stub
 from google.appengine.api import yaml_errors
 from google.appengine.api.capabilities import capability_stub
 from google.appengine.api.memcache import memcache_stub
+from google.appengine import dist
 from google.appengine.tools import dev_appserver_index
 from google.appengine.tools import dev_appserver_login
 PYTHON_LIB_VAR = '$PYTHON_LIB'
 ('.rss', 'application/rss+xml'),
 ('.text', 'text/plain'),
 ('.wbmp', 'image/vnd.wap.wbmp')):
 mimetypes.add_type(mime_type, ext)
-MAX_RUNTIME_RESPONSE_SIZE = 1 << 20
+MAX_RUNTIME_RESPONSE_SIZE = 10 << 20
 MAX_REQUEST_SIZE = 10 * 1024 * 1024
+API_VERSION = '1'
 class Error(Exception):
 """Base-class for exceptions in this module."""
 headers: Instance of mimetools.Message with headers from the request.
 infile: File-like object with input data from the request.
 outfile: File-like object where output data should be written.
 base_env_dict: Dictionary of CGI environment parameters if available.
 Defaults to None.
+Returns:
+None if request handling is complete.
+Tuple (path, headers, input_file) for an internal redirect:
+path: Path of URL to redirect to.
+headers: Headers to send to other dispatcher.
+input_file: New input to send to new dispatcher.
 """
 raise NotImplementedError
+def EndRedirect(self, dispatched_output, original_output):
+"""Process the end of an internal redirect.
+This method is called after all subsequent dispatch requests have finished.
+By default the output from the dispatched process is copied to the original.
+This will not be called on dispatchers that do not return an internal
+redirect.
+Args:
+dispatched_output: StringIO buffer containing the results from the
+dispatched
+"""
+original_output.write(dispatched_output.read())
 class URLMatcher(object):
 """Matches an arbitrary URL using a list of URL patterns from an application.
 '\r\n'
 'Current logged in user %s is not '
 'authorized to view this page.'
 % (httplib.FORBIDDEN, email))
 else:
-dispatcher.Dispatch(relative_url,
+forward = dispatcher.Dispatch(relative_url,
 matched_path,
 headers,
 infile,
 outfile,
 base_env_dict=base_env_dict)
+if forward:
+new_path, new_headers, new_input = forward
+logging.info('Internal redirection to %s' % new_path)
+new_outfile = cStringIO.StringIO()
+self.Dispatch(new_path,
+None,
+new_headers,
+new_input,
+new_outfile,
+dict(base_env_dict))
+new_outfile.seek(0)
+dispatcher.EndRedirect(new_outfile, outfile)
 return
 outfile.write('Status: %d URL did not match\r\n'
 '\r\n'
 env['HTTP_' + adjusted_name] = ', '.join(headers.getheaders(key))
 return env
-def FakeTemporaryFile(*args, **kwargs):
-"""Fake for tempfile.TemporaryFile that just uses StringIO."""
-return cStringIO.StringIO()
 def NotImplementedFake(*args, **kwargs):
 """Fake for methods/functions that are not implemented in the production
 environment.
 """
 raise NotImplementedError("This class/method is not available.")
 def FakeUname():
 """Fake version of os.uname."""
 return ('Linux', '', '', '', '')
+def FakeUnlink(path):
+"""Fake version of os.unlink."""
+if os.path.isdir(path):
+raise OSError(2, "Is a directory", path)
+else:
+raise OSError(1, "Operation not permitted", path)
+def FakeReadlink(path):
+"""Fake version of os.readlink."""
+raise OSError(22, "Invalid argument", path)
+def FakeAccess(path, mode):
+"""Fake version of os.access where only reads are supported."""
+if not os.path.exists(path) or mode != os.R_OK:
+return False
+else:
+return True
 def FakeSetLocale(category, value=None, original_setlocale=locale.setlocale):
 """Fake version of locale.setlocale that only supports the default."""
 if value not in (None, '', 'C', 'POSIX'):
 raise locale.Error, 'locale emulation only supports "C" locale'
 return original_setlocale(category, 'C')
 os.path.dirname(os.__file__), 'site-packages', path)))
 for path in [
 ])
+_original_file = file
+_root_path = None
 _application_paths = None
-_original_file = file
+_skip_files = None
+_static_file_config_matcher = None
+_availability_cache = {}
 @staticmethod
-def SetAllowedPaths(application_paths):
+def SetAllowedPaths(root_path, application_paths):
-"""Sets the root path of the application that is currently running.
+"""Configures which paths are allowed to be accessed.
 Must be called at least once before any file objects are created in the
 hardened environment.
 Args:
-root_path: Path to the root of the application.
+root_path: Absolute path to the root of the application.
+application_paths: List of additional paths that the application may
+access, this must include the App Engine runtime but
+not the Python library directories.
 """
 FakeFile._application_paths = (set(os.path.realpath(path)
 for path in application_paths) |
 set(os.path.abspath(path)
 for path in application_paths))
+FakeFile._application_paths.add(root_path)
+FakeFile._root_path = os.path.join(root_path, '')
+FakeFile._availability_cache = {}
+@staticmethod
+def SetSkippedFiles(skip_files):
+"""Sets which files in the application directory are to be ignored.
+Must be called at least once before any file objects are created in the
+hardened environment.
+Must be called whenever the configuration was updated.
+Args:
+skip_files: Object with .match() method (e.g. compiled regexp).
+"""
+FakeFile._skip_files = skip_files
+FakeFile._availability_cache = {}
+@staticmethod
+def SetStaticFileConfigMatcher(static_file_config_matcher):
+"""Sets StaticFileConfigMatcher instance for checking if a file is static.
+Must be called at least once before any file objects are created in the
+hardened environment.
+Must be called whenever the configuration was updated.
+Args:
+static_file_config_matcher: StaticFileConfigMatcher instance.
+"""
+FakeFile._static_file_config_matcher = static_file_config_matcher
+FakeFile._availability_cache = {}
 @staticmethod
 def IsFileAccessible(filename, normcase=os.path.normcase):
 """Determines if a file's path is accessible.
-SetAllowedPaths() must be called before this method or else all file
+SetAllowedPaths(), SetSkippedFiles() and SetStaticFileConfigMatcher() must
-accesses will raise an error.
+be called before this method or else all file accesses will raise an error.
 Args:
 filename: Path of the file to check (relative or absolute). May be a
 directory, in which case access for files inside that directory will
 be checked.
 """
 logical_filename = normcase(os.path.abspath(filename))
 if os.path.isdir(logical_filename):
 logical_filename = os.path.join(logical_filename, 'foo')
+result = FakeFile._availability_cache.get(logical_filename)
+if result is None:
+result = FakeFile._IsFileAccessibleNoCache(logical_filename,
+normcase=normcase)
+FakeFile._availability_cache[logical_filename] = result
+return result
+@staticmethod
+def _IsFileAccessibleNoCache(logical_filename, normcase=os.path.normcase):
+"""Determines if a file's path is accessible.
+This is an internal part of the IsFileAccessible implementation.
+Args:
+logical_filename: Absolute path of the file to check.
+normcase: Used for dependency injection.
+Returns:
+True if the file is accessible, False otherwise.
+"""
+if IsPathInSubdirectories(logical_filename, [FakeFile._root_path],
+normcase=normcase):
+relative_filename = logical_filename[len(FakeFile._root_path):]
+if FakeFile._skip_files.match(relative_filename):
+logging.warning('Blocking access to skipped file "%s"',
+logical_filename)
+return False
+if FakeFile._static_file_config_matcher.IsStaticFile(relative_filename):
+logging.warning('Blocking access to static file "%s"',
+logical_filename)
+return False
 if logical_filename in FakeFile.ALLOWED_FILES:
 return True
 if IsPathInSubdirectories(logical_filename,
 args: Positional format parameters for the logging message.
 """
 if HardenedModulesHook.ENABLE_LOGGING:
 indent = self._indent_level * '  '
 print >>sys.stderr, indent + (message % args)
-EMPTY_MODULE_FILE = '<empty module>'
 _WHITE_LIST_C_MODULES = [
 'array',
 'binascii',
 'bz2',
 ],
 'os': [
+'access',
 'altsep',
 'curdir',
 'defpath',
 'devnull',
 'environ',
 'O_WRONLY',
 'pardir',
 'path',
 'pathsep',
 'R_OK',
+'readlink',
+'remove',
 'SEEK_CUR',
 'SEEK_END',
 'SEEK_SET',
 'sep',
 'stat',
 'stat_float_times',
 'stat_result',
 'strerror',
 'TMP_MAX',
+'unlink',
 'urandom',
 'walk',
 'WCOREDUMP',
 'WEXITSTATUS',
 'WIFEXITED',
 'W_OK',
 'X_OK',
 ],
 }
-_EMPTY_MODULES = [
-'imp',
-'ftplib',
-'select',
-'socket',
-'tempfile',
-]
 _MODULE_OVERRIDES = {
 'locale': {
 'setlocale': FakeSetLocale,
 },
 'os': {
+'access': FakeAccess,
 'listdir': RestrictedPathFunction(os.listdir),
 'lstat': RestrictedPathFunction(os.stat),
+'readlink': FakeReadlink,
+'remove': FakeUnlink,
 'stat': RestrictedPathFunction(os.stat),
 'uname': FakeUname,
+'unlink': FakeUnlink,
 'urandom': FakeURandom,
-},
-'socket': {
-'AF_INET': None,
-'SOCK_STREAM': None,
-'SOCK_DGRAM': None,
-'_GLOBAL_DEFAULT_TIMEOUT': getattr(socket, '_GLOBAL_DEFAULT_TIMEOUT',
-None),
-},
-'tempfile': {
-'TemporaryFile': FakeTemporaryFile,
-'gettempdir': NotImplementedFake,
-'gettempprefix': NotImplementedFake,
-'mkdtemp': NotImplementedFake,
-'mkstemp': NotImplementedFake,
-'mktemp': NotImplementedFake,
-'NamedTemporaryFile': NotImplementedFake,
-'tempdir': NotImplementedFake,
 },
 }
 _ENABLED_FILE_TYPES = (
 imp.PKG_DIRECTORY,
 self._indent_level = 0
 @Trace
 def find_module(self, fullname, path=None):
 """See PEP 302."""
-if (fullname in ('cPickle', 'thread') or
+if fullname in ('cPickle', 'thread'):
-fullname in HardenedModulesHook._EMPTY_MODULES):
 return self
 search_path = path
 all_modules = fullname.split('.')
 try:
 for index, current_module in enumerate(all_modules):
 current_module_fullname = '.'.join(all_modules[:index + 1])
-if current_module_fullname == fullname:
+if (current_module_fullname == fullname and not
+self.StubModuleExists(fullname)):
 self.FindModuleRestricted(current_module,
 current_module_fullname,
 search_path)
 else:
 if current_module_fullname in self._module_dict:
 search_path = module.__path__
 except CouldNotFindModuleError:
 return None
 return self
+def StubModuleExists(self, name):
+"""Check if the named module has a stub replacement."""
+if name in sys.builtin_module_names:
+name = 'py_%s' % name
+if name in dist.__all__:
+return True
+return False
+def ImportStubModule(self, name):
+"""Import the stub module replacement for the specified module."""
+if name in sys.builtin_module_names:
+name = 'py_%s' % name
+module = __import__(dist.__name__, {}, {}, [name])
+return getattr(module, name)
 @Trace
 def FixModule(self, module):
 """Prunes and overrides restricted module attributes.
 ImportError exception if the module could not be loaded for whatever
 reason (e.g., missing, not allowed).
 """
 module = self._imp.new_module(submodule_fullname)
-if submodule_fullname in self._EMPTY_MODULES:
+if submodule_fullname == 'thread':
-module.__file__ = self.EMPTY_MODULE_FILE
-elif submodule_fullname == 'thread':
 module.__dict__.update(self._dummy_thread.__dict__)
 module.__name__ = 'thread'
 elif submodule_fullname == 'cPickle':
 module.__dict__.update(self._pickle.__dict__)
 module.__name__ = 'cPickle'
 elif submodule_fullname == 'os':
 module.__dict__.update(self._os.__dict__)
 self._module_dict['os.path'] = module.path
+elif self.StubModuleExists(submodule_fullname):
+module = self.ImportStubModule(submodule_fullname)
 else:
 source_file, pathname, description = self.FindModuleRestricted(submodule, submodule_fullname, search_path)
 module = self.LoadModuleRestricted(submodule_fullname,
 source_file,
 pathname,
 regex = entry.upload + '$'
 else:
 path = entry.static_dir
 if path[-1] == '/':
 path = path[:-1]
-regex = re.escape(path) + r'/(.*)'
+regex = re.escape(path + os.path.sep) + r'(.*)'
 try:
 path_re = re.compile(regex)
 except re.error, e:
 raise InvalidAppConfigError('regex %s does not compile: %s' %
 expiration = self._default_expiration
 else:
 expiration = appinfo.ParseExpiration(entry.expiration)
 self._patterns.append((path_re, entry.mime_type, expiration))
+def IsStaticFile(self, path):
+"""Tests if the given path points to a "static" file.
+Args:
+path: String containing the file's path relative to the app.
+Returns:
+Boolean, True if the file was configured to be static.
+"""
+for (path_re, _, _) in self._patterns:
+if path_re.match(path):
+return True
+return False
 def GetMimeType(self, path):
 """Returns the mime type that we should use when serving the specified file.
 Args:
 'content-encoding', 'accept-encoding', 'transfer-encoding',
 'server', 'date',
 ])
-def RewriteResponse(response_file):
+def IgnoreHeadersRewriter(status_code, status_message, headers, body):
-"""Interprets server-side headers and adjusts the HTTP response accordingly.
+"""Ignore specific response headers.
+Certain response headers cannot be modified by an Application.  For a
+complete list of these headers please see:
+http://code.google.com/appengine/docs/webapp/responseclass.html#Disallowed_HTTP_Response_Headers
+This rewriter simply removes those headers.
+"""
+for h in _IGNORE_RESPONSE_HEADERS:
+if h in headers:
+del headers[h]
+return status_code, status_message, headers, body
+def ParseStatusRewriter(status_code, status_message, headers, body):
+"""Parse status header, if it exists.
 Handles the server-side 'status' header, which instructs the server to change
 the HTTP response code accordingly. Handles the 'location' header, which
 issues an HTTP 302 redirect to the client. Also corrects the 'content-length'
 header to reflect actual content length in case extra information has been
 appended to the response body.
 If the 'status' header supplied by the client is invalid, this method will
 set the response to a 500 with an error message as content.
+"""
+location_value = headers.getheader('location')
+status_value = headers.getheader('status')
+if status_value:
+response_status = status_value
+del headers['status']
+elif location_value:
+response_status = '%d Redirecting' % httplib.FOUND
+else:
+return status_code, status_message, headers, body
+status_parts = response_status.split(' ', 1)
+status_code, status_message = (status_parts + [''])[:2]
+try:
+status_code = int(status_code)
+except ValueError:
+status_code = 500
+body = cStringIO.StringIO('Error: Invalid "status" header value returned.')
+return status_code, status_message, headers, body
+def CacheRewriter(status_code, status_message, headers, body):
+"""Update the cache header."""
+if not 'Cache-Control' in headers:
+headers['Cache-Control'] = 'no-cache'
+return status_code, status_message, headers, body
+def ContentLengthRewriter(status_code, status_message, headers, body):
+"""Rewrite the Content-Length header.
+Even though Content-Length is not a user modifiable header, App Engine
+sends a correct Content-Length to the user based on the actual response.
+"""
+current_position = body.tell()
+body.seek(0, 2)
+headers['Content-Length'] = str(body.tell() - current_position)
+body.seek(current_position)
+return status_code, status_message, headers, body
+def CreateResponseRewritersChain():
+"""Create the default response rewriter chain.
+A response rewriter is the a function that gets a final chance to change part
+of the dev_appservers response.  A rewriter is not like a dispatcher in that
+it is called after every request has been handled by the dispatchers
+regardless of which dispatcher was used.
+The order in which rewriters are registered will be the order in which they
+are used to rewrite the response.  Modifications from earlier rewriters
+are used as input to later rewriters.
+A response rewriter is a function that can rewrite the request in any way.
+Thefunction can returned modified values or the original values it was
+passed.
+A rewriter function has the following parameters and return values:
+Args:
+status_code: Status code of response from dev_appserver or previous
+rewriter.
+status_message: Text corresponding to status code.
+headers: mimetools.Message instance with parsed headers.  NOTE: These
+headers can contain its own 'status' field, but the default
+dev_appserver implementation will remove this.  Future rewriters
+should avoid re-introducing the status field and return new codes
+instead.
+body: File object containing the body of the response.  This position of
+this file may not be at the start of the file.  Any content before the
+files position is considered not to be part of the final body.
+Returns:
+status_code: Rewritten status code or original.
+status_message: Rewritter message or original.
+headers: Rewritten/modified headers or original.
+body: Rewritten/modified body or original.
+Returns:
+List of response rewriters.
+"""
+return [IgnoreHeadersRewriter,
+ParseStatusRewriter,
+CacheRewriter,
+ContentLengthRewriter,
+]
+def RewriteResponse(response_file, response_rewriters=None):
+"""Allows final rewrite of dev_appserver response.
+This function receives the unparsed HTTP response from the application
+or internal handler, parses out the basic structure and feeds that structure
+in to a chain of response rewriters.
+It also makes sure the final HTTP headers are properly terminated.
+For more about response rewriters, please see documentation for
+CreateResponeRewritersChain.
 Args:
 response_file: File-like object containing the full HTTP response including
 the response code, all headers, and the request body.
-gmtime: Function which returns current time in a format matching standard
+response_rewriters: A list of response rewriters.  If none is provided it
-time.gmtime().
+will create a new chain using CreateResponseRewritersChain.
 Returns:
 Tuple (status_code, status_message, header, body) where:
 status_code: Integer HTTP response status (e.g., 200, 302, 404, 500)
 status_message: String containing an informational message about the
 response code, possibly derived from the 'status' header, if supplied.
 header: String containing the HTTP headers of the response, without
 a trailing new-line (CRLF).
 body: String containing the body of the response.
 """
+if response_rewriters is None:
+response_rewriters = CreateResponseRewritersChain()
+status_code = 200
+status_message = 'Good to go'
 headers = mimetools.Message(response_file)
-for h in _IGNORE_RESPONSE_HEADERS:
+for response_rewriter in response_rewriters:
-if h in headers:
+status_code, status_message, headers, response_file = response_rewriter(
-del headers[h]
+status_code,
+status_message,
-response_status = '%d Good to go' % httplib.OK
+headers,
+response_file)
-location_value = headers.getheader('location')
-status_value = headers.getheader('status')
-if status_value:
-response_status = status_value
-del headers['status']
-elif location_value:
-response_status = '%d Redirecting' % httplib.FOUND
-if not 'Cache-Control' in headers:
-headers['Cache-Control'] = 'no-cache'
-status_parts = response_status.split(' ', 1)
-status_code, status_message = (status_parts + [''])[:2]
-try:
-status_code = int(status_code)
-except ValueError:
-status_code = 500
-body = 'Error: Invalid "status" header value returned.'
-else:
-body = response_file.read()
-headers['Content-Length'] = str(len(body))
 header_list = []
 for header in headers.headers:
 header = header.rstrip('\n')
 header = header.rstrip('\r')
 header_list.append(header)
 header_data = '\r\n'.join(header_list) + '\r\n'
-return status_code, status_message, header_data, body
+return status_code, status_message, header_data, response_file.read()
 class ModuleManager(object):
 """Manages loaded modules in the runtime.
 Path of the module's corresponding Python source file if it exists, or
 just the module's compiled Python file. If the module has an invalid
 __file__ attribute, None will be returned.
 """
 module_file = getattr(module, '__file__', None)
-if not module_file or module_file == HardenedModulesHook.EMPTY_MODULE_FILE:
+if module_file is None:
 return None
 source_file = module_file[:module_file.rfind('py') + 2]
 if is_file(source_file):
 template_module = module_dict.get('google.appengine.ext.webapp.template')
 if template_module is not None:
 template_module.template_cache.clear()
-def CreateRequestHandler(root_path, login_url, require_indexes=False,
+def CreateRequestHandler(root_path,
+login_url,
+require_indexes=False,
 static_caching=True):
 """Creates a new BaseHTTPRequestHandler sub-class for use with the Python
 BaseHTTPServer module's HTTP server.
 Python's built-in HTTP server does not support passing context information
 module_dict = application_module_dict
 module_manager = ModuleManager(application_module_dict)
 config_cache = application_config_cache
+rewriter_chain = CreateResponseRewritersChain()
 def __init__(self, *args, **kwargs):
 """Initializer.
 Args:
 root_path,
 login_url)
 config, explicit_matcher = LoadAppConfig(root_path, self.module_dict,
 cache=self.config_cache,
 static_caching=static_caching)
+if config.api_version != API_VERSION:
+logging.error("API versions cannot be switched dynamically: %r != %r"
+% (config.api_version, API_VERSION))
+sys.exit(1)
 env_dict['CURRENT_VERSION_ID'] = config.version + ".1"
 env_dict['APPLICATION_ID'] = config.application
 dispatcher = MatcherDispatcher(login_url,
 [implicit_matcher, explicit_matcher])
 self.module_manager.UpdateModuleFileModificationTimes()
 outfile.flush()
 outfile.seek(0)
-status_code, status_message, header_data, body = RewriteResponse(outfile)
+status_code, status_message, header_data, body = RewriteResponse(outfile, self.rewriter_chain)
 runtime_response_size = len(outfile.getvalue())
 if runtime_response_size > MAX_RUNTIME_RESPONSE_SIZE:
 status_code = 403
 status_message = 'Forbidden'
 Instance of URLMatcher with the supplied URLMap objects properly loaded.
 """
 url_matcher = create_url_matcher()
 path_adjuster = create_path_adjuster(root_path)
 cgi_dispatcher = create_cgi_dispatcher(module_dict, root_path, path_adjuster)
+static_file_config_matcher = StaticFileConfigMatcher(url_map_list,
+path_adjuster,
+default_expiration)
 file_dispatcher = create_file_dispatcher(path_adjuster,
-StaticFileConfigMatcher(url_map_list, path_adjuster, default_expiration))
+static_file_config_matcher)
+FakeFile.SetStaticFileConfigMatcher(static_file_config_matcher)
 for url_map in url_map_list:
 admin_only = url_map.login == appinfo.LOGIN_ADMIN
 requires_login = url_map.login == appinfo.LOGIN_REQUIRED or admin_only
 matcher = create_matcher(root_path,
 config.handlers,
 module_dict,
 default_expiration)
+FakeFile.SetSkippedFiles(config.skip_files)
 if cache is not None:
 cache.path = appinfo_path
 cache.config = config
 cache.matcher = matcher
 port,
 template_dir,
 serve_address='',
 require_indexes=False,
 static_caching=True,
-python_path_list=sys.path):
+python_path_list=sys.path,
+sdk_dir=os.path.dirname(os.path.dirname(google.__file__))):
 """Creates an new HTTPServer for an application.
+The sdk_dir argument must be specified for the directory storing all code for
+the SDK so as to allow for the sandboxing of module access to work for any
+and all SDK code. While typically this is where the 'google' package lives,
+it can be in another location because of API version support.
 Args:
 root_path: String containing the path to the root directory of the
 application where the app.yaml file is.
 login_url: Relative URL which should be used for handling user login/logout.
 are stored.
 serve_address: Address on which the server should serve.
 require_indexes: True if index.yaml is read-only gospel; default False.
 static_caching: True if browser caching of static files should be allowed.
 python_path_list: Used for dependency injection.
+sdk_dir: Directory where the SDK is stored.
 Returns:
 Instance of BaseHTTPServer.HTTPServer that's ready to start accepting.
 """
 absolute_root_path = os.path.realpath(root_path)
 SetupTemplates(template_dir)
-FakeFile.SetAllowedPaths([absolute_root_path,
+FakeFile.SetAllowedPaths(absolute_root_path,
-os.path.dirname(os.path.dirname(google.__file__)),
+[sdk_dir,
 template_dir])
-handler_class = CreateRequestHandler(absolute_root_path, login_url,
+handler_class = CreateRequestHandler(absolute_root_path,
-require_indexes, static_caching)
+login_url,
+require_indexes,
+static_caching)
 if absolute_root_path not in python_path_list:
 python_path_list.insert(0, absolute_root_path)
 return BaseHTTPServer.HTTPServer((serve_address, port), handler_class)

changeset 1278	a7766286a7be
parent 828	f5fd65cc3bf3
child 2172	ac7bd3b467ff