1 #!/usr/bin/env python

     2 #

     3 # Copyright 2007 Google Inc.

     4 #

     5 # Licensed under the Apache License, Version 2.0 (the "License");

     6 # you may not use this file except in compliance with the License.

     7 # You may obtain a copy of the License at

     8 #

     9 #     http://www.apache.org/licenses/LICENSE-2.0

    10 #

    11 # Unless required by applicable law or agreed to in writing, software

    12 # distributed under the License is distributed on an "AS IS" BASIS,

    13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

    14 # See the License for the specific language governing permissions and

    15 # limitations under the License.

    16 #

    17 

    18 """AppInfo tools

    19 

    20 Library for working with AppInfo records in memory, store and load from

    21 configuration files.

    22 """

    23 

    24 

    25 

    26 

    27 

    28 import re

    29 

    30 from google.appengine.api import appinfo_errors

    31 from google.appengine.api import validation

    32 from google.appengine.api import yaml_listener

    33 from google.appengine.api import yaml_builder

    34 from google.appengine.api import yaml_object

    35 

    36 

    37 _URL_REGEX = r'(?!\^)/|\.|(\(.).*(?!\$).'

    38 _FILES_REGEX = r'(?!\^).*(?!\$).'

    39 

    40 _DELTA_REGEX = r'([1-9][0-9]*)([DdHhMm]|[sS]?)'

    41 _EXPIRATION_REGEX = r'\s*(%s)(\s+%s)*\s*' % (_DELTA_REGEX, _DELTA_REGEX)

    42 

    43 APP_ID_MAX_LEN = 100

    44 MAJOR_VERSION_ID_MAX_LEN = 100

    45 MAX_URL_MAPS = 100

    46 

    47 APPLICATION_RE_STRING = r'(?!-)[a-z\d\-]{1,%d}' % APP_ID_MAX_LEN

    48 VERSION_RE_STRING = r'(?!-)[a-z\d\-]{1,%d}' % MAJOR_VERSION_ID_MAX_LEN

    49 

    50 HANDLER_STATIC_FILES = 'static_files'

    51 HANDLER_STATIC_DIR = 'static_dir'

    52 HANDLER_SCRIPT = 'script'

    53 

    54 LOGIN_OPTIONAL = 'optional'

    55 LOGIN_REQUIRED = 'required'

    56 LOGIN_ADMIN = 'admin'

    57 

    58 RUNTIME_PYTHON = 'python'

    59 

    60 DEFAULT_SKIP_FILES = (r"^(.*/)?("

    61                       r"(app\.yaml)|"

    62                       r"(app\.yml)|"

    63                       r"(index\.yaml)|"

    64                       r"(index\.yml)|"

    65                       r"(#.*#)|"

    66                       r"(.*~)|"

    67                       r"(.*\.py[co])|"

    68                       r"(.*/RCS/.*)|"

    69                       r"(\..*)|"

    70                       r")$")

    71 

    72 LOGIN = 'login'

    73 URL = 'url'

    74 STATIC_FILES = 'static_files'

    75 UPLOAD = 'upload'

    76 STATIC_DIR = 'static_dir'

    77 MIME_TYPE = 'mime_type'

    78 SCRIPT = 'script'

    79 EXPIRATION = 'expiration'

    80 

    81 APPLICATION = 'application'

    82 VERSION = 'version'

    83 RUNTIME = 'runtime'

    84 API_VERSION = 'api_version'

    85 HANDLERS = 'handlers'

    86 DEFAULT_EXPIRATION = 'default_expiration'

    87 SKIP_FILES = 'skip_files'

    88 

    89 

    90 class URLMap(validation.Validated):

    91   """Mapping from URLs to handlers.

    92 

    93   This class acts like something of a union type.  Its purpose is to

    94   describe a mapping between a set of URLs and their handlers.  What

    95   handler type a given instance has is determined by which handler-id

    96   attribute is used.

    97 

    98   Each mapping can have one and only one handler type.  Attempting to

    99   use more than one handler-id attribute will cause an UnknownHandlerType

   100   to be raised during validation.  Failure to provide any handler-id

   101   attributes will cause MissingHandlerType to be raised during validation.

   102 

   103   The regular expression used by the url field will be used to match against

   104   the entire URL path and query string of the request.  This means that

   105   partial maps will not be matched.  Specifying a url, say /admin, is the

   106   same as matching against the regular expression '^/admin$'.  Don't begin

   107   your matching url with ^ or end them with $.  These regular expressions

   108   won't be accepted and will raise ValueError.

   109 

   110   Attributes:

   111     login: Whether or not login is required to access URL.  Defaults to

   112       'optional'.

   113     url: Regular expression used to fully match against the request URLs path.

   114       See Special Cases for using static_dir.

   115     static_files: Handler id attribute that maps URL to the appropriate

   116       file.  Can use back regex references to the string matched to url.

   117     upload: Regular expression used by the application configuration

   118       program to know which files are uploaded as blobs.  It's very

   119       difficult to determine this using just the url and static_files

   120       so this attribute must be included.  Required when defining a

   121       static_files mapping.

   122       A matching file name must fully match against the upload regex, similar

   123       to how url is matched against the request path.  Do not begin upload

   124       with ^ or end it with $.

   125     static_dir: Handler id that maps the provided url to a sub-directory

   126       within the application directory.  See Special Cases.

   127     mime_type: When used with static_files and static_dir the mime-type

   128       of files served from those directories are overridden with this

   129       value.

   130     script: Handler id that maps URLs to scipt handler within the application

   131       directory that will run using CGI.

   132     expiration: When used with static files and directories, the time delta to

   133       use for cache expiration. Has the form '4d 5h 30m 15s', where each letter

   134       signifies days, hours, minutes, and seconds, respectively. The 's' for

   135       seconds may be omitted. Only one amount must be specified, combining

   136       multiple amounts is optional. Example good values: '10', '1d 6h',

   137       '1h 30m', '7d 7d 7d', '5m 30'.

   138 

   139   Special cases:

   140     When defining a static_dir handler, do not use a regular expression

   141     in the url attribute.  Both the url and static_dir attributes are

   142     automatically mapped to these equivalents:

   143 

   144       <url>/(.*)

   145       <static_dir>/\1

   146 

   147     For example:

   148 

   149       url: /images

   150       static_dir: images_folder

   151 

   152     Is the same as this static_files declaration:

   153 

   154       url: /images/(.*)

   155       static_files: images/\1

   156       upload: images/(.*)

   157   """

   158 

   159   ATTRIBUTES = {

   160 

   161     URL: validation.Optional(_URL_REGEX),

   162     LOGIN: validation.Options(LOGIN_OPTIONAL,

   163                               LOGIN_REQUIRED,

   164                               LOGIN_ADMIN,

   165                               default=LOGIN_OPTIONAL),

   166 

   167 

   168 

   169     HANDLER_STATIC_FILES: validation.Optional(_FILES_REGEX),

   170     UPLOAD: validation.Optional(_FILES_REGEX),

   171 

   172 

   173     HANDLER_STATIC_DIR: validation.Optional(_FILES_REGEX),

   174 

   175 

   176     MIME_TYPE: validation.Optional(str),

   177     EXPIRATION: validation.Optional(_EXPIRATION_REGEX),

   178 

   179 

   180     HANDLER_SCRIPT: validation.Optional(_FILES_REGEX),

   181   }

   182 

   183   COMMON_FIELDS = set([URL, LOGIN])

   184 

   185   ALLOWED_FIELDS = {

   186     HANDLER_STATIC_FILES: (MIME_TYPE, UPLOAD, EXPIRATION),

   187     HANDLER_STATIC_DIR: (MIME_TYPE, EXPIRATION),

   188     HANDLER_SCRIPT: (),

   189   }

   190 

   191   def GetHandler(self):

   192     """Get handler for mapping.

   193 

   194     Returns:

   195       Value of the handler (determined by handler id attribute).

   196     """

   197     return getattr(self, self.GetHandlerType())

   198 

   199   def GetHandlerType(self):

   200     """Get handler type of mapping.

   201 

   202     Returns:

   203       Handler type determined by which handler id attribute is set.

   204 

   205     Raises:

   206       UnknownHandlerType when none of the no handler id attributes

   207       are set.

   208 

   209       UnexpectedHandlerAttribute when an unexpected attribute

   210       is set for the discovered handler type.

   211 

   212       HandlerTypeMissingAttribute when the handler is missing a

   213       required attribute for its handler type.

   214     """

   215     for id_field in URLMap.ALLOWED_FIELDS.iterkeys():

   216       if getattr(self, id_field) is not None:

   217         mapping_type = id_field

   218         break

   219     else:

   220       raise appinfo_errors.UnknownHandlerType(

   221           'Unknown url handler type.\n%s' % str(self))

   222 

   223     allowed_fields = URLMap.ALLOWED_FIELDS[mapping_type]

   224 

   225     for attribute in self.ATTRIBUTES.iterkeys():

   226       if (getattr(self, attribute) is not None and

   227           not (attribute in allowed_fields or

   228                attribute in URLMap.COMMON_FIELDS or

   229                attribute == mapping_type)):

   230             raise appinfo_errors.UnexpectedHandlerAttribute(

   231                 'Unexpected attribute "%s" for mapping type %s.' %

   232                 (attribute, mapping_type))

   233 

   234     if mapping_type == HANDLER_STATIC_FILES and not self.upload:

   235       raise appinfo_errors.MissingHandlerAttribute(

   236           'Missing "%s" attribute for URL "%s".' % (UPLOAD, self.url))

   237 

   238     return mapping_type

   239 

   240   def CheckInitialized(self):

   241     """Adds additional checking to make sure handler has correct fields.

   242 

   243     In addition to normal ValidatedCheck calls GetHandlerType

   244     which validates all the handler fields are configured

   245     properly.

   246 

   247     Raises:

   248       UnknownHandlerType when none of the no handler id attributes

   249       are set.

   250 

   251       UnexpectedHandlerAttribute when an unexpected attribute

   252       is set for the discovered handler type.

   253 

   254       HandlerTypeMissingAttribute when the handler is missing a

   255       required attribute for its handler type.

   256     """

   257     super(URLMap, self).CheckInitialized()

   258     self.GetHandlerType()

   259 

   260 

   261 class AppInfoExternal(validation.Validated):

   262   """Class representing users application info.

   263 

   264   This class is passed to a yaml_object builder to provide the validation

   265   for the application information file format parser.

   266 

   267   Attributes:

   268     application: Unique identifier for application.

   269     version: Application's major version number.

   270     runtime: Runtime used by application.

   271     api_version: Which version of APIs to use.

   272     handlers: List of URL handlers.

   273     default_expiration: Default time delta to use for cache expiration for

   274       all static files, unless they have their own specific 'expiration' set.

   275       See the URLMap.expiration field's documentation for more information.

   276     skip_files: An re object.  Files that match this regular expression will

   277       not be uploaded by appcfg.py.  For example:

   278         skip_files: |

   279           .svn.*|

   280           #.*#

   281   """

   282 

   283   ATTRIBUTES = {

   284 

   285 

   286     APPLICATION: APPLICATION_RE_STRING,

   287     VERSION: VERSION_RE_STRING,

   288     RUNTIME: validation.Options(RUNTIME_PYTHON),

   289 

   290 

   291     API_VERSION: validation.Options('1', 'beta'),

   292     HANDLERS: validation.Optional(validation.Repeated(URLMap)),

   293     DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX),

   294     SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES)

   295   }

   296 

   297   def CheckInitialized(self):

   298     """Ensures that at least one url mapping is provided.

   299 

   300     Raises:

   301       MissingURLMapping when no URLMap objects are present in object.

   302       TooManyURLMappings when there are too many URLMap entries.

   303     """

   304     super(AppInfoExternal, self).CheckInitialized()

   305     if not self.handlers:

   306       raise appinfo_errors.MissingURLMapping(

   307           'No URLMap entries found in application configuration')

   308     if len(self.handlers) > MAX_URL_MAPS:

   309       raise appinfo_errors.TooManyURLMappings(

   310           'Found more than %d URLMap entries in application configuration' %

   311           MAX_URL_MAPS)

   312 

   313 

   314 def LoadSingleAppInfo(app_info):

   315   """Load a single AppInfo object where one and only one is expected.

   316 

   317   Args:

   318     app_info: A file-like object or string.  If it is a string, parse it as

   319     a configuration file.  If it is a file-like object, read in data and

   320     parse.

   321 

   322   Returns:

   323     An instance of AppInfoExternal as loaded from a YAML file.

   324 

   325   Raises:

   326     EmptyConfigurationFile when there are no documents in YAML file.

   327     MultipleConfigurationFile when there is more than one document in YAML

   328     file.

   329   """

   330   builder = yaml_object.ObjectBuilder(AppInfoExternal)

   331   handler = yaml_builder.BuilderHandler(builder)

   332   listener = yaml_listener.EventListener(handler)

   333   listener.Parse(app_info)

   334 

   335   app_infos = handler.GetResults()

   336   if len(app_infos) < 1:

   337     raise appinfo_errors.EmptyConfigurationFile()

   338   if len(app_infos) > 1:

   339     raise appinfo_errors.MultipleConfigurationFile()

   340   return app_infos[0]

   341 

   342 

   343 _file_path_positive_re = re.compile(r'^[ 0-9a-zA-Z\._\+/\$-]{1,256}$')

   344 

   345 _file_path_negative_1_re = re.compile(r'\.\.|^\./|\.$|/\./|^-')

   346 

   347 _file_path_negative_2_re = re.compile(r'//|/$')

   348 

   349 _file_path_negative_3_re = re.compile(r'^ | $|/ | /')

   350 

   351 

   352 def ValidFilename(filename):

   353   """Determines if filename is valid.

   354 

   355   filename must be a valid pathname.

   356   - It must contain only letters, numbers, _, +, /, $, ., and -.

   357   - It must be less than 256 chars.

   358   - It must not contain "/./", "/../", or "//".

   359   - It must not end in "/".

   360   - All spaces must be in the middle of a directory or file name.

   361 

   362   Args:

   363     filename: The filename to validate.

   364 

   365   Returns:

   366     An error string if the filename is invalid.  Returns '' if the filename

   367     is valid.

   368   """

   369   if _file_path_positive_re.match(filename) is None:

   370     return 'Invalid character in filename: %s' % filename

   371   if _file_path_negative_1_re.search(filename) is not None:

   372     return ('Filename cannot contain "." or ".." or start with "-": %s' %

   373             filename)

   374   if _file_path_negative_2_re.search(filename) is not None:

   375     return 'Filename cannot have trailing / or contain //: %s' % filename

   376   if _file_path_negative_3_re.search(filename) is not None:

   377     return 'Any spaces must be in the middle of a filename: %s' % filename

   378   return ''