1 #!/usr/bin/env python

     2 #

     3 # Copyright 2007 Google Inc.

     4 #

     5 # Licensed under the Apache License, Version 2.0 (the "License");

     6 # you may not use this file except in compliance with the License.

     7 # You may obtain a copy of the License at

     8 #

     9 #     http://www.apache.org/licenses/LICENSE-2.0

    10 #

    11 # Unless required by applicable law or agreed to in writing, software

    12 # distributed under the License is distributed on an "AS IS" BASIS,

    13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

    14 # See the License for the specific language governing permissions and

    15 # limitations under the License.

    16 #

    17 

    18 """QueueInfo tools.

    19 

    20 A library for working with QueueInfo records, describing task queue entries

    21 for an application. Supports loading the records from queue.yaml.

    22 

    23 A queue has two required parameters and one optional one. The required

    24 parameters are 'name' (must be unique for an appid) and 'rate' (the

    25 rate at which jobs in the queue are run). There is an optional 'bucket_size'

    26 that will allow tokens to be 'saved up' and bucket_size. Rate and bucket_size rate are

    27 expressed as number/unit, with number being an int or a float, and unit being

    28 one of 's' (seconds), 'm' (minutes), 'h' (hours) or 'd' (days).

    29 

    30 An example of the use of bucket_size rate: the free email quota is 2000/d, and the

    31 maximum you can send in a single minute is 11. So we can define a queue for

    32 sending email like this:

    33 

    34 queue:

    35 - name: mail_queue

    36   rate: 2000/d

    37   bucket_size: 10/m

    38 

    39 If this queue had been idle for a while before some jobs were submitted to it,

    40 the first 10 jobs submitted would be run immediately, then subsequent ones

    41 would be run once every 40s or so. The limit of 2000 per day would still apply.

    42 """

    43 

    44 

    45 

    46 from google.appengine.api import validation

    47 from google.appengine.api import yaml_builder

    48 from google.appengine.api import yaml_listener

    49 from google.appengine.api import yaml_object

    50 

    51 _NAME_REGEX = r'^[A-Za-z0-9-]{0,499}$'

    52 _RATE_REGEX = r'^[0-9]+(\.[0-9]+)?/[smhd]'

    53 

    54 QUEUE = 'queue'

    55 

    56 NAME = 'name'

    57 RATE = 'rate'

    58 BUCKET_SIZE = 'bucket_size'

    59 

    60 

    61 class MalformedQueueConfiguration(Exception):

    62   """Configuration file for Task Queue is malformed."""

    63 

    64 

    65 class QueueEntry(validation.Validated):

    66   """A queue entry describes a single task queue."""

    67   ATTRIBUTES = {

    68       NAME: _NAME_REGEX,

    69       RATE: _RATE_REGEX,

    70       BUCKET_SIZE: validation.Optional(validation.TYPE_INT),

    71   }

    72 

    73 

    74 class QueueInfoExternal(validation.Validated):

    75   """QueueInfoExternal describes all queue entries for an application."""

    76   ATTRIBUTES = {

    77       QUEUE: validation.Optional(validation.Repeated(QueueEntry))

    78   }

    79 

    80 

    81 def LoadSingleQueue(queue_info):

    82   """Load a queue.yaml file or string and return a QueueInfoExternal object.

    83 

    84   Args:

    85     queue_info: the contents of a queue.yaml file, as a string.

    86 

    87   Returns:

    88     A QueueInfoExternal object.

    89   """

    90   builder = yaml_object.ObjectBuilder(QueueInfoExternal)

    91   handler = yaml_builder.BuilderHandler(builder)

    92   listener = yaml_listener.EventListener(handler)

    93   listener.Parse(queue_info)

    94 

    95   queue_info = handler.GetResults()

    96   if len(queue_info) < 1:

    97     raise MalformedQueueConfiguration('Empty queue configuration.')

    98   if len(queue_info) > 1:

    99     raise MalformedQueueConfiguration('Multiple queue: sections '

   100                                       'in configuration.')

   101   return queue_info[0]

   102 

   103 

   104 def ParseRate(rate):

   105   """Parses a rate string in the form number/unit.

   106 

   107   The unit is one of s (seconds), m (minutes), h (hours) or d (days).

   108 

   109   Args:

   110     rate: the rate string.

   111 

   112   Returns:

   113     a floating point number representing the rate/second.

   114 

   115   Raises:

   116     MalformedQueueConfiguration: if the rate is invalid

   117   """

   118   elements = rate.split('/')

   119   if len(elements) != 2:

   120     raise MalformedQueueConfiguration('Rate "%s" is invalid.' % rate)

   121   number, unit = elements

   122   try:

   123     number = float(number)

   124   except ValueError:

   125     raise MalformedQueueConfiguration('Rate "%s" is invalid:'

   126                                           ' "%s" is not a number.' %

   127                                           (rate, number))

   128   if unit not in 'smhd':

   129     raise MalformedQueueConfiguration('Rate "%s" is invalid:'

   130                                           ' "%s" is not one of s, m, h, d.' %

   131                                           (rate, unit))

   132   if unit == 's':

   133     return number

   134   if unit == 'm':

   135     return number/60

   136   if unit == 'h':

   137     return number/(60 * 60)

   138   if unit == 'd':

   139     return number/(24 * 60 * 60)