app/soc/models/job.py
author Sverre Rabbelier <srabbelier@gmail.com>
Sun, 19 Apr 2009 17:42:44 +0000
changeset 2246 c29272f640b0
parent 2229 b36ecf371aef
child 2263 823fdb4e1781
permissions -rw-r--r--
Tweak the 'load balancing' algorithm In order to reduce contention we randomly skipped jobs, but this caused many jobs to end up stopping early. Now instead we keep on going until we time out (also increased the chance of doing work). Patch by: Sverre Rabbelier
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
2209
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     1
#!/usr/bin/python2.5
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     2
#
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     3
# Copyright 2009 the Melange authors.
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     4
#
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     5
# Licensed under the Apache License, Version 2.0 (the "License");
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     6
# you may not use this file except in compliance with the License.
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     7
# You may obtain a copy of the License at
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     8
#
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     9
#   http://www.apache.org/licenses/LICENSE-2.0
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    10
#
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    11
# Unless required by applicable law or agreed to in writing, software
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    12
# distributed under the License is distributed on an "AS IS" BASIS,
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    14
# See the License for the specific language governing permissions and
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    15
# limitations under the License.
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    16
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    17
"""This module contains the Job Model."""
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    18
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    19
__authors__ = [
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    20
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    21
]
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    22
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    23
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    24
from google.appengine.ext import db
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    25
2212
4095892a3c99 Removed unneeded imports and variables
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2209
diff changeset
    26
from soc.models import base
2209
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    27
2212
4095892a3c99 Removed unneeded imports and variables
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2209
diff changeset
    28
import soc.models.priority_group
2209
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    29
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    30
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    31
class Job(base.ModelWithFieldAttributes):
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    32
  """The Job model.
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    33
  """
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    34
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    35
  #: reference to the priority group this job belongs to
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    36
  priority_group = db.ReferenceProperty(
2212
4095892a3c99 Removed unneeded imports and variables
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2209
diff changeset
    37
      reference_class=soc.models.priority_group.PriorityGroup,
4095892a3c99 Removed unneeded imports and variables
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2209
diff changeset
    38
      required=True, collection_name='jobs')
2209
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    39
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    40
  #: the name of the task as defined in soc.cron.job
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    41
  task_name = db.StringProperty(required=True)
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    42
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    43
  #: field storing the status of this job
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    44
  #: Waiting means that this job is waiting to be run.
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    45
  #: Started means that this job has a worker that is running it.
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    46
  #: Finished means that this job has been completed.
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    47
  #: Aborted means that this job has been aborted due to a fatal error.
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    48
  status = db.StringProperty(default='waiting',
2225
c6a0f935504c Style fix in Job model.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2216
diff changeset
    49
      choices=['waiting', 'started', 'finished', 'aborted'])
2209
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    50
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    51
  #: the date this job was last modified on
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    52
  last_modified_on = db.DateTimeProperty(auto_now_add=True)
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    53
2216
2ac430aa5799 Fixed typo in documentation of the Job Model.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2212
diff changeset
    54
  #: the amount of times this job raised an Exception (other than a
2229
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2225
diff changeset
    55
  #: DeadlineExceededError).
2209
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    56
  errors = db.IntegerProperty(default=0)
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    57
2229
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2225
diff changeset
    58
  #: the amount of times this job raised an DeadlineExceededError.
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2225
diff changeset
    59
  timeouts = db.IntegerProperty(default=0)
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2225
diff changeset
    60
2209
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    61
  #: the data that the worker will use to process this job
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    62
  text_data = db.TextProperty(required=False, default="")
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    63
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    64
  #: the data that the worker will use to process this job
989c7c945d5f Add jobs (model, logic, and view)
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    65
  key_data = db.ListProperty(db.Key, default=[])