app/soc/cron/job.py
author Sverre Rabbelier <srabbelier@gmail.com>
Sun, 19 Apr 2009 17:42:44 +0000
changeset 2246 c29272f640b0
parent 2244 96d8083cf974
child 2247 64968b86b07c
permissions -rw-r--r--
Tweak the 'load balancing' algorithm In order to reduce contention we randomly skipped jobs, but this caused many jobs to end up stopping early. Now instead we keep on going until we time out (also increased the chance of doing work). Patch by: Sverre Rabbelier
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     1
#!/usr/bin/python2.5
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     2
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     3
# Copyright 2009 the Melange authors.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     4
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     5
# Licensed under the Apache License, Version 2.0 (the "License");
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     6
# you may not use this file except in compliance with the License.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     7
# You may obtain a copy of the License at
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     8
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     9
#   http://www.apache.org/licenses/LICENSE-2.0
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    10
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    11
# Unless required by applicable law or agreed to in writing, software
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    12
# distributed under the License is distributed on an "AS IS" BASIS,
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    14
# See the License for the specific language governing permissions and
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    15
# limitations under the License.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    16
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    17
"""Cron jobs.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    18
"""
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    19
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    20
__authors__ = [
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    21
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
2219
6f835882160e Fixed typo and added myself to authors in the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2217
diff changeset
    22
    '"Lennard de Rijk" <ljvderijk@gmail.com>',
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    23
  ]
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    24
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    25
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    26
import logging
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    27
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    28
from google.appengine.ext import db
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    29
from google.appengine.runtime import DeadlineExceededError
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    30
2224
fc719e902a70 Hook up the Student Proposal Mailer in the cron/jobs module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2219
diff changeset
    31
from soc.cron import student_proposal_mailer
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    32
from soc.models.job import Job
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    33
2217
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    34
class Error(Exception):
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    35
  """Base class for all exceptions raised by this module.
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    36
  """
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    37
  pass
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    38
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    39
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    40
class FatalJobError(Error):
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    41
  """Class for all errors that lead to immediate job abortion.
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    42
  """
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    43
  pass
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    44
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
    45
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    46
class Handler(object):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    47
  """A handler that dispatches a cron job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    48
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    49
  The tasks that are mapped into tasks will be called when a worker
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    50
  has claimed the job. However, there is no guarantee as to how long
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    51
  the task will be allowed to run. If an Exception is raised the task
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    52
  is automatically rescheduled for execution.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    53
  """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    54
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    55
  def __init__(self):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    56
    """Constructs a new Handler with all known jobs set.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    57
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    58
2243
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
    59
    self.OUT_OF_TIME = 0
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
    60
    self.ALREADY_CLAIMED = 1
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
    61
    self.SUCCESS = 2
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
    62
    self.ABORTED = 3
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
    63
    self.ERRORED = 4
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
    64
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    65
    self.tasks = {}
2224
fc719e902a70 Hook up the Student Proposal Mailer in the cron/jobs module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2219
diff changeset
    66
    self.tasks['setupStudentProposalMailing'] = \
fc719e902a70 Hook up the Student Proposal Mailer in the cron/jobs module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2219
diff changeset
    67
        student_proposal_mailer.setupStudentProposalMailing
fc719e902a70 Hook up the Student Proposal Mailer in the cron/jobs module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2219
diff changeset
    68
    self.tasks['sendStudentProposalMail'] = \
fc719e902a70 Hook up the Student Proposal Mailer in the cron/jobs module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2219
diff changeset
    69
        student_proposal_mailer.sendStudentProposalMail
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    70
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    71
  def claimJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    72
    """A transaction to claim a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    73
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    74
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    75
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    76
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    77
    if job.status != 'waiting':
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    78
      raise db.Rollback()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    79
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    80
    job.status = 'started'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    81
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    82
    if job.put():
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    83
      return job
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    84
    else:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    85
      return None
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    86
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    87
  def freeJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    88
    """A transaction to free a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    89
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    90
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    91
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    92
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    93
    job.status = 'waiting'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    94
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    95
    return job.put()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    96
2229
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
    97
  def timeoutJob(self, job_key):
2244
96d8083cf974 Fixed typo in cron.job module's timeOutJob docstring.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2243
diff changeset
    98
    """A transaction to timeout a job.
2229
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
    99
    """
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   100
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   101
    job = Job.get_by_id(job_key)
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   102
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   103
    job.timeouts += 1
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   104
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   105
    if job.timeouts > 50:
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   106
      job.status = 'aborted'
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   107
    else:
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   108
      job.status = 'waiting'
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   109
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   110
    job_id = job.key().id()
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   111
    logging.debug("job %d now timeout %d time(s)" % (job_id, job.timeouts))
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   112
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   113
    return job.put()
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   114
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   115
  def failJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   116
    """A transaction to fail a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   117
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   118
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   119
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   120
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   121
    job.errors += 1
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   122
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   123
    if job.errors > 5:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   124
      job.status = 'aborted'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   125
    else:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   126
      job.status = 'waiting'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   127
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   128
    job_id = job.key().id()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   129
    logging.warning("job %d now failed %d time(s)" % (job_id, job.errors))
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   130
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   131
    return job.put()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   132
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   133
  def finishJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   134
    """A transaction to finish a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   135
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   136
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   137
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   138
    job.status = 'finished'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   139
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   140
    return job.put()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   141
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   142
  def abortJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   143
    """A transaction to abort a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   144
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   145
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   146
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   147
    job.status = 'aborted'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   148
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   149
    return job.put()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   150
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   151
  def handle(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   152
    """Handle one job.
2213
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2211
diff changeset
   153
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2211
diff changeset
   154
    Returns: whether another job should be started after this one
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   155
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   156
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   157
    try:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   158
      job = db.run_in_transaction(self.claimJob, job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   159
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   160
      if not job:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   161
        # someone already claimed the job
2243
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
   162
        return self.ALREADY_CLAIMED
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   163
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   164
      if job.task_name not in self.tasks:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   165
        logging.error("Unknown job %s" % job.task_name)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   166
        db.run_in_transaction(self.abortJob, job_key)
2243
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
   167
        return self.ABORTED
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   168
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   169
      task = self.tasks[job.task_name]
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   170
2219
6f835882160e Fixed typo and added myself to authors in the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2217
diff changeset
   171
      # execute the actual job
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   172
      task(job)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   173
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   174
      db.run_in_transaction(self.finishJob, job_key)
2243
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
   175
      return self.SUCCESS
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   176
    except DeadlineExceededError, exception:
2229
b36ecf371aef Store how many times a job has timed out and abort if needed
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2224
diff changeset
   177
      db.run_in_transaction(self.timeoutJob, job_key)
2243
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
   178
      return self.OUT_OF_TIME
2217
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
   179
    except FatalJobError, exception:
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
   180
      logging.exception(exception)
2c6ec0357149 Added FatalJobError to the cron/job module.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2213
diff changeset
   181
      db.run_in_transaction(self.abortJob, job_key)
2243
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
   182
      return self.ABORTED
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   183
    except Exception, exception:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   184
      logging.exception(exception)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   185
      db.run_in_transaction(self.failJob, job_key)
2243
c61f9dd5e325 Use status codes in job.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2229
diff changeset
   186
      return self.ERRORED
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   187
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2244
diff changeset
   188
  def iterate(self, jobs, retry_jobs):
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2244
diff changeset
   189
    """Trivial iterator that iterates over jobs then retry_jobs
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2244
diff changeset
   190
    """
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2244
diff changeset
   191
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2244
diff changeset
   192
    for job in jobs:
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2244
diff changeset
   193
      yield job
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2244
diff changeset
   194
    while retry_jobs:
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2244
diff changeset
   195
      yield retry_jobs[0]
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   196
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   197
handler = Handler()