app/soc/cron/job.py
author Sverre Rabbelier <srabbelier@gmail.com>
Sat, 18 Apr 2009 14:04:11 +0000
changeset 2211 f7497180d037
child 2213 c0f52da7a808
permissions -rw-r--r--
Add cron, the core of the job system Patch by: Sverre Rabbelier
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     1
#!/usr/bin/python2.5
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     2
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     3
# Copyright 2009 the Melange authors.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     4
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     5
# Licensed under the Apache License, Version 2.0 (the "License");
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     6
# you may not use this file except in compliance with the License.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     7
# You may obtain a copy of the License at
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     8
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     9
#   http://www.apache.org/licenses/LICENSE-2.0
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    10
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    11
# Unless required by applicable law or agreed to in writing, software
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    12
# distributed under the License is distributed on an "AS IS" BASIS,
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    14
# See the License for the specific language governing permissions and
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    15
# limitations under the License.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    16
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    17
"""Cron jobs.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    18
"""
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    19
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    20
__authors__ = [
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    21
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    22
  ]
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    23
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    24
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    25
import logging
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    26
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    27
from google.appengine.ext import db
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    28
from google.appengine.runtime import DeadlineExceededError
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    29
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    30
from soc.models.job import Job
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    31
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    32
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    33
class Handler(object):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    34
  """A handler that dispatches a cron job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    35
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    36
  The tasks that are mapped into tasks will be called when a worker
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    37
  has claimed the job. However, there is no guarantee as to how long
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    38
  the task will be allowed to run. If an Exception is raised the task
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    39
  is automatically rescheduled for execution.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    40
  """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    41
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    42
  def __init__(self):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    43
    """Constructs a new Handler with all known jobs set.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    44
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    45
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    46
    self.tasks = {}
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    47
    self.tasks['sendAcceptanceEmail'] = logging.info
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    48
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    49
  def claimJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    50
    """A transaction to claim a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    51
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    52
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    53
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    54
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    55
    if job.status != 'waiting':
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    56
      raise db.Rollback()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    57
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    58
    job.status = 'started'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    59
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    60
    if job.put():
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    61
      return job
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    62
    else:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    63
      return None
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    64
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    65
  def freeJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    66
    """A transaction to free a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    67
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    68
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    69
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    70
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    71
    job.status = 'waiting'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    72
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    73
    return job.put()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    74
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    75
  def failJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    76
    """A transaction to fail a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    77
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    78
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    79
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    80
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    81
    job.errors += 1
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    82
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    83
    if job.errors > 5:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    84
      job.status = 'aborted'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    85
    else:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    86
      job.status = 'waiting'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    87
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    88
    job_id = job.key().id()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    89
    logging.warning("job %d now failed %d time(s)" % (job_id, job.errors))
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    90
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    91
    return job.put()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    92
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    93
  def finishJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    94
    """A transaction to finish a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    95
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    96
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    97
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    98
    job.status = 'finished'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    99
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   100
    return job.put()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   101
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   102
  def abortJob(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   103
    """A transaction to abort a job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   104
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   105
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   106
    job = Job.get_by_id(job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   107
    job.status = 'aborted'
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   108
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   109
    return job.put()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   110
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   111
  def handle(self, job_key):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   112
    """Handle one job.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   113
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   114
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   115
    try:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   116
      job = db.run_in_transaction(self.claimJob, job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   117
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   118
      if not job:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   119
        # someone already claimed the job
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   120
        return True
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   121
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   122
      if job.task_name not in self.tasks:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   123
        logging.error("Unknown job %s" % job.task_name)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   124
        db.run_in_transaction(self.abortJob, job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   125
        return True
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   126
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   127
      task = self.tasks[job.task_name]
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   128
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   129
      # excecute the actual job
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   130
      task(job)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   131
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   132
      db.run_in_transaction(self.finishJob, job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   133
      return True
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   134
    except DeadlineExceededError, exception:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   135
      db.run_in_transaction(self.freeJob, job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   136
      return False
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   137
    except Exception, exception:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   138
      logging.exception(exception)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   139
      db.run_in_transaction(self.failJob, job_key)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   140
      return True
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   141
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   142
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   143
handler = Handler()