author | Sverre Rabbelier <srabbelier@gmail.com> |
Sat, 18 Apr 2009 14:04:11 +0000 | |
changeset 2211 | f7497180d037 |
child 2213 | c0f52da7a808 |
permissions | -rw-r--r-- |
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
1 |
#!/usr/bin/python2.5 |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
2 |
# |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
3 |
# Copyright 2009 the Melange authors. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
4 |
# |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
5 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
6 |
# you may not use this file except in compliance with the License. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
7 |
# You may obtain a copy of the License at |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
8 |
# |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
9 |
# http://www.apache.org/licenses/LICENSE-2.0 |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
10 |
# |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
11 |
# Unless required by applicable law or agreed to in writing, software |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
12 |
# distributed under the License is distributed on an "AS IS" BASIS, |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
13 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
14 |
# See the License for the specific language governing permissions and |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
15 |
# limitations under the License. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
16 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
17 |
"""Cron jobs. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
18 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
19 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
20 |
__authors__ = [ |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
21 |
'"Sverre Rabbelier" <sverre@rabbelier.nl>', |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
22 |
] |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
23 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
24 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
25 |
import logging |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
26 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
27 |
from google.appengine.ext import db |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
28 |
from google.appengine.runtime import DeadlineExceededError |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
29 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
30 |
from soc.models.job import Job |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
31 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
32 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
33 |
class Handler(object): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
34 |
"""A handler that dispatches a cron job. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
35 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
36 |
The tasks that are mapped into tasks will be called when a worker |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
37 |
has claimed the job. However, there is no guarantee as to how long |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
38 |
the task will be allowed to run. If an Exception is raised the task |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
39 |
is automatically rescheduled for execution. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
40 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
41 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
42 |
def __init__(self): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
43 |
"""Constructs a new Handler with all known jobs set. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
44 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
45 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
46 |
self.tasks = {} |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
47 |
self.tasks['sendAcceptanceEmail'] = logging.info |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
48 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
49 |
def claimJob(self, job_key): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
50 |
"""A transaction to claim a job. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
51 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
52 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
53 |
job = Job.get_by_id(job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
54 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
55 |
if job.status != 'waiting': |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
56 |
raise db.Rollback() |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
57 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
58 |
job.status = 'started' |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
59 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
60 |
if job.put(): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
61 |
return job |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
62 |
else: |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
63 |
return None |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
64 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
65 |
def freeJob(self, job_key): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
66 |
"""A transaction to free a job. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
67 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
68 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
69 |
job = Job.get_by_id(job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
70 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
71 |
job.status = 'waiting' |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
72 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
73 |
return job.put() |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
74 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
75 |
def failJob(self, job_key): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
76 |
"""A transaction to fail a job. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
77 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
78 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
79 |
job = Job.get_by_id(job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
80 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
81 |
job.errors += 1 |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
82 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
83 |
if job.errors > 5: |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
84 |
job.status = 'aborted' |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
85 |
else: |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
86 |
job.status = 'waiting' |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
87 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
88 |
job_id = job.key().id() |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
89 |
logging.warning("job %d now failed %d time(s)" % (job_id, job.errors)) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
90 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
91 |
return job.put() |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
92 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
93 |
def finishJob(self, job_key): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
94 |
"""A transaction to finish a job. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
95 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
96 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
97 |
job = Job.get_by_id(job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
98 |
job.status = 'finished' |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
99 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
100 |
return job.put() |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
101 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
102 |
def abortJob(self, job_key): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
103 |
"""A transaction to abort a job. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
104 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
105 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
106 |
job = Job.get_by_id(job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
107 |
job.status = 'aborted' |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
108 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
109 |
return job.put() |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
110 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
111 |
def handle(self, job_key): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
112 |
"""Handle one job. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
113 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
114 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
115 |
try: |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
116 |
job = db.run_in_transaction(self.claimJob, job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
117 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
118 |
if not job: |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
119 |
# someone already claimed the job |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
120 |
return True |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
121 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
122 |
if job.task_name not in self.tasks: |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
123 |
logging.error("Unknown job %s" % job.task_name) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
124 |
db.run_in_transaction(self.abortJob, job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
125 |
return True |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
126 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
127 |
task = self.tasks[job.task_name] |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
128 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
129 |
# excecute the actual job |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
130 |
task(job) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
131 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
132 |
db.run_in_transaction(self.finishJob, job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
133 |
return True |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
134 |
except DeadlineExceededError, exception: |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
135 |
db.run_in_transaction(self.freeJob, job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
136 |
return False |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
137 |
except Exception, exception: |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
138 |
logging.exception(exception) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
139 |
db.run_in_transaction(self.failJob, job_key) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
140 |
return True |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
141 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
142 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
143 |
handler = Handler() |