author | Lennard de Rijk <ljvderijk@gmail.com> |
Tue, 28 Jul 2009 11:23:11 +0200 | |
changeset 2685 | 506cda0463e8 |
parent 2396 | 353941216ff9 |
permissions | -rw-r--r-- |
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
1 |
#!/usr/bin/python2.5 |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
2 |
# |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
3 |
# Copyright 2009 the Melange authors. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
4 |
# |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
5 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
6 |
# you may not use this file except in compliance with the License. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
7 |
# You may obtain a copy of the License at |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
8 |
# |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
9 |
# http://www.apache.org/licenses/LICENSE-2.0 |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
10 |
# |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
11 |
# Unless required by applicable law or agreed to in writing, software |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
12 |
# distributed under the License is distributed on an "AS IS" BASIS, |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
13 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
14 |
# See the License for the specific language governing permissions and |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
15 |
# limitations under the License. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
16 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
17 |
"""Views for Cron. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
18 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
19 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
20 |
__authors__ = [ |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
21 |
'"Sverre Rabbelier" <sverre@rabbelier.nl>', |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
22 |
] |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
23 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
24 |
|
2232
3c1e0b915803
Add a random chance to ignore a job to reduce contention
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2231
diff
changeset
|
25 |
import random |
3c1e0b915803
Add a random chance to ignore a job to reduce contention
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2231
diff
changeset
|
26 |
|
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
27 |
from django import http |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
28 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
29 |
from soc.logic import dicts |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
30 |
from soc.logic.models.priority_group import logic as priority_group_logic |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
31 |
from soc.logic.models.job import logic as job_logic |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
32 |
from soc.views.helper import access |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
33 |
from soc.views.models import base |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
34 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
35 |
import soc.cron.job |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
36 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
37 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
38 |
class View(base.View): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
39 |
"""View methods for the Cron model. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
40 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
41 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
42 |
def __init__(self, params=None): |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
43 |
"""Defines the fields and methods required for the base View class |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
44 |
to provide the user with list, public, create, edit and delete views. |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
45 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
46 |
Params: |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
47 |
params: a dict with params for this View |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
48 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
49 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
50 |
rights = access.Checker(params) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
51 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
52 |
new_params = {} |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
53 |
new_params['rights'] = rights |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
54 |
new_params['logic'] = priority_group_logic |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
55 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
56 |
new_params['name'] = "Cron" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
57 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
58 |
new_params['django_patterns_defaults'] = [ |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
59 |
(r'^%(url_name)s/(?P<access_type>poke)$', |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
60 |
'soc.views.models.%(module_name)s.poke', 'Poke %(name_short)s'), |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
61 |
] |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
62 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
63 |
params = dicts.merge(params, new_params) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
64 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
65 |
super(View, self).__init__(params=params) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
66 |
|
2213
c0f52da7a808
Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2212
diff
changeset
|
67 |
def poke(self, request, access_type, page_name): |
c0f52da7a808
Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2212
diff
changeset
|
68 |
"""View called by the cron system that handles jobs. |
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
69 |
|
2213
c0f52da7a808
Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2212
diff
changeset
|
70 |
Args: |
c0f52da7a808
Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2212
diff
changeset
|
71 |
request: the standard Django HTTP request object |
c0f52da7a808
Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2212
diff
changeset
|
72 |
access_type : the name of the access type which should be checked |
c0f52da7a808
Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2212
diff
changeset
|
73 |
page_name: the page name displayed in templates as page and header title |
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
74 |
""" |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
75 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
76 |
order = ['-priority'] |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
77 |
query = priority_group_logic.getQueryForFields(order=order) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
78 |
groups = priority_group_logic.getAll(query) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
79 |
handler = soc.cron.job.handler |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
80 |
|
2215
c5b7f6bc8c27
Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents:
2214
diff
changeset
|
81 |
groups_touched = 0 |
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
82 |
jobs_completed = 0 |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
83 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
84 |
for group in groups: |
2215
c5b7f6bc8c27
Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents:
2214
diff
changeset
|
85 |
groups_touched += 1 |
c5b7f6bc8c27
Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents:
2214
diff
changeset
|
86 |
|
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
87 |
filter = { |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
88 |
'priority_group': group, |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
89 |
'status': 'waiting', |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
90 |
} |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
91 |
|
2231
6077018a1e68
Do not fetch all jobs with each poke
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2215
diff
changeset
|
92 |
queryGen = lambda: job_logic.getQueryForFields(filter=filter) |
2396
353941216ff9
also rename batch_size in cron.py
Sverre Rabbelier <sverre@rabbelier.nl>
parents:
2362
diff
changeset
|
93 |
jobs = job_logic.entityIterator(queryGen, batch_size=10) |
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
94 |
|
2246
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
95 |
retry_jobs = [] |
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
96 |
|
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
97 |
for job in handler.iterate(jobs, retry_jobs): |
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
98 |
if random.randint(0, 3) > 0: |
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
99 |
retry_jobs.append(job) |
2232
3c1e0b915803
Add a random chance to ignore a job to reduce contention
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2231
diff
changeset
|
100 |
continue |
2246
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
101 |
|
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
102 |
job_key = job.key().id() |
2246
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
103 |
status = handler.handle(job_key) |
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
104 |
|
2246
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
105 |
if status is handler.OUT_OF_TIME: |
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
106 |
break |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
107 |
|
2246
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
108 |
if status is handler.ERRORED: |
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
109 |
retry_jobs.append(job) |
c29272f640b0
Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents:
2232
diff
changeset
|
110 |
|
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
111 |
jobs_completed += 1 |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
112 |
|
2215
c5b7f6bc8c27
Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents:
2214
diff
changeset
|
113 |
response = 'Completed %d jobs in %d priority groups.' % ( |
c5b7f6bc8c27
Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents:
2214
diff
changeset
|
114 |
jobs_completed, groups_touched) |
2211
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
115 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
116 |
return http.HttpResponse(response) |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
117 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
118 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
119 |
view = View() |
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
120 |
|
f7497180d037
Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff
changeset
|
121 |
poke = view.poke |