app/soc/views/models/cron.py
author Sverre Rabbelier <sverre@rabbelier.nl>
Fri, 05 Jun 2009 08:24:32 +0200
changeset 2396 353941216ff9
parent 2362 f211b75922ac
permissions -rw-r--r--
also rename batch_size in cron.py
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     1
#!/usr/bin/python2.5
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     2
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     3
# Copyright 2009 the Melange authors.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     4
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     5
# Licensed under the Apache License, Version 2.0 (the "License");
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     6
# you may not use this file except in compliance with the License.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     7
# You may obtain a copy of the License at
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     8
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     9
#   http://www.apache.org/licenses/LICENSE-2.0
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    10
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    11
# Unless required by applicable law or agreed to in writing, software
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    12
# distributed under the License is distributed on an "AS IS" BASIS,
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    14
# See the License for the specific language governing permissions and
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    15
# limitations under the License.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    16
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    17
"""Views for Cron.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    18
"""
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    19
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    20
__authors__ = [
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    21
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    22
  ]
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    23
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    24
2232
3c1e0b915803 Add a random chance to ignore a job to reduce contention
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2231
diff changeset
    25
import random
3c1e0b915803 Add a random chance to ignore a job to reduce contention
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2231
diff changeset
    26
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    27
from django import http
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    28
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    29
from soc.logic import dicts
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    30
from soc.logic.models.priority_group import logic as priority_group_logic
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    31
from soc.logic.models.job import logic as job_logic
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    32
from soc.views.helper import access
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    33
from soc.views.models import base
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    34
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    35
import soc.cron.job
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    36
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    37
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    38
class View(base.View):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    39
  """View methods for the Cron model.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    40
  """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    41
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    42
  def __init__(self, params=None):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    43
    """Defines the fields and methods required for the base View class
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    44
    to provide the user with list, public, create, edit and delete views.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    45
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    46
    Params:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    47
      params: a dict with params for this View
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    48
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    49
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    50
    rights = access.Checker(params)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    51
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    52
    new_params = {}
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    53
    new_params['rights'] = rights
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    54
    new_params['logic'] = priority_group_logic
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    55
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    56
    new_params['name'] = "Cron"
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    57
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    58
    new_params['django_patterns_defaults'] = [
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    59
        (r'^%(url_name)s/(?P<access_type>poke)$',
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    60
          'soc.views.models.%(module_name)s.poke', 'Poke %(name_short)s'),
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    61
        ]
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    62
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    63
    params = dicts.merge(params, new_params)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    64
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    65
    super(View, self).__init__(params=params)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    66
2213
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    67
  def poke(self, request, access_type, page_name):
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    68
    """View called by the cron system that handles jobs.
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    69
2213
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    70
    Args:
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    71
      request: the standard Django HTTP request object
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    72
      access_type : the name of the access type which should be checked
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    73
      page_name: the page name displayed in templates as page and header title
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    74
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    75
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    76
    order = ['-priority']
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    77
    query = priority_group_logic.getQueryForFields(order=order)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    78
    groups = priority_group_logic.getAll(query)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    79
    handler = soc.cron.job.handler
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    80
2215
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
    81
    groups_touched = 0
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    82
    jobs_completed = 0
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    83
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    84
    for group in groups:
2215
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
    85
      groups_touched += 1
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
    86
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    87
      filter = {
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    88
          'priority_group': group,
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    89
          'status': 'waiting',
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    90
          }
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    91
2231
6077018a1e68 Do not fetch all jobs with each poke
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2215
diff changeset
    92
      queryGen = lambda: job_logic.getQueryForFields(filter=filter)
2396
353941216ff9 also rename batch_size in cron.py
Sverre Rabbelier <sverre@rabbelier.nl>
parents: 2362
diff changeset
    93
      jobs = job_logic.entityIterator(queryGen, batch_size=10)
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    94
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    95
      retry_jobs = []
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    96
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    97
      for job in handler.iterate(jobs, retry_jobs):
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    98
        if random.randint(0, 3) > 0:
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    99
          retry_jobs.append(job)
2232
3c1e0b915803 Add a random chance to ignore a job to reduce contention
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2231
diff changeset
   100
          continue
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   101
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   102
        job_key = job.key().id()
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   103
        status = handler.handle(job_key)
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   104
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   105
        if status is handler.OUT_OF_TIME:
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   106
          break
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   107
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   108
        if status is handler.ERRORED:
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   109
          retry_jobs.append(job)
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   110
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   111
        jobs_completed += 1
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   112
2215
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
   113
    response = 'Completed %d jobs in %d priority groups.' % (
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
   114
        jobs_completed, groups_touched)
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   115
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   116
    return http.HttpResponse(response)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   117
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   118
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   119
view = View()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   120
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   121
poke = view.poke