app/soc/views/models/cron.py
author Lennard de Rijk <ljvderijk@gmail.com>
Tue, 28 Jul 2009 11:23:11 +0200
changeset 2685 506cda0463e8
parent 2396 353941216ff9
permissions -rw-r--r--
Added columns to the manage project page that show the amount of evaluations. The total possible amount of evaluations that is shown is based on surveys which have at least one Record.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     1
#!/usr/bin/python2.5
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     2
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     3
# Copyright 2009 the Melange authors.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     4
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     5
# Licensed under the Apache License, Version 2.0 (the "License");
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     6
# you may not use this file except in compliance with the License.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     7
# You may obtain a copy of the License at
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     8
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     9
#   http://www.apache.org/licenses/LICENSE-2.0
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    10
#
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    11
# Unless required by applicable law or agreed to in writing, software
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    12
# distributed under the License is distributed on an "AS IS" BASIS,
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    14
# See the License for the specific language governing permissions and
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    15
# limitations under the License.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    16
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    17
"""Views for Cron.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    18
"""
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    19
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    20
__authors__ = [
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    21
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    22
  ]
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    23
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    24
2232
3c1e0b915803 Add a random chance to ignore a job to reduce contention
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2231
diff changeset
    25
import random
3c1e0b915803 Add a random chance to ignore a job to reduce contention
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2231
diff changeset
    26
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    27
from django import http
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    28
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    29
from soc.logic import dicts
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    30
from soc.logic.models.priority_group import logic as priority_group_logic
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    31
from soc.logic.models.job import logic as job_logic
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    32
from soc.views.helper import access
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    33
from soc.views.models import base
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    34
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    35
import soc.cron.job
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    36
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    37
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    38
class View(base.View):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    39
  """View methods for the Cron model.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    40
  """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    41
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    42
  def __init__(self, params=None):
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    43
    """Defines the fields and methods required for the base View class
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    44
    to provide the user with list, public, create, edit and delete views.
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    45
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    46
    Params:
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    47
      params: a dict with params for this View
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    48
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    49
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    50
    rights = access.Checker(params)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    51
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    52
    new_params = {}
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    53
    new_params['rights'] = rights
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    54
    new_params['logic'] = priority_group_logic
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    55
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    56
    new_params['name'] = "Cron"
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    57
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    58
    new_params['django_patterns_defaults'] = [
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    59
        (r'^%(url_name)s/(?P<access_type>poke)$',
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    60
          'soc.views.models.%(module_name)s.poke', 'Poke %(name_short)s'),
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    61
        ]
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    62
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    63
    params = dicts.merge(params, new_params)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    64
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    65
    super(View, self).__init__(params=params)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    66
2213
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    67
  def poke(self, request, access_type, page_name):
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    68
    """View called by the cron system that handles jobs.
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    69
2213
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    70
    Args:
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    71
      request: the standard Django HTTP request object
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    72
      access_type : the name of the access type which should be checked
c0f52da7a808 Remove testing method and update docstring for cron system
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2212
diff changeset
    73
      page_name: the page name displayed in templates as page and header title
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    74
    """
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    75
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    76
    order = ['-priority']
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    77
    query = priority_group_logic.getQueryForFields(order=order)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    78
    groups = priority_group_logic.getAll(query)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    79
    handler = soc.cron.job.handler
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    80
2215
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
    81
    groups_touched = 0
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    82
    jobs_completed = 0
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    83
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    84
    for group in groups:
2215
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
    85
      groups_touched += 1
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
    86
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    87
      filter = {
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    88
          'priority_group': group,
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    89
          'status': 'waiting',
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    90
          }
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    91
2231
6077018a1e68 Do not fetch all jobs with each poke
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2215
diff changeset
    92
      queryGen = lambda: job_logic.getQueryForFields(filter=filter)
2396
353941216ff9 also rename batch_size in cron.py
Sverre Rabbelier <sverre@rabbelier.nl>
parents: 2362
diff changeset
    93
      jobs = job_logic.entityIterator(queryGen, batch_size=10)
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    94
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    95
      retry_jobs = []
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    96
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    97
      for job in handler.iterate(jobs, retry_jobs):
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    98
        if random.randint(0, 3) > 0:
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
    99
          retry_jobs.append(job)
2232
3c1e0b915803 Add a random chance to ignore a job to reduce contention
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2231
diff changeset
   100
          continue
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   101
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   102
        job_key = job.key().id()
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   103
        status = handler.handle(job_key)
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   104
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   105
        if status is handler.OUT_OF_TIME:
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   106
          break
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   107
2246
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   108
        if status is handler.ERRORED:
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   109
          retry_jobs.append(job)
c29272f640b0 Tweak the 'load balancing' algorithm
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2232
diff changeset
   110
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   111
        jobs_completed += 1
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   112
2215
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
   113
    response = 'Completed %d jobs in %d priority groups.' % (
c5b7f6bc8c27 Changed the response message in the cron poke view.
Lennard de Rijk <ljvderijk@gmail.com>
parents: 2214
diff changeset
   114
        jobs_completed, groups_touched)
2211
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   115
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   116
    return http.HttpResponse(response)
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   117
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   118
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   119
view = View()
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   120
f7497180d037 Add cron, the core of the job system
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   121
poke = view.poke