# HG changeset patch # User Daniel Hans # Date 1254087099 -7200 # Node ID eeee8c8544382c0f085ee84f436403e49023636a # Parent 8a99de852dc274aa29afa6327a78cf4be2699c1b Add task and iterative_task decorator Also add getBatchOfData method to the base logic and tasks logic module. Reviewed-by: Sverre Rabbelier diff -r 8a99de852dc2 -r eeee8c854438 app/soc/logic/models/base.py --- a/app/soc/logic/models/base.py Sun Sep 27 18:18:53 2009 +0200 +++ b/app/soc/logic/models/base.py Sun Sep 27 23:31:39 2009 +0200 @@ -586,8 +586,36 @@ offset = offset + chunk return result + + def getBatchOfData(self, filter=None, order=None, next_key=None): + """Returns one batch of entities + + Args: + filter: a dict for the properties that the entities should have + order: a list with the sort order + next_key: a key for the first entity that should be returned + + Returns: + A tuple: list of fetched entities and key value for the entity + that should be fetched at first for the next batch + """ + + query = self.getQueryForFields(filter=filter, order=order) + + if next_key is not None: + query.filter('__key__ >=', next_key) + + entities = query.fetch(self.BATCH_SIZE + 1) + + next_key = None + if len(entities) == self.BATCH_SIZE + 1: + next_entity = entities.pop() + next_key = next_entity.key() + + return entities, next_key + # pylint: disable-msg=C0103 - def entityIterator(self, queryGen, batch_size = 100): + def entityIterator(self, queryGen, batch_size=100): """Iterator that yields an entity in batches. Args: diff -r 8a99de852dc2 -r eeee8c854438 app/soc/logic/tasks.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/app/soc/logic/tasks.py Sun Sep 27 23:31:39 2009 +0200 @@ -0,0 +1,64 @@ +#!/usr/bin/python2.5 +# +# Copyright 2008 the Melange authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers functions for dealing with task queue api +""" + +__authors__ = [ + '"Daniel Hans" ', + ] + +import logging +from django import http + +# TODO(labs): fix when taskqueue graduates from labs +from google.appengine.api.labs import taskqueue + + +class Error(Exception): + """Base class for all exceptions raised by this module. + """ + + pass + + +class FatalTaskError(Error): + """Class for all errors that lead to immediate task abortion. + """ + pass + + +def startTask(url, queue_name='default', context=None, **kwargs): + """Adds a new task to one of the queues + """ + + queue = taskqueue.Queue(name=queue_name) + return queue.add(taskqueue.Task(url=url, params=context)) + + +def terminateTask(): + """Generates http response which causes that the task is + is not added to the queue again + """ + + return http.HttpResponse(status=200) + +def repeatTask(): + """Generates http response which causes that the task is + added to the queue again + """ + + return http.HttpResponse(status=500) diff -r 8a99de852dc2 -r eeee8c854438 app/soc/views/helper/decorators.py --- a/app/soc/views/helper/decorators.py Sun Sep 27 18:18:53 2009 +0200 +++ b/app/soc/views/helper/decorators.py Sun Sep 27 23:31:39 2009 +0200 @@ -31,6 +31,7 @@ from django.utils.translation import ugettext from soc.logic import dicts +from soc.logic import tasks from soc.views.helper import responses @@ -116,3 +117,97 @@ return func(self, request, access_type, *args, **kwargs) return wrapper + + +def task(func): + """Task decorator wrapper method + """ + + @wraps(func) + def wrapper(request, *args, **kwargs): + """Decorator wrapper method + """ + + try: + return func(request, *args, **kwargs) + except tasks.FatalTaskError, error: + logging.exception(error) + return tasks.terminateTask() + except Exception, exception: + logging.exception(exception) + return tasks.repeatTask() + + return wrapper + + +def iterative_task(func): + """Iterative wrapper method + """ + + @wraps(func) + def wrapper(request, *args, **kwargs): + """Decorator wrapper method + + Params usage: + logic: name of the logic for the data model to iterate through + filter: a dict for the properties that the entities should have + order: a list with the sort order + json: json object with additional parameters + + Returns: + Standard http django response + """ + + post_dict = request.POST + + if 'logic' not in post_dict: + return tasks.terminateTask() + + _temp = __import__(post_dict['logic'], globals(), locals(), ['logic'], -1) + logic = _temp.logic + + filter = None + if 'filter' in post_dict: + filter = simplejson.loads(post_dict['filter']) + + order = None + if 'order' in post_dict: + order = simplejson.loads(post_dict['order']) + + start_key = None + if 'next_key' in post_dict: + start_key = db.Key(post_dict['start_key']) + + json = None + if 'json' in post_dict: + json = post_dict['json'] + + entities, start_key = logic.getBatchOfData(filter, order, start_key) + + try: + new_json = func(request, entities=entities, json=json, *args, **kwargs) + except tasks.FatalTaskError, error: + logging.error(error) + return tasks.terminateTask() + except Exception, exception: + logging.error(exception) + return tasks.repeatTask() + + if start_key is None: + logging.debug('Task sucessfully completed') + else: + context = post_dict.copy() + + if 'json' in context: + del context['json'] + + context.update({'start_key': start_key}) + + if new_json is not None: + context.update({'json': new_json}) + + tasks.startTask(url=request.path, context=context) + + return tasks.terminateTask() + + return wrapper