# HG changeset patch
# User Sverre Rabbelier <srabbelier@gmail.com>
# Date 1238713556 0
# Node ID 773b13d8630905c0c0ef83bd6af0b4ee24f16a37
# Parent  990c82e5baa970c4d14b55b84c9d4247b9b413ca
Add dateFetch

This way it's possible to incrementally fetch new or modified
entities given a certain base set.

Patch by: Sverre Rabbelier

diff -r 990c82e5baa9 -r 773b13d86309 scripts/stats.py
--- a/scripts/stats.py	Thu Apr 02 23:05:43 2009 +0000
+++ b/scripts/stats.py	Thu Apr 02 23:05:56 2009 +0000
@@ -29,6 +29,42 @@
 import interactive
 
 
+def dateFetch(queryGen, last=None, batchSize=100):
+  """Iterator that yields an entity in batches.
+
+  Args:
+    queryGen: should return a Query object
+    last: used to .filter() for last_modified_on
+    batchSize: how many entities to retrieve in one datastore call
+
+  Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook).
+  """
+
+  from google.appengine.ext import db
+
+   # AppEngine will not fetch more than 1000 results
+  batchSize = min(batchSize,1000)
+
+  query = None
+  done = False
+  count = 0
+
+  while not done:
+    print count
+    query = queryGen()
+    query.order('last_modified_on')
+    if last:
+      query.filter("last_modified_on > ", last)
+    results = query.fetch(batchSize)
+    for result in results:
+      count += 1
+      yield result
+    if batchSize > len(results):
+      done = True
+    else:
+      last = results[-1].last_modified_on
+
+
 def addKey(target, fieldname):
   """Adds the key of the specified field.
   """