equal
deleted
inserted
replaced
25 >>> result = [i for i in it] |
25 >>> result = [i for i in it] |
26 """ |
26 """ |
27 |
27 |
28 __authors__ = [ |
28 __authors__ = [ |
29 '"Sverre Rabbelier" <sverre@rabbelier.nl>', |
29 '"Sverre Rabbelier" <sverre@rabbelier.nl>', |
|
30 '"Pawel Solyga" <pawel.solyga@gmail.com>', |
30 ] |
31 ] |
31 |
|
32 |
32 |
33 |
33 |
34 import code |
34 import code |
35 import getpass |
35 import getpass |
36 import os |
36 import os |
42 """ |
42 """ |
43 |
43 |
44 return raw_input('Username:'), getpass.getpass('Password:') |
44 return raw_input('Username:'), getpass.getpass('Password:') |
45 |
45 |
46 |
46 |
47 def deepFetch(queryGen,key=None,batchSize = 100): |
47 def deepFetch(queryGen, key=None, filters=None, batchSize = 100): |
48 """Iterator that yields an entity in batches. |
48 """Iterator that yields an entity in batches. |
49 |
49 |
50 Args: |
50 Args: |
51 queryGen: should return a Query object |
51 queryGen: should return a Query object |
52 key: used to .filter() for __key__ |
52 key: used to .filter() for __key__ |
|
53 filters: dict with additional filters |
53 batchSize: how many entities to retrieve in one datastore call |
54 batchSize: how many entities to retrieve in one datastore call |
54 |
55 |
55 Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook). |
56 Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook). |
56 """ |
57 """ |
57 |
58 |
58 from google.appengine.ext import db |
59 from google.appengine.ext import db |
59 |
60 |
60 # AppEngine will not fetch more than 1000 results |
61 # AppEngine will not fetch more than 1000 results |
61 batchSize = min(batchSize,1000) |
62 batchSize = min(batchSize, 1000) |
62 |
63 |
63 query = None |
64 query = None |
64 done = False |
65 done = False |
65 count = 0 |
66 count = 0 |
66 |
67 |
68 key = db.Key(key) |
69 key = db.Key(key) |
69 |
70 |
70 while not done: |
71 while not done: |
71 print count |
72 print count |
72 query = queryGen() |
73 query = queryGen() |
|
74 |
|
75 if filters: |
|
76 for key, value in filters.items(): |
|
77 query.filter(key, value) |
|
78 |
73 if key: |
79 if key: |
74 query.filter("__key__ > ",key) |
80 query.filter("__key__ > ", key) |
|
81 |
75 results = query.fetch(batchSize) |
82 results = query.fetch(batchSize) |
76 for result in results: |
83 for result in results: |
77 count += 1 |
84 count += 1 |
78 yield result |
85 yield result |
79 if batchSize > len(results): |
86 if batchSize > len(results): |