513 |
513 |
514 result.extend(data) |
514 result.extend(data) |
515 offset = offset + chunk |
515 offset = offset + chunk |
516 |
516 |
517 return result |
517 return result |
518 |
518 # pylint: disable-msg=C0103 |
519 def entityIterator(self, queryGen, batchSize = 100): |
519 def entityIterator(self, queryGen, batch_size = 100): |
520 """Iterator that yields an entity in batches. |
520 """Iterator that yields an entity in batches. |
521 |
521 |
522 Args: |
522 Args: |
523 queryGen: should return a Query object |
523 queryGen: should return a Query object |
524 batchSize: how many entities to retrieve in one datastore call |
524 batchSize: how many entities to retrieve in one datastore call |
525 |
525 |
526 Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook). |
526 Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook). |
527 """ |
527 """ |
528 |
528 |
529 # AppEngine will not fetch more than 1000 results |
529 # AppEngine will not fetch more than 1000 results |
530 batchSize = min(batchSize,1000) |
530 batch_size = min(batch_size, 1000) |
531 |
531 |
532 done = False |
532 done = False |
533 count = 0 |
533 count = 0 |
534 key = None |
534 key = None |
535 |
535 |
536 while not done: |
536 while not done: |
537 query = queryGen() |
537 query = queryGen() |
538 if key: |
538 if key: |
539 query.filter("__key__ > ",key) |
539 query.filter("__key__ > ", key) |
540 results = query.fetch(batchSize) |
540 results = query.fetch(batch_size) |
541 for result in results: |
541 for result in results: |
542 count += 1 |
542 count += 1 |
543 yield result |
543 yield result |
544 if batchSize > len(results): |
544 if batch_size > len(results): |
545 done = True |
545 done = True |
546 else: |
546 else: |
547 key = results[-1].key() |
547 key = results[-1].key() |
548 |
548 |
549 def _createField(self, entity_properties, name): |
549 def _createField(self, entity_properties, name): |