Add support for prefetching fields
authorSverre Rabbelier <srabbelier@gmail.com>
Wed, 02 Sep 2009 10:24:10 +0200
changeset 2854 003a84e774e2
parent 2853 80a71738cb4f
child 2855 ec2ed1571e3a
Add support for prefetching fields This makes it possible to specify the fields that should be prefetched as part of a getForFields query. Doing a prefetch on a ReferenceProperty when it is known that the property will be referenced in (almost all) entities is far more efficient than doing the invididual fetches sequentially.
app/soc/logic/models/base.py
--- a/app/soc/logic/models/base.py	Mon Aug 31 05:06:17 2009 -0700
+++ b/app/soc/logic/models/base.py	Wed Sep 02 10:24:10 2009 +0200
@@ -312,8 +312,37 @@
 
     raise out_of_band.Error(msg, status=404)
 
+  def prefetchField(self, field, data):
+    """Prefetches all fields in data from the datastore in one fetch.
+
+    Args:
+      field: the field that should be fetched
+      data: the data for which the prefetch should be done
+    """
+
+    prop = getattr(self._model, field, None)
+
+    if not prop:
+      logging.exception("Model %s does not have attribute %s" %
+                        (self._model, field))
+      return
+
+    if not isinstance(prop, db.ReferenceProperty):
+      logging.exception("Property %s of %s is not a ReferenceProperty but a %s" %
+                        (field, self._model.kind(), prop.__class__.__name__))
+      return
+
+    keys = [prop.get_value_for_datastore(i) for i in data]
+
+    prefetched_entities = db.get(keys)
+    prefetched_dict = dict((i.key(), i) for i in prefetched_entities)
+
+    for i in data:
+      value = prefetched_dict[prop.get_value_for_datastore(i)]
+      setattr(i, field, value)
+
   def getForFields(self, filter=None, unique=False, limit=1000, offset=0,
-                   ancestors=None, order=None):
+                   ancestors=None, order=None, prefetch=None):
     """Returns all entities that have the specified properties.
 
     Args:
@@ -323,11 +352,16 @@
       offset: the position to start at
       ancestors: list of ancestors properties to set for this query
       order: a list with the sort order
+      prefetch: the fields of the data that should be pre-fetched,
+          has no effect if unique is True
     """
 
     if unique:
       limit = 1
 
+    if not prefetch:
+      prefetch = []
+
     query = self.getQueryForFields(filter=filter, 
                                    ancestors=ancestors, order=order)
 
@@ -342,6 +376,9 @@
     if unique:
       return result[0] if result else None
 
+    for field in prefetch:
+      self.prefetchField(field, result)
+
     return result
 
   def getQueryForFields(self, filter=None, ancestors=None, order=None):