thirdparty/google_appengine/google/appengine/api/datastore.py
changeset 2864 2e0b0af889be
parent 2413 d0b7dac5325c
child 3031 7678f72140e6
equal deleted inserted replaced
2862:27971a13089f 2864:2e0b0af889be
    47 from google.appengine.datastore import datastore_index
    47 from google.appengine.datastore import datastore_index
    48 from google.appengine.datastore import datastore_pb
    48 from google.appengine.datastore import datastore_pb
    49 from google.appengine.runtime import apiproxy_errors
    49 from google.appengine.runtime import apiproxy_errors
    50 from google.appengine.datastore import entity_pb
    50 from google.appengine.datastore import entity_pb
    51 
    51 
       
    52 try:
       
    53   from google.appengine.api.labs.taskqueue import taskqueue_service_pb
       
    54 except ImportError:
       
    55   from google.appengine.api.taskqueue import taskqueue_service_pb
       
    56 
    52 MAX_ALLOWABLE_QUERIES = 30
    57 MAX_ALLOWABLE_QUERIES = 30
    53 
    58 
    54 DEFAULT_TRANSACTION_RETRIES = 3
    59 DEFAULT_TRANSACTION_RETRIES = 3
    55 
    60 
    56 _MAX_INDEXED_PROPERTIES = 5000
    61 _MAX_INDEXED_PROPERTIES = 5000
       
    62 
       
    63 _MAX_ID_BATCH_SIZE = 1000 * 1000 * 1000
    57 
    64 
    58 Key = datastore_types.Key
    65 Key = datastore_types.Key
    59 typename = datastore_types.typename
    66 typename = datastore_types.typename
    60 
    67 
    61 _txes = {}
    68 _txes = {}
   145 
   152 
   146   if multiple and not entities:
   153   if multiple and not entities:
   147     return []
   154     return []
   148 
   155 
   149   for entity in entities:
   156   for entity in entities:
   150     if not entity.kind() or not entity.app():
   157     if not entity.kind() or not entity.app_id_namespace():
   151       raise datastore_errors.BadRequestError(
   158       raise datastore_errors.BadRequestError(
   152           'App and kind must not be empty, in entity: %s' % entity)
   159           'App and kind must not be empty, in entity: %s' % entity)
   153 
   160 
   154   req = datastore_pb.PutRequest()
   161   req = datastore_pb.PutRequest()
   155   req.entity_list().extend([e._ToPb() for e in entities])
   162   req.entity_list().extend([e._ToPb() for e in entities])
   156 
   163 
   157   keys = [e.key() for e in entities]
   164   keys = [e.key() for e in entities]
   158   tx = _MaybeSetupTransaction(req, keys)
   165   tx = _MaybeSetupTransaction(req, keys)
   159   if tx:
       
   160     tx.RecordModifiedKeys([k for k in keys if k.has_id_or_name()])
       
   161 
   166 
   162   resp = datastore_pb.PutResponse()
   167   resp = datastore_pb.PutResponse()
   163   try:
   168   try:
   164     apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Put', req, resp)
   169     apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Put', req, resp)
   165   except apiproxy_errors.ApplicationError, err:
   170   except apiproxy_errors.ApplicationError, err:
   175 
   180 
   176   for entity, key in zip(entities, keys):
   181   for entity, key in zip(entities, keys):
   177     entity._Entity__key._Key__reference.CopyFrom(key)
   182     entity._Entity__key._Key__reference.CopyFrom(key)
   178 
   183 
   179   if tx:
   184   if tx:
   180     tx.RecordModifiedKeys([e.key() for e in entities], error_on_repeat=False)
       
   181     tx.entity_group = entities[0].entity_group()
   185     tx.entity_group = entities[0].entity_group()
   182 
   186 
   183   if multiple:
   187   if multiple:
   184     return [Key._FromPb(k) for k in keys]
   188     return [Key._FromPb(k) for k in keys]
   185   else:
   189   else:
   257 
   261 
   258   req = datastore_pb.DeleteRequest()
   262   req = datastore_pb.DeleteRequest()
   259   req.key_list().extend([key._Key__reference for key in keys])
   263   req.key_list().extend([key._Key__reference for key in keys])
   260 
   264 
   261   tx = _MaybeSetupTransaction(req, keys)
   265   tx = _MaybeSetupTransaction(req, keys)
   262   if tx:
       
   263     tx.RecordModifiedKeys(keys)
       
   264 
   266 
   265   resp = datastore_pb.DeleteResponse()
   267   resp = datastore_pb.DeleteResponse()
   266   try:
   268   try:
   267     apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Delete', req, resp)
   269     apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Delete', req, resp)
   268   except apiproxy_errors.ApplicationError, err:
   270   except apiproxy_errors.ApplicationError, err:
   273   """A datastore entity.
   275   """A datastore entity.
   274 
   276 
   275   Includes read-only accessors for app id, kind, and primary key. Also
   277   Includes read-only accessors for app id, kind, and primary key. Also
   276   provides dictionary-style access to properties.
   278   provides dictionary-style access to properties.
   277   """
   279   """
   278   def __init__(self, kind, parent=None, _app=None, name=None,
   280   def __init__(self, kind, parent=None, _app=None, name=None, id=None,
   279                unindexed_properties=[]):
   281                unindexed_properties=[], _namespace=None):
   280     """Constructor. Takes the kind and transaction root, which cannot be
   282     """Constructor. Takes the kind and transaction root, which cannot be
   281     changed after the entity is constructed, and an optional parent. Raises
   283     changed after the entity is constructed, and an optional parent. Raises
   282     BadArgumentError or BadKeyError if kind is invalid or parent is not an
   284     BadArgumentError or BadKeyError if kind is invalid or parent is not an
   283     existing Entity or Key in the datastore.
   285     existing Entity or Key in the datastore.
   284 
   286 
   287       kind: string
   289       kind: string
   288       # if provided, this entity's parent. Its key must be complete.
   290       # if provided, this entity's parent. Its key must be complete.
   289       parent: Entity or Key
   291       parent: Entity or Key
   290       # if provided, this entity's name.
   292       # if provided, this entity's name.
   291       name: string
   293       name: string
       
   294       # if provided, this entity's id.
       
   295       id: integer
   292       # if provided, a sequence of property names that should not be indexed
   296       # if provided, a sequence of property names that should not be indexed
   293       # by the built-in single property indices.
   297       # by the built-in single property indices.
   294       unindexed_properties: list or tuple of strings
   298       unindexed_properties: list or tuple of strings
   295     """
   299     """
   296     ref = entity_pb.Reference()
   300     ref = entity_pb.Reference()
   297     _app = datastore_types.ResolveAppId(_app)
   301     _app_namespace = datastore_types.ResolveAppIdNamespace(_app, _namespace)
   298     ref.set_app(_app)
   302     ref.set_app(_app_namespace.to_encoded())
   299 
   303 
   300     datastore_types.ValidateString(kind, 'kind',
   304     datastore_types.ValidateString(kind, 'kind',
   301                                    datastore_errors.BadArgumentError)
   305                                    datastore_errors.BadArgumentError)
   302 
       
   303     if parent is not None:
   306     if parent is not None:
   304       parent = _GetCompleteKeyOrError(parent)
   307       parent = _GetCompleteKeyOrError(parent)
   305       if _app != parent.app():
   308       if _app_namespace != parent.app_id_namespace():
   306         raise datastore_errors.BadArgumentError(
   309         raise datastore_errors.BadArgumentError(
   307             "_app %s doesn't match parent's app %s" % (_app, parent.app()))
   310             " %s doesn't match parent's app_namespace %s" %
       
   311             (_app_namespace, parent.app_id_namespace()))
   308       ref.CopyFrom(parent._Key__reference)
   312       ref.CopyFrom(parent._Key__reference)
   309 
   313 
   310     last_path = ref.mutable_path().add_element()
   314     last_path = ref.mutable_path().add_element()
   311     last_path.set_type(kind.encode('utf-8'))
   315     last_path.set_type(kind.encode('utf-8'))
   312 
   316 
       
   317     if name is not None and id is not None:
       
   318       raise datastore_errors.BadArgumentError(
       
   319           "Cannot set both name and id on an Entity")
       
   320 
   313     if name is not None:
   321     if name is not None:
   314       datastore_types.ValidateString(name, 'name')
   322       datastore_types.ValidateString(name, 'name')
   315       if name[0] in string.digits:
       
   316         raise datastore_errors.BadValueError('name cannot begin with a digit')
       
   317       last_path.set_name(name.encode('utf-8'))
   323       last_path.set_name(name.encode('utf-8'))
       
   324 
       
   325     if id is not None:
       
   326       datastore_types.ValidateInteger(id, 'id')
       
   327       last_path.set_id(id)
   318 
   328 
   319     unindexed_properties, multiple = NormalizeAndTypeCheck(unindexed_properties, basestring)
   329     unindexed_properties, multiple = NormalizeAndTypeCheck(unindexed_properties, basestring)
   320     if not multiple:
   330     if not multiple:
   321       raise datastore_errors.BadArgumentError(
   331       raise datastore_errors.BadArgumentError(
   322         'unindexed_properties must be a sequence; received %s (a %s).' %
   332         'unindexed_properties must be a sequence; received %s (a %s).' %
   327 
   337 
   328     self.__key = Key._FromPb(ref)
   338     self.__key = Key._FromPb(ref)
   329 
   339 
   330   def app(self):
   340   def app(self):
   331     """Returns the name of the application that created this entity, a
   341     """Returns the name of the application that created this entity, a
   332     string.
   342     string or None if not set.
   333     """
   343     """
   334     return self.__key.app()
   344     return self.__key.app()
       
   345 
       
   346   def namespace(self):
       
   347     """Returns the namespace of this entity, a string or None.
       
   348     """
       
   349     return self.__key.namespace()
       
   350 
       
   351   def app_id_namespace(self):
       
   352     """Returns the AppIdNamespace of this entity or None if not set.
       
   353     """
       
   354     return self.__key.app_id_namespace()
   335 
   355 
   336   def kind(self):
   356   def kind(self):
   337     """Returns this entity's kind, a string.
   357     """Returns this entity's kind, a string.
   338     """
   358     """
   339     return self.__key.kind()
   359     return self.__key.kind()
       
   360 
       
   361   def is_saved(self):
       
   362     """Returns if this entity has been saved to the datastore
       
   363     """
       
   364     last_path = self.__key._Key__reference.path().element_list()[-1]
       
   365     return ((last_path.has_name() ^ last_path.has_id()) and
       
   366             self.__key.has_id_or_name())
   340 
   367 
   341   def key(self):
   368   def key(self):
   342     """Returns this entity's primary key, a Key instance.
   369     """Returns this entity's primary key, a Key instance.
   343     """
   370     """
   344     return self.__key
   371     return self.__key
   481         else:
   508         else:
   482           xml.append(saxutils.escape(unicode(val)))
   509           xml.append(saxutils.escape(unicode(val)))
   483 
   510 
   484     return xml
   511     return xml
   485 
   512 
   486   def _ToPb(self):
   513   def ToPb(self):
       
   514     """Converts this Entity to its protocol buffer representation.
       
   515 
       
   516     Returns:
       
   517       entity_pb.Entity
       
   518     """
       
   519     return self._ToPb(False)
       
   520 
       
   521   def _ToPb(self, mark_key_as_saved=True):
   487     """Converts this Entity to its protocol buffer representation. Not
   522     """Converts this Entity to its protocol buffer representation. Not
   488     intended to be used by application developers.
   523     intended to be used by application developers.
   489 
   524 
   490     Returns:
   525     Returns:
   491       entity_pb.Entity
   526       entity_pb.Entity
   492     """
   527     """
   493 
   528 
   494     pb = entity_pb.EntityProto()
   529     pb = entity_pb.EntityProto()
   495     pb.mutable_key().CopyFrom(self.key()._ToPb())
   530     pb.mutable_key().CopyFrom(self.key()._ToPb())
       
   531     last_path = pb.key().path().element_list()[-1]
       
   532     if mark_key_as_saved and last_path.has_name() and last_path.has_id():
       
   533       last_path.clear_id()
   496 
   534 
   497     group = pb.mutable_entity_group()
   535     group = pb.mutable_entity_group()
   498     if self.__key.has_id_or_name():
   536     if self.__key.has_id_or_name():
   499       root = pb.key().path().element(0)
   537       root = pb.key().path().element(0)
   500       group.add_element().CopyFrom(root)
   538       group.add_element().CopyFrom(root)
   521           'Too many indexed properties for entity %r.' % self.key())
   559           'Too many indexed properties for entity %r.' % self.key())
   522 
   560 
   523     return pb
   561     return pb
   524 
   562 
   525   @staticmethod
   563   @staticmethod
   526   def _FromPb(pb):
   564   def FromPb(pb):
       
   565     """Static factory method. Returns the Entity representation of the
       
   566     given protocol buffer (datastore_pb.Entity).
       
   567 
       
   568     Args:
       
   569       pb: datastore_pb.Entity or str encoding of a datastore_pb.Entity
       
   570 
       
   571     Returns:
       
   572       Entity: the Entity representation of pb
       
   573     """
       
   574     if isinstance(pb, str):
       
   575       real_pb = entity_pb.EntityProto()
       
   576       real_pb.ParseFromString(pb)
       
   577       pb = real_pb
       
   578 
       
   579     return Entity._FromPb(pb, require_valid_key=False)
       
   580 
       
   581   @staticmethod
       
   582   def _FromPb(pb, require_valid_key=True):
   527     """Static factory method. Returns the Entity representation of the
   583     """Static factory method. Returns the Entity representation of the
   528     given protocol buffer (datastore_pb.Entity). Not intended to be used by
   584     given protocol buffer (datastore_pb.Entity). Not intended to be used by
   529     application developers.
   585     application developers.
   530 
   586 
   531     The Entity PB's key must be complete. If it isn't, an AssertionError is
   587     The Entity PB's key must be complete. If it isn't, an AssertionError is
   540       Entity
   596       Entity
   541     """
   597     """
   542     assert pb.key().path().element_size() > 0
   598     assert pb.key().path().element_size() > 0
   543 
   599 
   544     last_path = pb.key().path().element_list()[-1]
   600     last_path = pb.key().path().element_list()[-1]
   545     assert last_path.has_id() ^ last_path.has_name()
   601     if require_valid_key:
   546     if last_path.has_id():
   602       assert last_path.has_id() ^ last_path.has_name()
   547       assert last_path.id() != 0
   603       if last_path.has_id():
   548     else:
   604         assert last_path.id() != 0
   549       assert last_path.has_name()
   605       else:
   550       assert last_path.name()
   606         assert last_path.has_name()
       
   607         assert last_path.name()
   551 
   608 
   552     unindexed_properties = [p.name() for p in pb.raw_property_list()]
   609     unindexed_properties = [p.name() for p in pb.raw_property_list()]
   553 
   610 
   554     e = Entity(unicode(last_path.type().decode('utf-8')),
   611     e = Entity(unicode(last_path.type().decode('utf-8')),
   555                unindexed_properties=unindexed_properties)
   612                unindexed_properties=unindexed_properties)
   699   __filter_counter = 0
   756   __filter_counter = 0
   700 
   757 
   701   __inequality_prop = None
   758   __inequality_prop = None
   702   __inequality_count = 0
   759   __inequality_count = 0
   703 
   760 
   704   def __init__(self, kind, filters={}, _app=None, keys_only=False):
   761   def __init__(self, kind=None, filters={}, _app=None, keys_only=False,
       
   762                _namespace=None):
   705     """Constructor.
   763     """Constructor.
   706 
   764 
   707     Raises BadArgumentError if kind is not a string. Raises BadValueError or
   765     Raises BadArgumentError if kind is not a string. Raises BadValueError or
   708     BadFilterError if filters is not a dictionary of valid filters.
   766     BadFilterError if filters is not a dictionary of valid filters.
   709 
   767 
   712       # as an initial set of property filters. keys_only defaults to False.
   770       # as an initial set of property filters. keys_only defaults to False.
   713       kind: string
   771       kind: string
   714       filters: dict
   772       filters: dict
   715       keys_only: boolean
   773       keys_only: boolean
   716     """
   774     """
   717     datastore_types.ValidateString(kind, 'kind',
   775     if kind is not None:
   718                                    datastore_errors.BadArgumentError)
   776       datastore_types.ValidateString(kind, 'kind',
       
   777                                      datastore_errors.BadArgumentError)
   719 
   778 
   720     self.__kind = kind
   779     self.__kind = kind
   721     self.__orderings = []
   780     self.__orderings = []
   722     self.__filter_order = {}
   781     self.__filter_order = {}
   723     self.update(filters)
   782     self.update(filters)
   724 
   783 
   725     self.__app = datastore_types.ResolveAppId(_app)
   784     self.__app = datastore_types.ResolveAppIdNamespace(_app,
       
   785                                                        _namespace).to_encoded()
   726     self.__keys_only = keys_only
   786     self.__keys_only = keys_only
   727 
   787 
   728   def Order(self, *orderings):
   788   def Order(self, *orderings):
   729     """Specify how the query results should be sorted.
   789     """Specify how the query results should be sorted.
   730 
   790 
   792           raise datastore_errors.BadArgumentError(
   852           raise datastore_errors.BadArgumentError(
   793             'Order() expects Query.ASCENDING or DESCENDING; received %s' %
   853             'Order() expects Query.ASCENDING or DESCENDING; received %s' %
   794             str(direction))
   854             str(direction))
   795         direction = Query.ASCENDING
   855         direction = Query.ASCENDING
   796 
   856 
       
   857       if (self.__kind is None and
       
   858           (property != datastore_types._KEY_SPECIAL_PROPERTY or
       
   859           direction != Query.ASCENDING)):
       
   860         raise datastore_errors.BadArgumentError(
       
   861             'Only %s ascending orders are supported on kindless queries' %
       
   862             datastore_types._KEY_SPECIAL_PROPERTY)
       
   863 
   797       orderings[i] = (property, direction)
   864       orderings[i] = (property, direction)
   798 
   865 
   799     if (orderings and self.__inequality_prop and
   866     if (orderings and self.__inequality_prop and
   800         orderings[0][0] != self.__inequality_prop):
   867         orderings[0][0] != self.__inequality_prop):
   801       raise datastore_errors.BadArgumentError(
   868       raise datastore_errors.BadArgumentError(
   882       # an iterator that provides access to the query results
   949       # an iterator that provides access to the query results
   883       Iterator
   950       Iterator
   884     """
   951     """
   885     return self._Run()
   952     return self._Run()
   886 
   953 
   887   def _Run(self, limit=None, offset=None):
   954   def _Run(self, limit=None, offset=None,
       
   955            prefetch_count=None, next_count=None):
   888     """Runs this query, with an optional result limit and an optional offset.
   956     """Runs this query, with an optional result limit and an optional offset.
   889 
   957 
   890     Identical to Run, with the extra optional limit and offset parameters.
   958     Identical to Run, with the extra optional limit, offset, prefetch_count,
   891     limit and offset must both be integers >= 0.
   959     next_count parameters. These parameters must be integers >= 0.
   892 
   960 
   893     This is not intended to be used by application developers. Use Get()
   961     This is not intended to be used by application developers. Use Get()
   894     instead!
   962     instead!
   895     """
   963     """
   896     pb = self._ToPb(limit, offset)
   964     pb = self._ToPb(limit, offset, prefetch_count)
   897     result = datastore_pb.QueryResult()
   965     result = datastore_pb.QueryResult()
   898 
   966 
   899     try:
   967     try:
   900       apiproxy_stub_map.MakeSyncCall('datastore_v3', 'RunQuery', pb, result)
   968       apiproxy_stub_map.MakeSyncCall('datastore_v3', 'RunQuery', pb, result)
   901     except apiproxy_errors.ApplicationError, err:
   969     except apiproxy_errors.ApplicationError, err:
   905         yaml = datastore_index.IndexYamlForQuery(
   973         yaml = datastore_index.IndexYamlForQuery(
   906           *datastore_index.CompositeIndexForQuery(pb)[1:-1])
   974           *datastore_index.CompositeIndexForQuery(pb)[1:-1])
   907         raise datastore_errors.NeedIndexError(
   975         raise datastore_errors.NeedIndexError(
   908           str(exc) + '\nThis query needs this index:\n' + yaml)
   976           str(exc) + '\nThis query needs this index:\n' + yaml)
   909 
   977 
   910     return Iterator(result)
   978     return Iterator(result, batch_size=next_count)
   911 
   979 
   912   def Get(self, limit, offset=0):
   980   def Get(self, limit, offset=0):
   913     """Fetches and returns a maximum number of results from the query.
   981     """Fetches and returns a maximum number of results from the query.
   914 
   982 
   915     This method fetches and returns a list of resulting entities that matched
   983     This method fetches and returns a list of resulting entities that matched
   954     if not isinstance(offset, (int, long)) or offset < 0:
  1022     if not isinstance(offset, (int, long)) or offset < 0:
   955       raise datastore_errors.BadArgumentError(
  1023       raise datastore_errors.BadArgumentError(
   956           'Argument to Get named \'offset\' must be an int greater than or '
  1024           'Argument to Get named \'offset\' must be an int greater than or '
   957           'equal to 0; received %s (a %s)' % (offset, typename(offset)))
  1025           'equal to 0; received %s (a %s)' % (offset, typename(offset)))
   958 
  1026 
   959     return self._Run(limit, offset)._Get(limit)
  1027     return self._Run(limit=limit, offset=offset,
       
  1028                      prefetch_count=limit)._Get(limit)
   960 
  1029 
   961   def Count(self, limit=None):
  1030   def Count(self, limit=None):
   962     """Returns the number of entities that this query matches. The returned
  1031     """Returns the number of entities that this query matches. The returned
   963     count is cached; successive Count() calls will not re-scan the datastore
  1032     count is cached; successive Count() calls will not re-scan the datastore
   964     unless the query is changed.
  1033     unless the query is changed.
  1106         raise datastore_errors.BadFilterError(
  1175         raise datastore_errors.BadFilterError(
  1107           'Inequality operators (%s) must be on the same property as the '
  1176           'Inequality operators (%s) must be on the same property as the '
  1108           'first sort order, if any sort orders are supplied' %
  1177           'first sort order, if any sort orders are supplied' %
  1109           ', '.join(self.INEQUALITY_OPERATORS))
  1178           ', '.join(self.INEQUALITY_OPERATORS))
  1110 
  1179 
       
  1180     if (self.__kind is None and
       
  1181         property != datastore_types._KEY_SPECIAL_PROPERTY):
       
  1182       raise datastore_errors.BadFilterError(
       
  1183           'Only %s filters are allowed on kindless queries.' %
       
  1184           datastore_types._KEY_SPECIAL_PROPERTY)
       
  1185 
  1111     if property in datastore_types._SPECIAL_PROPERTIES:
  1186     if property in datastore_types._SPECIAL_PROPERTIES:
  1112       if property == datastore_types._KEY_SPECIAL_PROPERTY:
  1187       if property == datastore_types._KEY_SPECIAL_PROPERTY:
  1113         for value in values:
  1188         for value in values:
  1114           if not isinstance(value, Key):
  1189           if not isinstance(value, Key):
  1115             raise datastore_errors.BadFilterError(
  1190             raise datastore_errors.BadFilterError(
  1116               '%s filter value must be a Key; received %s (a %s)' %
  1191               '%s filter value must be a Key; received %s (a %s)' %
  1117               (datastore_types._KEY_SPECIAL_PROPERTY, value, typename(value)))
  1192               (datastore_types._KEY_SPECIAL_PROPERTY, value, typename(value)))
  1118 
  1193 
  1119     return match
  1194     return match
  1120 
  1195 
  1121   def _ToPb(self, limit=None, offset=None):
  1196   def _ToPb(self, limit=None, offset=None, count=None):
  1122     """Converts this Query to its protocol buffer representation. Not
  1197     """Converts this Query to its protocol buffer representation. Not
  1123     intended to be used by application developers. Enforced by hiding the
  1198     intended to be used by application developers. Enforced by hiding the
  1124     datastore_pb classes.
  1199     datastore_pb classes.
  1125 
  1200 
  1126     Args:
  1201     Args:
  1127       # an upper bound on the number of results returned by the query.
  1202       # an upper bound on the number of results returned by the query.
  1128       limit: int
  1203       limit: int
  1129       # number of results that match the query to skip.  limit is applied
  1204       # number of results that match the query to skip.  limit is applied
  1130       # after the offset is fulfilled
  1205       # after the offset is fulfilled
  1131       offset: int
  1206       offset: int
       
  1207       # the requested initial batch size
       
  1208       count: int
  1132 
  1209 
  1133     Returns:
  1210     Returns:
  1134       # the PB representation of this Query
  1211       # the PB representation of this Query
  1135       datastore_pb.Query
  1212       datastore_pb.Query
  1136 
  1213 
  1137     Raises:
  1214     Raises:
  1138       BadRequestError if called inside a transaction and the query does not
  1215       BadRequestError if called inside a transaction and the query does not
  1139       include an ancestor.
  1216       include an ancestor.
  1140     """
  1217     """
       
  1218 
  1141     if not self.__ancestor and _CurrentTransactionKey():
  1219     if not self.__ancestor and _CurrentTransactionKey():
  1142       raise datastore_errors.BadRequestError(
  1220       raise datastore_errors.BadRequestError(
  1143         'Only ancestor queries are allowed inside transactions.')
  1221         'Only ancestor queries are allowed inside transactions.')
  1144 
  1222 
  1145     pb = datastore_pb.Query()
  1223     pb = datastore_pb.Query()
  1146     _MaybeSetupTransaction(pb, [self.__ancestor])
  1224     _MaybeSetupTransaction(pb, [self.__ancestor])
  1147 
  1225 
  1148     pb.set_kind(self.__kind.encode('utf-8'))
  1226     if self.__kind is not None:
       
  1227       pb.set_kind(self.__kind.encode('utf-8'))
  1149     pb.set_keys_only(bool(self.__keys_only))
  1228     pb.set_keys_only(bool(self.__keys_only))
  1150     if self.__app:
  1229     if self.__app:
  1151       pb.set_app(self.__app.encode('utf-8'))
  1230       pb.set_app(self.__app.encode('utf-8'))
  1152     if limit is not None:
  1231     if limit is not None:
  1153       pb.set_limit(limit)
  1232       pb.set_limit(limit)
  1154     if offset is not None:
  1233     if offset is not None:
  1155       pb.set_offset(offset)
  1234       pb.set_offset(offset)
       
  1235     if count is not None:
       
  1236       pb.set_count(count)
  1156     if self.__ancestor:
  1237     if self.__ancestor:
  1157       pb.mutable_ancestor().CopyFrom(self.__ancestor._Key__reference)
  1238       pb.mutable_ancestor().CopyFrom(self.__ancestor._Key__reference)
  1158 
  1239 
  1159     if ((self.__hint == self.ORDER_FIRST and self.__orderings) or
  1240     if ((self.__hint == self.ORDER_FIRST and self.__orderings) or
  1160         (self.__hint == self.ANCESTOR_FIRST and self.__ancestor) or
  1241         (self.__hint == self.ANCESTOR_FIRST and self.__ancestor) or
  1189       order = pb.add_order()
  1270       order = pb.add_order()
  1190       order.set_property(property.encode('utf-8'))
  1271       order.set_property(property.encode('utf-8'))
  1191       order.set_direction(direction)
  1272       order.set_direction(direction)
  1192 
  1273 
  1193     return pb
  1274     return pb
       
  1275 
       
  1276 
       
  1277 def AllocateIds(model_key, size):
       
  1278   """Allocates a range of IDs of size for the key defined by model_key
       
  1279 
       
  1280   Allocates a range of IDs in the datastore such that those IDs will not
       
  1281   be automatically assigned to new entities. You can only allocate IDs
       
  1282   for model keys from your app. If there is an error, raises a subclass of
       
  1283   datastore_errors.Error.
       
  1284 
       
  1285   Args:
       
  1286     model_key: Key or string to serve as a model specifying the ID sequence
       
  1287                in which to allocate IDs
       
  1288 
       
  1289   Returns:
       
  1290     (start, end) of the allocated range, inclusive.
       
  1291   """
       
  1292   keys, multiple = NormalizeAndTypeCheckKeys(model_key)
       
  1293 
       
  1294   if len(keys) > 1:
       
  1295     raise datastore_errors.BadArgumentError(
       
  1296         'Cannot allocate IDs for more than one model key at a time')
       
  1297 
       
  1298   if size > _MAX_ID_BATCH_SIZE:
       
  1299     raise datastore_errors.BadArgumentError(
       
  1300         'Cannot allocate more than %s ids at a time' % _MAX_ID_BATCH_SIZE)
       
  1301 
       
  1302   req = datastore_pb.AllocateIdsRequest()
       
  1303   req.mutable_model_key().CopyFrom(keys[0]._Key__reference)
       
  1304   req.set_size(size)
       
  1305 
       
  1306   resp = datastore_pb.AllocateIdsResponse()
       
  1307   try:
       
  1308     apiproxy_stub_map.MakeSyncCall('datastore_v3', 'AllocateIds', req, resp)
       
  1309   except apiproxy_errors.ApplicationError, err:
       
  1310     raise _ToDatastoreError(err)
       
  1311 
       
  1312   return resp.start(), resp.end()
  1194 
  1313 
  1195 
  1314 
  1196 class MultiQuery(Query):
  1315 class MultiQuery(Query):
  1197   """Class representing a query which requires multiple datastore queries.
  1316   """Class representing a query which requires multiple datastore queries.
  1198 
  1317 
  1515 
  1634 
  1516   > it = Query('Person').Run()
  1635   > it = Query('Person').Run()
  1517   > for person in it:
  1636   > for person in it:
  1518   >   print 'Hi, %s!' % person['name']
  1637   >   print 'Hi, %s!' % person['name']
  1519   """
  1638   """
  1520   def __init__(self, query_result_pb):
  1639   def __init__(self, query_result_pb, batch_size=None):
  1521     self.__cursor = query_result_pb.cursor()
  1640     self.__cursor = query_result_pb.cursor()
  1522     self.__keys_only = query_result_pb.keys_only()
  1641     self.__keys_only = query_result_pb.keys_only()
       
  1642     self.__batch_size = batch_size
  1523     self.__buffer = self._ProcessQueryResult(query_result_pb)
  1643     self.__buffer = self._ProcessQueryResult(query_result_pb)
  1524 
  1644 
  1525   def _Get(self, count):
  1645   def _Get(self, count):
  1526     """Gets the next count result(s) of the query.
  1646     """Gets the next count result(s) of the query.
  1527 
  1647 
  1545 
  1665 
  1546     Returns:
  1666     Returns:
  1547       # a list of entities or keys
  1667       # a list of entities or keys
  1548       [Entity or Key, ...]
  1668       [Entity or Key, ...]
  1549     """
  1669     """
  1550     entityList = self._Next(count)
  1670     entity_list = self._Next(count)
  1551     while len(entityList) < count and self.__more_results:
  1671     while len(entity_list) < count and self.__more_results:
  1552       next_results = self._Next(count - len(entityList))
  1672       next_results = self._Next(count - len(entity_list), self.__batch_size)
  1553       if not next_results:
  1673       if not next_results:
  1554         break
  1674         break
  1555       entityList += next_results
  1675       entity_list += next_results
  1556     return entityList;
  1676     return entity_list;
  1557 
  1677 
  1558   def _Next(self, count):
  1678   def _Next(self, count=None):
  1559     """Returns the next result(s) of the query.
  1679     """Returns the next batch of results.
  1560 
  1680 
  1561     Not intended to be used by application developers. Use the python
  1681     Not intended to be used by application developers. Use the python
  1562     iterator protocol instead.
  1682     iterator protocol instead.
  1563 
  1683 
  1564     This method returns the next entities or keys from the list of matching
  1684     This method returns the next entities or keys from the list of matching
  1565     results. If the query specified a sort order, results are returned in that
  1685     results. If the query specified a sort order, results are returned in that
  1566     order. Otherwise, the order is undefined.
  1686     order. Otherwise, the order is undefined.
  1567 
  1687 
  1568     The argument, count, specifies the number of results to return. However, the
  1688     The optional argument, count, specifies the number of results to return.
  1569     length of the returned list may be smaller than count. This is the case if
  1689     However, the length of the returned list may be smaller than count. This is
  1570     count is greater than the number of remaining results or the size of the
  1690     the case if count is greater than the number of remaining results or the
  1571     remaining results exciteds the RPC buffer limit. Use _Get to insure all
  1691     size of the remaining results exceeds the RPC buffer limit. Use _Get to
  1572     possible entities are retrieved.
  1692     insure all possible entities are retrieved.
       
  1693 
       
  1694     If the count is omitted, the datastore backend decides how many entities to
       
  1695     send.
  1573 
  1696 
  1574     There is an internal buffer for use with the next() method. If this buffer
  1697     There is an internal buffer for use with the next() method. If this buffer
  1575     is not empty, up to 'count' values are removed from this buffer and
  1698     is not empty, up to 'count' values are removed from this buffer and
  1576     returned. It's best not to mix _Next() and next().
  1699     returned. It's best not to mix _Next() and next().
  1577 
  1700 
  1578     The results are always returned as a list. If there are no results left,
  1701     The results are always returned as a list. If there are no results left,
  1579     an empty list is returned.
  1702     an empty list is returned.
  1580 
  1703 
  1581     Args:
  1704     Args:
  1582       # the number of results to return; must be >= 1
  1705       # the number of results to return; must be >= 1
  1583       count: int or long
  1706       count: int or long or None
  1584 
  1707 
  1585     Returns:
  1708     Returns:
  1586       # a list of entities or keys
  1709       # a list of entities or keys
  1587       [Entity or Key, ...]
  1710       [Entity or Key, ...]
  1588     """
  1711     """
  1589     if not isinstance(count, (int, long)) or count <= 0:
  1712     if count is not None and (not isinstance(count, (int, long)) or count <= 0):
  1590       raise datastore_errors.BadArgumentError(
  1713       raise datastore_errors.BadArgumentError(
  1591         'Argument to _Next must be an int greater than 0; received %s (a %s)' %
  1714         'Argument to _Next must be an int greater than 0; received %s (a %s)' %
  1592         (count, typename(count)))
  1715         (count, typename(count)))
  1593 
  1716 
  1594     if self.__buffer:
  1717     if self.__buffer:
  1595       if count <= len(self.__buffer):
  1718       if count is None:
       
  1719         entity_list = self.__buffer
       
  1720         self.__buffer = []
       
  1721         return entity_list
       
  1722       elif count <= len(self.__buffer):
  1596         entity_list = self.__buffer[:count]
  1723         entity_list = self.__buffer[:count]
  1597         del self.__buffer[:count]
  1724         del self.__buffer[:count]
  1598         return entity_list
  1725         return entity_list
  1599       else:
  1726       else:
  1600         entity_list = self.__buffer
  1727         entity_list = self.__buffer
  1601         self.__buffer = []
  1728         self.__buffer = []
  1602         count -= len(entity_list)
  1729         count -= len(entity_list)
  1603     else:
  1730     else:
  1604         entity_list=[]
  1731         entity_list = []
       
  1732 
  1605 
  1733 
  1606     if not self.__more_results:
  1734     if not self.__more_results:
  1607       return entity_list
  1735       return entity_list
  1608 
  1736 
  1609     req = datastore_pb.NextRequest()
  1737     req = datastore_pb.NextRequest()
  1610     req.set_count(count)
  1738     if count is not None:
       
  1739       req.set_count(count)
  1611     req.mutable_cursor().CopyFrom(self.__cursor)
  1740     req.mutable_cursor().CopyFrom(self.__cursor)
  1612     result = datastore_pb.QueryResult()
  1741     result = datastore_pb.QueryResult()
  1613     try:
  1742     try:
  1614       apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Next', req, result)
  1743       apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Next', req, result)
  1615     except apiproxy_errors.ApplicationError, err:
  1744     except apiproxy_errors.ApplicationError, err:
  1640     if self.__keys_only:
  1769     if self.__keys_only:
  1641       return [Key._FromPb(e.key()) for e in result.result_list()]
  1770       return [Key._FromPb(e.key()) for e in result.result_list()]
  1642     else:
  1771     else:
  1643       return [Entity._FromPb(e) for e in result.result_list()]
  1772       return [Entity._FromPb(e) for e in result.result_list()]
  1644 
  1773 
  1645   _BUFFER_SIZE = 20
       
  1646 
       
  1647   def next(self):
  1774   def next(self):
  1648     if not self.__buffer:
  1775     if not self.__buffer:
  1649       self.__buffer = self._Next(self._BUFFER_SIZE)
  1776       self.__buffer = self._Next(self.__batch_size)
  1650     try:
  1777     try:
  1651       return self.__buffer.pop(0)
  1778       return self.__buffer.pop(0)
  1652     except IndexError:
  1779     except IndexError:
  1653       raise StopIteration
  1780       raise StopIteration
  1654 
  1781 
  1655   def __iter__(self): return self
  1782   def __iter__(self): return self
  1656 
  1783 
  1657 class _Transaction(object):
  1784 class _Transaction(object):
  1658   """Encapsulates a transaction currently in progress.
  1785   """Encapsulates a transaction currently in progress.
  1659 
  1786 
  1660   If we've sent a BeginTransaction call, then handle will be a
       
  1661   datastore_pb.Transaction that holds the transaction handle.
       
  1662 
       
  1663   If we know the entity group for this transaction, it's stored in the
  1787   If we know the entity group for this transaction, it's stored in the
  1664   entity_group attribute, which is set by RecordModifiedKeys().
  1788   entity_group attribute, which is set by RunInTransaction().
  1665 
  1789 
  1666   modified_keys is a set containing the Keys of all entities modified (ie put
  1790   modified_keys is a set containing the Keys of all entities modified (ie put
  1667   or deleted) in this transaction. If an entity is modified more than once, a
  1791   or deleted) in this transaction. If an entity is modified more than once, a
  1668   BadRequestError is raised.
  1792   BadRequestError is raised.
  1669   """
  1793   """
  1670   def __init__(self):
  1794   def __init__(self, handle):
  1671     """Initializes modified_keys to the empty set."""
  1795     """Initializes the transaction.
  1672     self.handle = None
  1796 
       
  1797     Args:
       
  1798       handle: a datastore_pb.Transaction returned by a BeginTransaction call
       
  1799     """
       
  1800     assert isinstance(handle, datastore_pb.Transaction)
       
  1801     explanation = []
       
  1802     assert handle.IsInitialized(explanation), explanation
       
  1803 
       
  1804     self.handle = handle
  1673     self.entity_group = None
  1805     self.entity_group = None
  1674     self.modified_keys = None
  1806     self.modified_keys = None
  1675     self.modified_keys = set()
  1807     self.modified_keys = set()
  1676 
       
  1677   def RecordModifiedKeys(self, keys, error_on_repeat=True):
       
  1678     """Updates the modified keys seen so far.
       
  1679 
       
  1680     If error_on_repeat is True and any of the given keys have already been
       
  1681     modified, raises BadRequestError.
       
  1682 
       
  1683     Args:
       
  1684       keys: sequence of Keys
       
  1685     """
       
  1686     keys, _ = NormalizeAndTypeCheckKeys(keys)
       
  1687     keys = set(keys)
       
  1688 
       
  1689     if error_on_repeat:
       
  1690       already_modified = self.modified_keys.intersection(keys)
       
  1691       if already_modified:
       
  1692         raise datastore_errors.BadRequestError(
       
  1693           "Can't update entity more than once in a transaction: %r" %
       
  1694           already_modified.pop())
       
  1695 
       
  1696     self.modified_keys.update(keys)
       
  1697 
  1808 
  1698 
  1809 
  1699 def RunInTransaction(function, *args, **kwargs):
  1810 def RunInTransaction(function, *args, **kwargs):
  1700   """Runs a function inside a datastore transaction.
  1811   """Runs a function inside a datastore transaction.
  1701 
  1812 
  1797 
  1908 
  1798   tx_key = None
  1909   tx_key = None
  1799 
  1910 
  1800   try:
  1911   try:
  1801     tx_key = _NewTransactionKey()
  1912     tx_key = _NewTransactionKey()
  1802     tx = _Transaction()
       
  1803     _txes[tx_key] = tx
       
  1804 
  1913 
  1805     for i in range(0, retries + 1):
  1914     for i in range(0, retries + 1):
  1806       tx.modified_keys.clear()
  1915       handle = datastore_pb.Transaction()
       
  1916       try:
       
  1917         apiproxy_stub_map.MakeSyncCall('datastore_v3', 'BeginTransaction',
       
  1918                                        api_base_pb.VoidProto(), handle)
       
  1919       except apiproxy_errors.ApplicationError, err:
       
  1920         raise _ToDatastoreError(err)
       
  1921 
       
  1922       tx = _Transaction(handle)
       
  1923       _txes[tx_key] = tx
  1807 
  1924 
  1808       try:
  1925       try:
  1809         result = function(*args, **kwargs)
  1926         result = function(*args, **kwargs)
  1810       except:
  1927       except:
  1811         original_exception = sys.exc_info()
  1928         original_exception = sys.exc_info()
  1812 
  1929 
  1813         if tx.handle:
  1930         try:
  1814           try:
  1931           resp = api_base_pb.VoidProto()
  1815             resp = api_base_pb.VoidProto()
  1932           apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Rollback',
  1816             apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Rollback',
  1933                                          tx.handle, resp)
  1817                                            tx.handle, resp)
  1934         except:
  1818           except:
  1935           exc_info = sys.exc_info()
  1819             exc_info = sys.exc_info()
  1936           logging.info('Exception sending Rollback:\n' +
  1820             logging.info('Exception sending Rollback:\n' +
  1937                        ''.join(traceback.format_exception(*exc_info)))
  1821                          ''.join(traceback.format_exception(*exc_info)))
       
  1822 
  1938 
  1823         type, value, trace = original_exception
  1939         type, value, trace = original_exception
  1824         if type is datastore_errors.Rollback:
  1940         if type is datastore_errors.Rollback:
  1825           return
  1941           return
  1826         else:
  1942         else:
  1827           raise type, value, trace
  1943           raise type, value, trace
  1828 
  1944 
  1829       if tx.handle:
  1945       try:
  1830         try:
  1946         resp = datastore_pb.CommitResponse()
  1831           resp = datastore_pb.CommitResponse()
  1947         apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Commit',
  1832           apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Commit',
  1948                                        tx.handle, resp)
  1833                                          tx.handle, resp)
  1949       except apiproxy_errors.ApplicationError, err:
  1834         except apiproxy_errors.ApplicationError, err:
  1950         if (err.application_error ==
  1835           if (err.application_error ==
  1951             datastore_pb.Error.CONCURRENT_TRANSACTION):
  1836               datastore_pb.Error.CONCURRENT_TRANSACTION):
  1952           logging.warning('Transaction collision for entity group with '
  1837             logging.warning('Transaction collision for entity group with '
  1953                           'key %r. Retrying...', tx.entity_group)
  1838                             'key %r. Retrying...', tx.entity_group)
  1954           tx.handle = None
  1839             tx.handle = None
  1955           tx.entity_group = None
  1840             tx.entity_group = None
  1956           continue
  1841             continue
  1957         else:
  1842           else:
  1958           raise _ToDatastoreError(err)
  1843             raise _ToDatastoreError(err)
       
  1844 
  1959 
  1845       return result
  1960       return result
  1846 
  1961 
  1847     raise datastore_errors.TransactionFailedError(
  1962     raise datastore_errors.TransactionFailedError(
  1848       'The transaction could not be committed. Please try again.')
  1963       'The transaction could not be committed. Please try again.')
  1852       del _txes[tx_key]
  1967       del _txes[tx_key]
  1853     del tx_key
  1968     del tx_key
  1854 
  1969 
  1855 
  1970 
  1856 def _MaybeSetupTransaction(request, keys):
  1971 def _MaybeSetupTransaction(request, keys):
  1857   """Begins a transaction, if necessary, and populates it in the request.
  1972   """If we're in a transaction, validates and populates it in the request.
  1858 
  1973 
  1859   If we're currently inside a transaction, this records the entity group,
  1974   If we're currently inside a transaction, this records the entity group,
  1860   checks that the keys are all in that entity group, creates the transaction
  1975   checks that the keys are all in that entity group, and populates the
  1861   PB, and sends the BeginTransaction. It then populates the transaction handle
  1976   transaction handle in the request.
  1862   in the request.
       
  1863 
  1977 
  1864   Raises BadRequestError if the entity has a different entity group than the
  1978   Raises BadRequestError if the entity has a different entity group than the
  1865   current transaction.
  1979   current transaction.
  1866 
  1980 
  1867   Args:
  1981   Args:
  1870 
  1984 
  1871   Returns:
  1985   Returns:
  1872     _Transaction if we're inside a transaction, otherwise None
  1986     _Transaction if we're inside a transaction, otherwise None
  1873   """
  1987   """
  1874   assert isinstance(request, (datastore_pb.GetRequest, datastore_pb.PutRequest,
  1988   assert isinstance(request, (datastore_pb.GetRequest, datastore_pb.PutRequest,
  1875                               datastore_pb.DeleteRequest, datastore_pb.Query))
  1989                               datastore_pb.DeleteRequest, datastore_pb.Query,
       
  1990                               taskqueue_service_pb.TaskQueueAddRequest,
       
  1991                               )), request.__class__
  1876   tx_key = None
  1992   tx_key = None
  1877 
  1993 
  1878   try:
  1994   try:
  1879     tx_key = _CurrentTransactionKey()
  1995     tx_key = _CurrentTransactionKey()
  1880     if tx_key:
  1996     if tx_key:
  1881       tx = _txes[tx_key]
  1997       tx = _txes[tx_key]
  1882 
  1998 
  1883       groups = [k.entity_group() for k in keys]
  1999       groups = [k.entity_group() for k in keys]
  1884       if tx.entity_group:
  2000       if tx.entity_group:
  1885         expected_group = tx.entity_group
  2001         expected_group = tx.entity_group
       
  2002       elif groups:
       
  2003         expected_group = groups[0]
  1886       else:
  2004       else:
  1887         expected_group = groups[0]
  2005         expected_group = None
  1888 
  2006 
  1889       for group in groups:
  2007       for group in groups:
  1890         if (group != expected_group or
  2008         if (group != expected_group or
  1891 
  2009 
  1892 
  2010 
  1899           raise _DifferentEntityGroupError(expected_group, group)
  2017           raise _DifferentEntityGroupError(expected_group, group)
  1900 
  2018 
  1901         if not tx.entity_group and group.has_id_or_name():
  2019         if not tx.entity_group and group.has_id_or_name():
  1902           tx.entity_group = group
  2020           tx.entity_group = group
  1903 
  2021 
  1904       if not tx.handle:
  2022       assert tx.handle.IsInitialized()
  1905         tx.handle = datastore_pb.Transaction()
       
  1906         req = api_base_pb.VoidProto()
       
  1907         apiproxy_stub_map.MakeSyncCall('datastore_v3', 'BeginTransaction', req,
       
  1908                                        tx.handle)
       
  1909 
       
  1910       request.mutable_transaction().CopyFrom(tx.handle)
  2023       request.mutable_transaction().CopyFrom(tx.handle)
  1911 
  2024 
  1912       return tx
  2025       return tx
  1913 
  2026 
  1914   finally:
  2027   finally: