47 from google.appengine.datastore import datastore_index |
47 from google.appengine.datastore import datastore_index |
48 from google.appengine.datastore import datastore_pb |
48 from google.appengine.datastore import datastore_pb |
49 from google.appengine.runtime import apiproxy_errors |
49 from google.appengine.runtime import apiproxy_errors |
50 from google.appengine.datastore import entity_pb |
50 from google.appengine.datastore import entity_pb |
51 |
51 |
|
52 try: |
|
53 from google.appengine.api.labs.taskqueue import taskqueue_service_pb |
|
54 except ImportError: |
|
55 from google.appengine.api.taskqueue import taskqueue_service_pb |
|
56 |
52 MAX_ALLOWABLE_QUERIES = 30 |
57 MAX_ALLOWABLE_QUERIES = 30 |
53 |
58 |
54 DEFAULT_TRANSACTION_RETRIES = 3 |
59 DEFAULT_TRANSACTION_RETRIES = 3 |
55 |
60 |
56 _MAX_INDEXED_PROPERTIES = 5000 |
61 _MAX_INDEXED_PROPERTIES = 5000 |
|
62 |
|
63 _MAX_ID_BATCH_SIZE = 1000 * 1000 * 1000 |
57 |
64 |
58 Key = datastore_types.Key |
65 Key = datastore_types.Key |
59 typename = datastore_types.typename |
66 typename = datastore_types.typename |
60 |
67 |
61 _txes = {} |
68 _txes = {} |
145 |
152 |
146 if multiple and not entities: |
153 if multiple and not entities: |
147 return [] |
154 return [] |
148 |
155 |
149 for entity in entities: |
156 for entity in entities: |
150 if not entity.kind() or not entity.app(): |
157 if not entity.kind() or not entity.app_id_namespace(): |
151 raise datastore_errors.BadRequestError( |
158 raise datastore_errors.BadRequestError( |
152 'App and kind must not be empty, in entity: %s' % entity) |
159 'App and kind must not be empty, in entity: %s' % entity) |
153 |
160 |
154 req = datastore_pb.PutRequest() |
161 req = datastore_pb.PutRequest() |
155 req.entity_list().extend([e._ToPb() for e in entities]) |
162 req.entity_list().extend([e._ToPb() for e in entities]) |
156 |
163 |
157 keys = [e.key() for e in entities] |
164 keys = [e.key() for e in entities] |
158 tx = _MaybeSetupTransaction(req, keys) |
165 tx = _MaybeSetupTransaction(req, keys) |
159 if tx: |
|
160 tx.RecordModifiedKeys([k for k in keys if k.has_id_or_name()]) |
|
161 |
166 |
162 resp = datastore_pb.PutResponse() |
167 resp = datastore_pb.PutResponse() |
163 try: |
168 try: |
164 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Put', req, resp) |
169 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Put', req, resp) |
165 except apiproxy_errors.ApplicationError, err: |
170 except apiproxy_errors.ApplicationError, err: |
273 """A datastore entity. |
275 """A datastore entity. |
274 |
276 |
275 Includes read-only accessors for app id, kind, and primary key. Also |
277 Includes read-only accessors for app id, kind, and primary key. Also |
276 provides dictionary-style access to properties. |
278 provides dictionary-style access to properties. |
277 """ |
279 """ |
278 def __init__(self, kind, parent=None, _app=None, name=None, |
280 def __init__(self, kind, parent=None, _app=None, name=None, id=None, |
279 unindexed_properties=[]): |
281 unindexed_properties=[], _namespace=None): |
280 """Constructor. Takes the kind and transaction root, which cannot be |
282 """Constructor. Takes the kind and transaction root, which cannot be |
281 changed after the entity is constructed, and an optional parent. Raises |
283 changed after the entity is constructed, and an optional parent. Raises |
282 BadArgumentError or BadKeyError if kind is invalid or parent is not an |
284 BadArgumentError or BadKeyError if kind is invalid or parent is not an |
283 existing Entity or Key in the datastore. |
285 existing Entity or Key in the datastore. |
284 |
286 |
287 kind: string |
289 kind: string |
288 # if provided, this entity's parent. Its key must be complete. |
290 # if provided, this entity's parent. Its key must be complete. |
289 parent: Entity or Key |
291 parent: Entity or Key |
290 # if provided, this entity's name. |
292 # if provided, this entity's name. |
291 name: string |
293 name: string |
|
294 # if provided, this entity's id. |
|
295 id: integer |
292 # if provided, a sequence of property names that should not be indexed |
296 # if provided, a sequence of property names that should not be indexed |
293 # by the built-in single property indices. |
297 # by the built-in single property indices. |
294 unindexed_properties: list or tuple of strings |
298 unindexed_properties: list or tuple of strings |
295 """ |
299 """ |
296 ref = entity_pb.Reference() |
300 ref = entity_pb.Reference() |
297 _app = datastore_types.ResolveAppId(_app) |
301 _app_namespace = datastore_types.ResolveAppIdNamespace(_app, _namespace) |
298 ref.set_app(_app) |
302 ref.set_app(_app_namespace.to_encoded()) |
299 |
303 |
300 datastore_types.ValidateString(kind, 'kind', |
304 datastore_types.ValidateString(kind, 'kind', |
301 datastore_errors.BadArgumentError) |
305 datastore_errors.BadArgumentError) |
302 |
|
303 if parent is not None: |
306 if parent is not None: |
304 parent = _GetCompleteKeyOrError(parent) |
307 parent = _GetCompleteKeyOrError(parent) |
305 if _app != parent.app(): |
308 if _app_namespace != parent.app_id_namespace(): |
306 raise datastore_errors.BadArgumentError( |
309 raise datastore_errors.BadArgumentError( |
307 "_app %s doesn't match parent's app %s" % (_app, parent.app())) |
310 " %s doesn't match parent's app_namespace %s" % |
|
311 (_app_namespace, parent.app_id_namespace())) |
308 ref.CopyFrom(parent._Key__reference) |
312 ref.CopyFrom(parent._Key__reference) |
309 |
313 |
310 last_path = ref.mutable_path().add_element() |
314 last_path = ref.mutable_path().add_element() |
311 last_path.set_type(kind.encode('utf-8')) |
315 last_path.set_type(kind.encode('utf-8')) |
312 |
316 |
|
317 if name is not None and id is not None: |
|
318 raise datastore_errors.BadArgumentError( |
|
319 "Cannot set both name and id on an Entity") |
|
320 |
313 if name is not None: |
321 if name is not None: |
314 datastore_types.ValidateString(name, 'name') |
322 datastore_types.ValidateString(name, 'name') |
315 if name[0] in string.digits: |
|
316 raise datastore_errors.BadValueError('name cannot begin with a digit') |
|
317 last_path.set_name(name.encode('utf-8')) |
323 last_path.set_name(name.encode('utf-8')) |
|
324 |
|
325 if id is not None: |
|
326 datastore_types.ValidateInteger(id, 'id') |
|
327 last_path.set_id(id) |
318 |
328 |
319 unindexed_properties, multiple = NormalizeAndTypeCheck(unindexed_properties, basestring) |
329 unindexed_properties, multiple = NormalizeAndTypeCheck(unindexed_properties, basestring) |
320 if not multiple: |
330 if not multiple: |
321 raise datastore_errors.BadArgumentError( |
331 raise datastore_errors.BadArgumentError( |
322 'unindexed_properties must be a sequence; received %s (a %s).' % |
332 'unindexed_properties must be a sequence; received %s (a %s).' % |
327 |
337 |
328 self.__key = Key._FromPb(ref) |
338 self.__key = Key._FromPb(ref) |
329 |
339 |
330 def app(self): |
340 def app(self): |
331 """Returns the name of the application that created this entity, a |
341 """Returns the name of the application that created this entity, a |
332 string. |
342 string or None if not set. |
333 """ |
343 """ |
334 return self.__key.app() |
344 return self.__key.app() |
|
345 |
|
346 def namespace(self): |
|
347 """Returns the namespace of this entity, a string or None. |
|
348 """ |
|
349 return self.__key.namespace() |
|
350 |
|
351 def app_id_namespace(self): |
|
352 """Returns the AppIdNamespace of this entity or None if not set. |
|
353 """ |
|
354 return self.__key.app_id_namespace() |
335 |
355 |
336 def kind(self): |
356 def kind(self): |
337 """Returns this entity's kind, a string. |
357 """Returns this entity's kind, a string. |
338 """ |
358 """ |
339 return self.__key.kind() |
359 return self.__key.kind() |
|
360 |
|
361 def is_saved(self): |
|
362 """Returns if this entity has been saved to the datastore |
|
363 """ |
|
364 last_path = self.__key._Key__reference.path().element_list()[-1] |
|
365 return ((last_path.has_name() ^ last_path.has_id()) and |
|
366 self.__key.has_id_or_name()) |
340 |
367 |
341 def key(self): |
368 def key(self): |
342 """Returns this entity's primary key, a Key instance. |
369 """Returns this entity's primary key, a Key instance. |
343 """ |
370 """ |
344 return self.__key |
371 return self.__key |
481 else: |
508 else: |
482 xml.append(saxutils.escape(unicode(val))) |
509 xml.append(saxutils.escape(unicode(val))) |
483 |
510 |
484 return xml |
511 return xml |
485 |
512 |
486 def _ToPb(self): |
513 def ToPb(self): |
|
514 """Converts this Entity to its protocol buffer representation. |
|
515 |
|
516 Returns: |
|
517 entity_pb.Entity |
|
518 """ |
|
519 return self._ToPb(False) |
|
520 |
|
521 def _ToPb(self, mark_key_as_saved=True): |
487 """Converts this Entity to its protocol buffer representation. Not |
522 """Converts this Entity to its protocol buffer representation. Not |
488 intended to be used by application developers. |
523 intended to be used by application developers. |
489 |
524 |
490 Returns: |
525 Returns: |
491 entity_pb.Entity |
526 entity_pb.Entity |
492 """ |
527 """ |
493 |
528 |
494 pb = entity_pb.EntityProto() |
529 pb = entity_pb.EntityProto() |
495 pb.mutable_key().CopyFrom(self.key()._ToPb()) |
530 pb.mutable_key().CopyFrom(self.key()._ToPb()) |
|
531 last_path = pb.key().path().element_list()[-1] |
|
532 if mark_key_as_saved and last_path.has_name() and last_path.has_id(): |
|
533 last_path.clear_id() |
496 |
534 |
497 group = pb.mutable_entity_group() |
535 group = pb.mutable_entity_group() |
498 if self.__key.has_id_or_name(): |
536 if self.__key.has_id_or_name(): |
499 root = pb.key().path().element(0) |
537 root = pb.key().path().element(0) |
500 group.add_element().CopyFrom(root) |
538 group.add_element().CopyFrom(root) |
521 'Too many indexed properties for entity %r.' % self.key()) |
559 'Too many indexed properties for entity %r.' % self.key()) |
522 |
560 |
523 return pb |
561 return pb |
524 |
562 |
525 @staticmethod |
563 @staticmethod |
526 def _FromPb(pb): |
564 def FromPb(pb): |
|
565 """Static factory method. Returns the Entity representation of the |
|
566 given protocol buffer (datastore_pb.Entity). |
|
567 |
|
568 Args: |
|
569 pb: datastore_pb.Entity or str encoding of a datastore_pb.Entity |
|
570 |
|
571 Returns: |
|
572 Entity: the Entity representation of pb |
|
573 """ |
|
574 if isinstance(pb, str): |
|
575 real_pb = entity_pb.EntityProto() |
|
576 real_pb.ParseFromString(pb) |
|
577 pb = real_pb |
|
578 |
|
579 return Entity._FromPb(pb, require_valid_key=False) |
|
580 |
|
581 @staticmethod |
|
582 def _FromPb(pb, require_valid_key=True): |
527 """Static factory method. Returns the Entity representation of the |
583 """Static factory method. Returns the Entity representation of the |
528 given protocol buffer (datastore_pb.Entity). Not intended to be used by |
584 given protocol buffer (datastore_pb.Entity). Not intended to be used by |
529 application developers. |
585 application developers. |
530 |
586 |
531 The Entity PB's key must be complete. If it isn't, an AssertionError is |
587 The Entity PB's key must be complete. If it isn't, an AssertionError is |
540 Entity |
596 Entity |
541 """ |
597 """ |
542 assert pb.key().path().element_size() > 0 |
598 assert pb.key().path().element_size() > 0 |
543 |
599 |
544 last_path = pb.key().path().element_list()[-1] |
600 last_path = pb.key().path().element_list()[-1] |
545 assert last_path.has_id() ^ last_path.has_name() |
601 if require_valid_key: |
546 if last_path.has_id(): |
602 assert last_path.has_id() ^ last_path.has_name() |
547 assert last_path.id() != 0 |
603 if last_path.has_id(): |
548 else: |
604 assert last_path.id() != 0 |
549 assert last_path.has_name() |
605 else: |
550 assert last_path.name() |
606 assert last_path.has_name() |
|
607 assert last_path.name() |
551 |
608 |
552 unindexed_properties = [p.name() for p in pb.raw_property_list()] |
609 unindexed_properties = [p.name() for p in pb.raw_property_list()] |
553 |
610 |
554 e = Entity(unicode(last_path.type().decode('utf-8')), |
611 e = Entity(unicode(last_path.type().decode('utf-8')), |
555 unindexed_properties=unindexed_properties) |
612 unindexed_properties=unindexed_properties) |
712 # as an initial set of property filters. keys_only defaults to False. |
770 # as an initial set of property filters. keys_only defaults to False. |
713 kind: string |
771 kind: string |
714 filters: dict |
772 filters: dict |
715 keys_only: boolean |
773 keys_only: boolean |
716 """ |
774 """ |
717 datastore_types.ValidateString(kind, 'kind', |
775 if kind is not None: |
718 datastore_errors.BadArgumentError) |
776 datastore_types.ValidateString(kind, 'kind', |
|
777 datastore_errors.BadArgumentError) |
719 |
778 |
720 self.__kind = kind |
779 self.__kind = kind |
721 self.__orderings = [] |
780 self.__orderings = [] |
722 self.__filter_order = {} |
781 self.__filter_order = {} |
723 self.update(filters) |
782 self.update(filters) |
724 |
783 |
725 self.__app = datastore_types.ResolveAppId(_app) |
784 self.__app = datastore_types.ResolveAppIdNamespace(_app, |
|
785 _namespace).to_encoded() |
726 self.__keys_only = keys_only |
786 self.__keys_only = keys_only |
727 |
787 |
728 def Order(self, *orderings): |
788 def Order(self, *orderings): |
729 """Specify how the query results should be sorted. |
789 """Specify how the query results should be sorted. |
730 |
790 |
792 raise datastore_errors.BadArgumentError( |
852 raise datastore_errors.BadArgumentError( |
793 'Order() expects Query.ASCENDING or DESCENDING; received %s' % |
853 'Order() expects Query.ASCENDING or DESCENDING; received %s' % |
794 str(direction)) |
854 str(direction)) |
795 direction = Query.ASCENDING |
855 direction = Query.ASCENDING |
796 |
856 |
|
857 if (self.__kind is None and |
|
858 (property != datastore_types._KEY_SPECIAL_PROPERTY or |
|
859 direction != Query.ASCENDING)): |
|
860 raise datastore_errors.BadArgumentError( |
|
861 'Only %s ascending orders are supported on kindless queries' % |
|
862 datastore_types._KEY_SPECIAL_PROPERTY) |
|
863 |
797 orderings[i] = (property, direction) |
864 orderings[i] = (property, direction) |
798 |
865 |
799 if (orderings and self.__inequality_prop and |
866 if (orderings and self.__inequality_prop and |
800 orderings[0][0] != self.__inequality_prop): |
867 orderings[0][0] != self.__inequality_prop): |
801 raise datastore_errors.BadArgumentError( |
868 raise datastore_errors.BadArgumentError( |
882 # an iterator that provides access to the query results |
949 # an iterator that provides access to the query results |
883 Iterator |
950 Iterator |
884 """ |
951 """ |
885 return self._Run() |
952 return self._Run() |
886 |
953 |
887 def _Run(self, limit=None, offset=None): |
954 def _Run(self, limit=None, offset=None, |
|
955 prefetch_count=None, next_count=None): |
888 """Runs this query, with an optional result limit and an optional offset. |
956 """Runs this query, with an optional result limit and an optional offset. |
889 |
957 |
890 Identical to Run, with the extra optional limit and offset parameters. |
958 Identical to Run, with the extra optional limit, offset, prefetch_count, |
891 limit and offset must both be integers >= 0. |
959 next_count parameters. These parameters must be integers >= 0. |
892 |
960 |
893 This is not intended to be used by application developers. Use Get() |
961 This is not intended to be used by application developers. Use Get() |
894 instead! |
962 instead! |
895 """ |
963 """ |
896 pb = self._ToPb(limit, offset) |
964 pb = self._ToPb(limit, offset, prefetch_count) |
897 result = datastore_pb.QueryResult() |
965 result = datastore_pb.QueryResult() |
898 |
966 |
899 try: |
967 try: |
900 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'RunQuery', pb, result) |
968 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'RunQuery', pb, result) |
901 except apiproxy_errors.ApplicationError, err: |
969 except apiproxy_errors.ApplicationError, err: |
954 if not isinstance(offset, (int, long)) or offset < 0: |
1022 if not isinstance(offset, (int, long)) or offset < 0: |
955 raise datastore_errors.BadArgumentError( |
1023 raise datastore_errors.BadArgumentError( |
956 'Argument to Get named \'offset\' must be an int greater than or ' |
1024 'Argument to Get named \'offset\' must be an int greater than or ' |
957 'equal to 0; received %s (a %s)' % (offset, typename(offset))) |
1025 'equal to 0; received %s (a %s)' % (offset, typename(offset))) |
958 |
1026 |
959 return self._Run(limit, offset)._Get(limit) |
1027 return self._Run(limit=limit, offset=offset, |
|
1028 prefetch_count=limit)._Get(limit) |
960 |
1029 |
961 def Count(self, limit=None): |
1030 def Count(self, limit=None): |
962 """Returns the number of entities that this query matches. The returned |
1031 """Returns the number of entities that this query matches. The returned |
963 count is cached; successive Count() calls will not re-scan the datastore |
1032 count is cached; successive Count() calls will not re-scan the datastore |
964 unless the query is changed. |
1033 unless the query is changed. |
1106 raise datastore_errors.BadFilterError( |
1175 raise datastore_errors.BadFilterError( |
1107 'Inequality operators (%s) must be on the same property as the ' |
1176 'Inequality operators (%s) must be on the same property as the ' |
1108 'first sort order, if any sort orders are supplied' % |
1177 'first sort order, if any sort orders are supplied' % |
1109 ', '.join(self.INEQUALITY_OPERATORS)) |
1178 ', '.join(self.INEQUALITY_OPERATORS)) |
1110 |
1179 |
|
1180 if (self.__kind is None and |
|
1181 property != datastore_types._KEY_SPECIAL_PROPERTY): |
|
1182 raise datastore_errors.BadFilterError( |
|
1183 'Only %s filters are allowed on kindless queries.' % |
|
1184 datastore_types._KEY_SPECIAL_PROPERTY) |
|
1185 |
1111 if property in datastore_types._SPECIAL_PROPERTIES: |
1186 if property in datastore_types._SPECIAL_PROPERTIES: |
1112 if property == datastore_types._KEY_SPECIAL_PROPERTY: |
1187 if property == datastore_types._KEY_SPECIAL_PROPERTY: |
1113 for value in values: |
1188 for value in values: |
1114 if not isinstance(value, Key): |
1189 if not isinstance(value, Key): |
1115 raise datastore_errors.BadFilterError( |
1190 raise datastore_errors.BadFilterError( |
1116 '%s filter value must be a Key; received %s (a %s)' % |
1191 '%s filter value must be a Key; received %s (a %s)' % |
1117 (datastore_types._KEY_SPECIAL_PROPERTY, value, typename(value))) |
1192 (datastore_types._KEY_SPECIAL_PROPERTY, value, typename(value))) |
1118 |
1193 |
1119 return match |
1194 return match |
1120 |
1195 |
1121 def _ToPb(self, limit=None, offset=None): |
1196 def _ToPb(self, limit=None, offset=None, count=None): |
1122 """Converts this Query to its protocol buffer representation. Not |
1197 """Converts this Query to its protocol buffer representation. Not |
1123 intended to be used by application developers. Enforced by hiding the |
1198 intended to be used by application developers. Enforced by hiding the |
1124 datastore_pb classes. |
1199 datastore_pb classes. |
1125 |
1200 |
1126 Args: |
1201 Args: |
1127 # an upper bound on the number of results returned by the query. |
1202 # an upper bound on the number of results returned by the query. |
1128 limit: int |
1203 limit: int |
1129 # number of results that match the query to skip. limit is applied |
1204 # number of results that match the query to skip. limit is applied |
1130 # after the offset is fulfilled |
1205 # after the offset is fulfilled |
1131 offset: int |
1206 offset: int |
|
1207 # the requested initial batch size |
|
1208 count: int |
1132 |
1209 |
1133 Returns: |
1210 Returns: |
1134 # the PB representation of this Query |
1211 # the PB representation of this Query |
1135 datastore_pb.Query |
1212 datastore_pb.Query |
1136 |
1213 |
1137 Raises: |
1214 Raises: |
1138 BadRequestError if called inside a transaction and the query does not |
1215 BadRequestError if called inside a transaction and the query does not |
1139 include an ancestor. |
1216 include an ancestor. |
1140 """ |
1217 """ |
|
1218 |
1141 if not self.__ancestor and _CurrentTransactionKey(): |
1219 if not self.__ancestor and _CurrentTransactionKey(): |
1142 raise datastore_errors.BadRequestError( |
1220 raise datastore_errors.BadRequestError( |
1143 'Only ancestor queries are allowed inside transactions.') |
1221 'Only ancestor queries are allowed inside transactions.') |
1144 |
1222 |
1145 pb = datastore_pb.Query() |
1223 pb = datastore_pb.Query() |
1146 _MaybeSetupTransaction(pb, [self.__ancestor]) |
1224 _MaybeSetupTransaction(pb, [self.__ancestor]) |
1147 |
1225 |
1148 pb.set_kind(self.__kind.encode('utf-8')) |
1226 if self.__kind is not None: |
|
1227 pb.set_kind(self.__kind.encode('utf-8')) |
1149 pb.set_keys_only(bool(self.__keys_only)) |
1228 pb.set_keys_only(bool(self.__keys_only)) |
1150 if self.__app: |
1229 if self.__app: |
1151 pb.set_app(self.__app.encode('utf-8')) |
1230 pb.set_app(self.__app.encode('utf-8')) |
1152 if limit is not None: |
1231 if limit is not None: |
1153 pb.set_limit(limit) |
1232 pb.set_limit(limit) |
1154 if offset is not None: |
1233 if offset is not None: |
1155 pb.set_offset(offset) |
1234 pb.set_offset(offset) |
|
1235 if count is not None: |
|
1236 pb.set_count(count) |
1156 if self.__ancestor: |
1237 if self.__ancestor: |
1157 pb.mutable_ancestor().CopyFrom(self.__ancestor._Key__reference) |
1238 pb.mutable_ancestor().CopyFrom(self.__ancestor._Key__reference) |
1158 |
1239 |
1159 if ((self.__hint == self.ORDER_FIRST and self.__orderings) or |
1240 if ((self.__hint == self.ORDER_FIRST and self.__orderings) or |
1160 (self.__hint == self.ANCESTOR_FIRST and self.__ancestor) or |
1241 (self.__hint == self.ANCESTOR_FIRST and self.__ancestor) or |
1189 order = pb.add_order() |
1270 order = pb.add_order() |
1190 order.set_property(property.encode('utf-8')) |
1271 order.set_property(property.encode('utf-8')) |
1191 order.set_direction(direction) |
1272 order.set_direction(direction) |
1192 |
1273 |
1193 return pb |
1274 return pb |
|
1275 |
|
1276 |
|
1277 def AllocateIds(model_key, size): |
|
1278 """Allocates a range of IDs of size for the key defined by model_key |
|
1279 |
|
1280 Allocates a range of IDs in the datastore such that those IDs will not |
|
1281 be automatically assigned to new entities. You can only allocate IDs |
|
1282 for model keys from your app. If there is an error, raises a subclass of |
|
1283 datastore_errors.Error. |
|
1284 |
|
1285 Args: |
|
1286 model_key: Key or string to serve as a model specifying the ID sequence |
|
1287 in which to allocate IDs |
|
1288 |
|
1289 Returns: |
|
1290 (start, end) of the allocated range, inclusive. |
|
1291 """ |
|
1292 keys, multiple = NormalizeAndTypeCheckKeys(model_key) |
|
1293 |
|
1294 if len(keys) > 1: |
|
1295 raise datastore_errors.BadArgumentError( |
|
1296 'Cannot allocate IDs for more than one model key at a time') |
|
1297 |
|
1298 if size > _MAX_ID_BATCH_SIZE: |
|
1299 raise datastore_errors.BadArgumentError( |
|
1300 'Cannot allocate more than %s ids at a time' % _MAX_ID_BATCH_SIZE) |
|
1301 |
|
1302 req = datastore_pb.AllocateIdsRequest() |
|
1303 req.mutable_model_key().CopyFrom(keys[0]._Key__reference) |
|
1304 req.set_size(size) |
|
1305 |
|
1306 resp = datastore_pb.AllocateIdsResponse() |
|
1307 try: |
|
1308 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'AllocateIds', req, resp) |
|
1309 except apiproxy_errors.ApplicationError, err: |
|
1310 raise _ToDatastoreError(err) |
|
1311 |
|
1312 return resp.start(), resp.end() |
1194 |
1313 |
1195 |
1314 |
1196 class MultiQuery(Query): |
1315 class MultiQuery(Query): |
1197 """Class representing a query which requires multiple datastore queries. |
1316 """Class representing a query which requires multiple datastore queries. |
1198 |
1317 |
1545 |
1665 |
1546 Returns: |
1666 Returns: |
1547 # a list of entities or keys |
1667 # a list of entities or keys |
1548 [Entity or Key, ...] |
1668 [Entity or Key, ...] |
1549 """ |
1669 """ |
1550 entityList = self._Next(count) |
1670 entity_list = self._Next(count) |
1551 while len(entityList) < count and self.__more_results: |
1671 while len(entity_list) < count and self.__more_results: |
1552 next_results = self._Next(count - len(entityList)) |
1672 next_results = self._Next(count - len(entity_list), self.__batch_size) |
1553 if not next_results: |
1673 if not next_results: |
1554 break |
1674 break |
1555 entityList += next_results |
1675 entity_list += next_results |
1556 return entityList; |
1676 return entity_list; |
1557 |
1677 |
1558 def _Next(self, count): |
1678 def _Next(self, count=None): |
1559 """Returns the next result(s) of the query. |
1679 """Returns the next batch of results. |
1560 |
1680 |
1561 Not intended to be used by application developers. Use the python |
1681 Not intended to be used by application developers. Use the python |
1562 iterator protocol instead. |
1682 iterator protocol instead. |
1563 |
1683 |
1564 This method returns the next entities or keys from the list of matching |
1684 This method returns the next entities or keys from the list of matching |
1565 results. If the query specified a sort order, results are returned in that |
1685 results. If the query specified a sort order, results are returned in that |
1566 order. Otherwise, the order is undefined. |
1686 order. Otherwise, the order is undefined. |
1567 |
1687 |
1568 The argument, count, specifies the number of results to return. However, the |
1688 The optional argument, count, specifies the number of results to return. |
1569 length of the returned list may be smaller than count. This is the case if |
1689 However, the length of the returned list may be smaller than count. This is |
1570 count is greater than the number of remaining results or the size of the |
1690 the case if count is greater than the number of remaining results or the |
1571 remaining results exciteds the RPC buffer limit. Use _Get to insure all |
1691 size of the remaining results exceeds the RPC buffer limit. Use _Get to |
1572 possible entities are retrieved. |
1692 insure all possible entities are retrieved. |
|
1693 |
|
1694 If the count is omitted, the datastore backend decides how many entities to |
|
1695 send. |
1573 |
1696 |
1574 There is an internal buffer for use with the next() method. If this buffer |
1697 There is an internal buffer for use with the next() method. If this buffer |
1575 is not empty, up to 'count' values are removed from this buffer and |
1698 is not empty, up to 'count' values are removed from this buffer and |
1576 returned. It's best not to mix _Next() and next(). |
1699 returned. It's best not to mix _Next() and next(). |
1577 |
1700 |
1578 The results are always returned as a list. If there are no results left, |
1701 The results are always returned as a list. If there are no results left, |
1579 an empty list is returned. |
1702 an empty list is returned. |
1580 |
1703 |
1581 Args: |
1704 Args: |
1582 # the number of results to return; must be >= 1 |
1705 # the number of results to return; must be >= 1 |
1583 count: int or long |
1706 count: int or long or None |
1584 |
1707 |
1585 Returns: |
1708 Returns: |
1586 # a list of entities or keys |
1709 # a list of entities or keys |
1587 [Entity or Key, ...] |
1710 [Entity or Key, ...] |
1588 """ |
1711 """ |
1589 if not isinstance(count, (int, long)) or count <= 0: |
1712 if count is not None and (not isinstance(count, (int, long)) or count <= 0): |
1590 raise datastore_errors.BadArgumentError( |
1713 raise datastore_errors.BadArgumentError( |
1591 'Argument to _Next must be an int greater than 0; received %s (a %s)' % |
1714 'Argument to _Next must be an int greater than 0; received %s (a %s)' % |
1592 (count, typename(count))) |
1715 (count, typename(count))) |
1593 |
1716 |
1594 if self.__buffer: |
1717 if self.__buffer: |
1595 if count <= len(self.__buffer): |
1718 if count is None: |
|
1719 entity_list = self.__buffer |
|
1720 self.__buffer = [] |
|
1721 return entity_list |
|
1722 elif count <= len(self.__buffer): |
1596 entity_list = self.__buffer[:count] |
1723 entity_list = self.__buffer[:count] |
1597 del self.__buffer[:count] |
1724 del self.__buffer[:count] |
1598 return entity_list |
1725 return entity_list |
1599 else: |
1726 else: |
1600 entity_list = self.__buffer |
1727 entity_list = self.__buffer |
1601 self.__buffer = [] |
1728 self.__buffer = [] |
1602 count -= len(entity_list) |
1729 count -= len(entity_list) |
1603 else: |
1730 else: |
1604 entity_list=[] |
1731 entity_list = [] |
|
1732 |
1605 |
1733 |
1606 if not self.__more_results: |
1734 if not self.__more_results: |
1607 return entity_list |
1735 return entity_list |
1608 |
1736 |
1609 req = datastore_pb.NextRequest() |
1737 req = datastore_pb.NextRequest() |
1610 req.set_count(count) |
1738 if count is not None: |
|
1739 req.set_count(count) |
1611 req.mutable_cursor().CopyFrom(self.__cursor) |
1740 req.mutable_cursor().CopyFrom(self.__cursor) |
1612 result = datastore_pb.QueryResult() |
1741 result = datastore_pb.QueryResult() |
1613 try: |
1742 try: |
1614 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Next', req, result) |
1743 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Next', req, result) |
1615 except apiproxy_errors.ApplicationError, err: |
1744 except apiproxy_errors.ApplicationError, err: |
1640 if self.__keys_only: |
1769 if self.__keys_only: |
1641 return [Key._FromPb(e.key()) for e in result.result_list()] |
1770 return [Key._FromPb(e.key()) for e in result.result_list()] |
1642 else: |
1771 else: |
1643 return [Entity._FromPb(e) for e in result.result_list()] |
1772 return [Entity._FromPb(e) for e in result.result_list()] |
1644 |
1773 |
1645 _BUFFER_SIZE = 20 |
|
1646 |
|
1647 def next(self): |
1774 def next(self): |
1648 if not self.__buffer: |
1775 if not self.__buffer: |
1649 self.__buffer = self._Next(self._BUFFER_SIZE) |
1776 self.__buffer = self._Next(self.__batch_size) |
1650 try: |
1777 try: |
1651 return self.__buffer.pop(0) |
1778 return self.__buffer.pop(0) |
1652 except IndexError: |
1779 except IndexError: |
1653 raise StopIteration |
1780 raise StopIteration |
1654 |
1781 |
1655 def __iter__(self): return self |
1782 def __iter__(self): return self |
1656 |
1783 |
1657 class _Transaction(object): |
1784 class _Transaction(object): |
1658 """Encapsulates a transaction currently in progress. |
1785 """Encapsulates a transaction currently in progress. |
1659 |
1786 |
1660 If we've sent a BeginTransaction call, then handle will be a |
|
1661 datastore_pb.Transaction that holds the transaction handle. |
|
1662 |
|
1663 If we know the entity group for this transaction, it's stored in the |
1787 If we know the entity group for this transaction, it's stored in the |
1664 entity_group attribute, which is set by RecordModifiedKeys(). |
1788 entity_group attribute, which is set by RunInTransaction(). |
1665 |
1789 |
1666 modified_keys is a set containing the Keys of all entities modified (ie put |
1790 modified_keys is a set containing the Keys of all entities modified (ie put |
1667 or deleted) in this transaction. If an entity is modified more than once, a |
1791 or deleted) in this transaction. If an entity is modified more than once, a |
1668 BadRequestError is raised. |
1792 BadRequestError is raised. |
1669 """ |
1793 """ |
1670 def __init__(self): |
1794 def __init__(self, handle): |
1671 """Initializes modified_keys to the empty set.""" |
1795 """Initializes the transaction. |
1672 self.handle = None |
1796 |
|
1797 Args: |
|
1798 handle: a datastore_pb.Transaction returned by a BeginTransaction call |
|
1799 """ |
|
1800 assert isinstance(handle, datastore_pb.Transaction) |
|
1801 explanation = [] |
|
1802 assert handle.IsInitialized(explanation), explanation |
|
1803 |
|
1804 self.handle = handle |
1673 self.entity_group = None |
1805 self.entity_group = None |
1674 self.modified_keys = None |
1806 self.modified_keys = None |
1675 self.modified_keys = set() |
1807 self.modified_keys = set() |
1676 |
|
1677 def RecordModifiedKeys(self, keys, error_on_repeat=True): |
|
1678 """Updates the modified keys seen so far. |
|
1679 |
|
1680 If error_on_repeat is True and any of the given keys have already been |
|
1681 modified, raises BadRequestError. |
|
1682 |
|
1683 Args: |
|
1684 keys: sequence of Keys |
|
1685 """ |
|
1686 keys, _ = NormalizeAndTypeCheckKeys(keys) |
|
1687 keys = set(keys) |
|
1688 |
|
1689 if error_on_repeat: |
|
1690 already_modified = self.modified_keys.intersection(keys) |
|
1691 if already_modified: |
|
1692 raise datastore_errors.BadRequestError( |
|
1693 "Can't update entity more than once in a transaction: %r" % |
|
1694 already_modified.pop()) |
|
1695 |
|
1696 self.modified_keys.update(keys) |
|
1697 |
1808 |
1698 |
1809 |
1699 def RunInTransaction(function, *args, **kwargs): |
1810 def RunInTransaction(function, *args, **kwargs): |
1700 """Runs a function inside a datastore transaction. |
1811 """Runs a function inside a datastore transaction. |
1701 |
1812 |
1797 |
1908 |
1798 tx_key = None |
1909 tx_key = None |
1799 |
1910 |
1800 try: |
1911 try: |
1801 tx_key = _NewTransactionKey() |
1912 tx_key = _NewTransactionKey() |
1802 tx = _Transaction() |
|
1803 _txes[tx_key] = tx |
|
1804 |
1913 |
1805 for i in range(0, retries + 1): |
1914 for i in range(0, retries + 1): |
1806 tx.modified_keys.clear() |
1915 handle = datastore_pb.Transaction() |
|
1916 try: |
|
1917 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'BeginTransaction', |
|
1918 api_base_pb.VoidProto(), handle) |
|
1919 except apiproxy_errors.ApplicationError, err: |
|
1920 raise _ToDatastoreError(err) |
|
1921 |
|
1922 tx = _Transaction(handle) |
|
1923 _txes[tx_key] = tx |
1807 |
1924 |
1808 try: |
1925 try: |
1809 result = function(*args, **kwargs) |
1926 result = function(*args, **kwargs) |
1810 except: |
1927 except: |
1811 original_exception = sys.exc_info() |
1928 original_exception = sys.exc_info() |
1812 |
1929 |
1813 if tx.handle: |
1930 try: |
1814 try: |
1931 resp = api_base_pb.VoidProto() |
1815 resp = api_base_pb.VoidProto() |
1932 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Rollback', |
1816 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Rollback', |
1933 tx.handle, resp) |
1817 tx.handle, resp) |
1934 except: |
1818 except: |
1935 exc_info = sys.exc_info() |
1819 exc_info = sys.exc_info() |
1936 logging.info('Exception sending Rollback:\n' + |
1820 logging.info('Exception sending Rollback:\n' + |
1937 ''.join(traceback.format_exception(*exc_info))) |
1821 ''.join(traceback.format_exception(*exc_info))) |
|
1822 |
1938 |
1823 type, value, trace = original_exception |
1939 type, value, trace = original_exception |
1824 if type is datastore_errors.Rollback: |
1940 if type is datastore_errors.Rollback: |
1825 return |
1941 return |
1826 else: |
1942 else: |
1827 raise type, value, trace |
1943 raise type, value, trace |
1828 |
1944 |
1829 if tx.handle: |
1945 try: |
1830 try: |
1946 resp = datastore_pb.CommitResponse() |
1831 resp = datastore_pb.CommitResponse() |
1947 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Commit', |
1832 apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Commit', |
1948 tx.handle, resp) |
1833 tx.handle, resp) |
1949 except apiproxy_errors.ApplicationError, err: |
1834 except apiproxy_errors.ApplicationError, err: |
1950 if (err.application_error == |
1835 if (err.application_error == |
1951 datastore_pb.Error.CONCURRENT_TRANSACTION): |
1836 datastore_pb.Error.CONCURRENT_TRANSACTION): |
1952 logging.warning('Transaction collision for entity group with ' |
1837 logging.warning('Transaction collision for entity group with ' |
1953 'key %r. Retrying...', tx.entity_group) |
1838 'key %r. Retrying...', tx.entity_group) |
1954 tx.handle = None |
1839 tx.handle = None |
1955 tx.entity_group = None |
1840 tx.entity_group = None |
1956 continue |
1841 continue |
1957 else: |
1842 else: |
1958 raise _ToDatastoreError(err) |
1843 raise _ToDatastoreError(err) |
|
1844 |
1959 |
1845 return result |
1960 return result |
1846 |
1961 |
1847 raise datastore_errors.TransactionFailedError( |
1962 raise datastore_errors.TransactionFailedError( |
1848 'The transaction could not be committed. Please try again.') |
1963 'The transaction could not be committed. Please try again.') |
1870 |
1984 |
1871 Returns: |
1985 Returns: |
1872 _Transaction if we're inside a transaction, otherwise None |
1986 _Transaction if we're inside a transaction, otherwise None |
1873 """ |
1987 """ |
1874 assert isinstance(request, (datastore_pb.GetRequest, datastore_pb.PutRequest, |
1988 assert isinstance(request, (datastore_pb.GetRequest, datastore_pb.PutRequest, |
1875 datastore_pb.DeleteRequest, datastore_pb.Query)) |
1989 datastore_pb.DeleteRequest, datastore_pb.Query, |
|
1990 taskqueue_service_pb.TaskQueueAddRequest, |
|
1991 )), request.__class__ |
1876 tx_key = None |
1992 tx_key = None |
1877 |
1993 |
1878 try: |
1994 try: |
1879 tx_key = _CurrentTransactionKey() |
1995 tx_key = _CurrentTransactionKey() |
1880 if tx_key: |
1996 if tx_key: |
1881 tx = _txes[tx_key] |
1997 tx = _txes[tx_key] |
1882 |
1998 |
1883 groups = [k.entity_group() for k in keys] |
1999 groups = [k.entity_group() for k in keys] |
1884 if tx.entity_group: |
2000 if tx.entity_group: |
1885 expected_group = tx.entity_group |
2001 expected_group = tx.entity_group |
|
2002 elif groups: |
|
2003 expected_group = groups[0] |
1886 else: |
2004 else: |
1887 expected_group = groups[0] |
2005 expected_group = None |
1888 |
2006 |
1889 for group in groups: |
2007 for group in groups: |
1890 if (group != expected_group or |
2008 if (group != expected_group or |
1891 |
2009 |
1892 |
2010 |