94 self.protobuf = entity |
94 self.protobuf = entity |
95 |
95 |
96 self.encoded_protobuf = entity.Encode() |
96 self.encoded_protobuf = entity.Encode() |
97 |
97 |
98 self.native = datastore.Entity._FromPb(entity) |
98 self.native = datastore.Entity._FromPb(entity) |
|
99 |
|
100 |
|
101 class _Cursor(object): |
|
102 """A query cursor. |
|
103 |
|
104 Public properties: |
|
105 cursor: the integer cursor |
|
106 count: the original total number of results |
|
107 keys_only: whether the query is keys_only |
|
108 """ |
|
109 def __init__(self, results, keys_only): |
|
110 """Constructor. |
|
111 |
|
112 Args: |
|
113 # the query results, in order, such that pop(0) is the next result |
|
114 results: list of entity_pb.EntityProto |
|
115 keys_only: integer |
|
116 """ |
|
117 self.__results = results |
|
118 self.count = len(results) |
|
119 self.keys_only = keys_only |
|
120 self.cursor = id(self) |
|
121 |
|
122 def PopulateQueryResult(self, result, count): |
|
123 """Populates a QueryResult with this cursor and the given number of results. |
|
124 |
|
125 Args: |
|
126 result: datastore_pb.QueryResult |
|
127 count: integer |
|
128 """ |
|
129 result.mutable_cursor().set_cursor(self.cursor) |
|
130 result.set_keys_only(self.keys_only) |
|
131 |
|
132 results_pbs = [r._ToPb() for r in self.__results[:count]] |
|
133 result.result_list().extend(results_pbs) |
|
134 del self.__results[:count] |
|
135 |
|
136 result.set_more_results(len(self.__results) > 0) |
99 |
137 |
100 |
138 |
101 class DatastoreFileStub(apiproxy_stub.APIProxyStub): |
139 class DatastoreFileStub(apiproxy_stub.APIProxyStub): |
102 """ Persistent stub for the Python datastore API. |
140 """ Persistent stub for the Python datastore API. |
103 |
141 |
187 self.__require_indexes = require_indexes |
225 self.__require_indexes = require_indexes |
188 |
226 |
189 self.__query_history = {} |
227 self.__query_history = {} |
190 |
228 |
191 self.__next_id = 1 |
229 self.__next_id = 1 |
192 self.__next_cursor = 1 |
|
193 self.__next_tx_handle = 1 |
230 self.__next_tx_handle = 1 |
194 self.__next_index_id = 1 |
231 self.__next_index_id = 1 |
195 self.__id_lock = threading.Lock() |
232 self.__id_lock = threading.Lock() |
196 self.__cursor_lock = threading.Lock() |
|
197 self.__tx_handle_lock = threading.Lock() |
233 self.__tx_handle_lock = threading.Lock() |
198 self.__index_id_lock = threading.Lock() |
234 self.__index_id_lock = threading.Lock() |
199 self.__tx_lock = threading.Lock() |
235 self.__tx_lock = threading.Lock() |
200 self.__entities_lock = threading.Lock() |
236 self.__entities_lock = threading.Lock() |
201 self.__file_lock = threading.Lock() |
237 self.__file_lock = threading.Lock() |
579 datastore_pb.Query_Filter.GREATER_THAN: '>', |
615 datastore_pb.Query_Filter.GREATER_THAN: '>', |
580 datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: '>=', |
616 datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: '>=', |
581 datastore_pb.Query_Filter.EQUAL: '==', |
617 datastore_pb.Query_Filter.EQUAL: '==', |
582 } |
618 } |
583 |
619 |
|
620 def has_prop_indexed(entity, prop): |
|
621 """Returns True if prop is in the entity and is indexed.""" |
|
622 if prop in datastore_types._SPECIAL_PROPERTIES: |
|
623 return True |
|
624 elif prop in entity.unindexed_properties(): |
|
625 return False |
|
626 |
|
627 values = entity.get(prop, []) |
|
628 if not isinstance(values, (tuple, list)): |
|
629 values = [values] |
|
630 |
|
631 for value in values: |
|
632 if type(value) not in datastore_types._RAW_PROPERTY_TYPES: |
|
633 return True |
|
634 return False |
|
635 |
584 for filt in query.filter_list(): |
636 for filt in query.filter_list(): |
585 assert filt.op() != datastore_pb.Query_Filter.IN |
637 assert filt.op() != datastore_pb.Query_Filter.IN |
586 |
638 |
587 prop = filt.property(0).name().decode('utf-8') |
639 prop = filt.property(0).name().decode('utf-8') |
588 op = operators[filt.op()] |
640 op = operators[filt.op()] |
589 |
641 |
590 filter_val_list = [datastore_types.FromPropertyPb(filter_prop) |
642 filter_val_list = [datastore_types.FromPropertyPb(filter_prop) |
591 for filter_prop in filt.property_list()] |
643 for filter_prop in filt.property_list()] |
592 |
644 |
593 def passes(entity): |
645 def passes_filter(entity): |
594 """ Returns True if the entity passes the filter, False otherwise. """ |
646 """Returns True if the entity passes the filter, False otherwise. |
595 if prop in datastore_types._SPECIAL_PROPERTIES: |
647 |
596 entity_vals = self.__GetSpecialPropertyValue(entity, prop) |
648 The filter being evaluated is filt, the current filter that we're on |
597 else: |
649 in the list of filters in the query. |
598 entity_vals = entity.get(prop, []) |
650 """ |
|
651 if not has_prop_indexed(entity, prop): |
|
652 return False |
|
653 |
|
654 try: |
|
655 entity_vals = datastore._GetPropertyValue(entity, prop) |
|
656 except KeyError: |
|
657 entity_vals = [] |
599 |
658 |
600 if not isinstance(entity_vals, list): |
659 if not isinstance(entity_vals, list): |
601 entity_vals = [entity_vals] |
660 entity_vals = [entity_vals] |
602 |
661 |
603 for fixed_entity_val in entity_vals: |
662 for fixed_entity_val in entity_vals: |
604 if type(fixed_entity_val) in datastore_types._RAW_PROPERTY_TYPES: |
|
605 continue |
|
606 |
|
607 for filter_val in filter_val_list: |
663 for filter_val in filter_val_list: |
608 fixed_entity_type = self._PROPERTY_TYPE_TAGS.get( |
664 fixed_entity_type = self._PROPERTY_TYPE_TAGS.get( |
609 fixed_entity_val.__class__) |
665 fixed_entity_val.__class__) |
610 filter_type = self._PROPERTY_TYPE_TAGS.get(filter_val.__class__) |
666 filter_type = self._PROPERTY_TYPE_TAGS.get(filter_val.__class__) |
611 if fixed_entity_type == filter_type: |
667 if fixed_entity_type == filter_type: |
625 except TypeError: |
681 except TypeError: |
626 pass |
682 pass |
627 |
683 |
628 return False |
684 return False |
629 |
685 |
630 results = filter(passes, results) |
686 results = filter(passes_filter, results) |
631 |
|
632 def has_prop_indexed(entity, prop): |
|
633 """Returns True if prop is in the entity and is not a raw property, or |
|
634 is a special property.""" |
|
635 if prop in datastore_types._SPECIAL_PROPERTIES: |
|
636 return True |
|
637 |
|
638 values = entity.get(prop, []) |
|
639 if not isinstance(values, (tuple, list)): |
|
640 values = [values] |
|
641 |
|
642 for value in values: |
|
643 if type(value) not in datastore_types._RAW_PROPERTY_TYPES: |
|
644 return True |
|
645 return False |
|
646 |
687 |
647 for order in query.order_list(): |
688 for order in query.order_list(): |
648 prop = order.property().decode('utf-8') |
689 prop = order.property().decode('utf-8') |
649 results = [entity for entity in results if has_prop_indexed(entity, prop)] |
690 results = [entity for entity in results if has_prop_indexed(entity, prop)] |
650 |
691 |
656 for o in query.order_list(): |
697 for o in query.order_list(): |
657 prop = o.property().decode('utf-8') |
698 prop = o.property().decode('utf-8') |
658 |
699 |
659 reverse = (o.direction() is datastore_pb.Query_Order.DESCENDING) |
700 reverse = (o.direction() is datastore_pb.Query_Order.DESCENDING) |
660 |
701 |
661 if prop in datastore_types._SPECIAL_PROPERTIES: |
702 a_val = datastore._GetPropertyValue(a, prop) |
662 a_val = self.__GetSpecialPropertyValue(a, prop) |
703 if isinstance(a_val, list): |
663 b_val = self.__GetSpecialPropertyValue(b, prop) |
704 a_val = sorted(a_val, order_compare_properties, reverse=reverse)[0] |
664 else: |
705 |
665 a_val = a[prop] |
706 b_val = datastore._GetPropertyValue(b, prop) |
666 if isinstance(a_val, list): |
707 if isinstance(b_val, list): |
667 a_val = sorted(a_val, order_compare_properties, reverse=reverse)[0] |
708 b_val = sorted(b_val, order_compare_properties, reverse=reverse)[0] |
668 |
|
669 b_val = b[prop] |
|
670 if isinstance(b_val, list): |
|
671 b_val = sorted(b_val, order_compare_properties, reverse=reverse)[0] |
|
672 |
709 |
673 cmped = order_compare_properties(a_val, b_val) |
710 cmped = order_compare_properties(a_val, b_val) |
674 |
711 |
675 if o.direction() is datastore_pb.Query_Order.DESCENDING: |
712 if o.direction() is datastore_pb.Query_Order.DESCENDING: |
676 cmped = -cmped |
713 cmped = -cmped |
723 self.__query_history[clone] += 1 |
760 self.__query_history[clone] += 1 |
724 else: |
761 else: |
725 self.__query_history[clone] = 1 |
762 self.__query_history[clone] = 1 |
726 self.__WriteHistory() |
763 self.__WriteHistory() |
727 |
764 |
728 self.__cursor_lock.acquire() |
765 cursor = _Cursor(results, query.keys_only()) |
729 cursor = self.__next_cursor |
766 self.__queries[cursor.cursor] = cursor |
730 self.__next_cursor += 1 |
767 cursor.PopulateQueryResult(query_result, 0) |
731 self.__cursor_lock.release() |
|
732 self.__queries[cursor] = (results, len(results)) |
|
733 |
|
734 query_result.mutable_cursor().set_cursor(cursor) |
|
735 query_result.set_more_results(len(results) > 0) |
|
736 |
768 |
737 def _Dynamic_Next(self, next_request, query_result): |
769 def _Dynamic_Next(self, next_request, query_result): |
738 cursor = next_request.cursor().cursor() |
770 cursor_handle = next_request.cursor().cursor() |
739 |
771 |
740 try: |
772 try: |
741 results, orig_count = self.__queries[cursor] |
773 cursor = self.__queries[cursor_handle] |
742 except KeyError: |
774 except KeyError: |
743 raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, |
775 raise apiproxy_errors.ApplicationError( |
744 'Cursor %d not found' % cursor) |
776 datastore_pb.Error.BAD_REQUEST, 'Cursor %d not found' % cursor_handle) |
745 |
777 |
746 count = next_request.count() |
778 cursor.PopulateQueryResult(query_result, next_request.count()) |
747 |
|
748 results_pb = [r._ToPb() for r in results[:count]] |
|
749 query_result.result_list().extend(results_pb) |
|
750 del results[:count] |
|
751 |
|
752 query_result.set_more_results(len(results) > 0) |
|
753 |
779 |
754 def _Dynamic_Count(self, query, integer64proto): |
780 def _Dynamic_Count(self, query, integer64proto): |
755 self.__ValidateAppId(query.app()) |
781 self.__ValidateAppId(query.app()) |
756 query_result = datastore_pb.QueryResult() |
782 query_result = datastore_pb.QueryResult() |
757 self._Dynamic_RunQuery(query, query_result) |
783 self._Dynamic_RunQuery(query, query_result) |
758 cursor = query_result.cursor().cursor() |
784 cursor = query_result.cursor().cursor() |
759 results, count = self.__queries[cursor] |
785 integer64proto.set_value(self.__queries[cursor].count) |
760 integer64proto.set_value(count) |
|
761 del self.__queries[cursor] |
786 del self.__queries[cursor] |
762 |
787 |
763 def _Dynamic_BeginTransaction(self, request, transaction): |
788 def _Dynamic_BeginTransaction(self, request, transaction): |
764 self.__tx_handle_lock.acquire() |
789 self.__tx_handle_lock.acquire() |
765 handle = self.__next_tx_handle |
790 handle = self.__next_tx_handle |
943 for stored_index in self.__indexes[app]: |
968 for stored_index in self.__indexes[app]: |
944 if index.definition() == stored_index.definition(): |
969 if index.definition() == stored_index.definition(): |
945 return stored_index |
970 return stored_index |
946 |
971 |
947 return None |
972 return None |
948 |
|
949 @classmethod |
|
950 def __GetSpecialPropertyValue(cls, entity, property): |
|
951 """Returns an entity's value for a special property. |
|
952 |
|
953 Right now, the only special property is __key__, whose value is the |
|
954 entity's key. |
|
955 |
|
956 Args: |
|
957 entity: datastore.Entity |
|
958 |
|
959 Returns: |
|
960 property value. For __key__, a datastore_types.Key. |
|
961 |
|
962 Raises: |
|
963 AssertionError, if the given property is not special. |
|
964 """ |
|
965 assert property in datastore_types._SPECIAL_PROPERTIES |
|
966 if property == datastore_types._KEY_SPECIAL_PROPERTY: |
|
967 return entity.key() |
|