py_tasks_melange: comparison thirdparty/google_appengine/google/appengine/ext/search/__init_

equal deleted inserted replaced

-:27971a13089f
+:2e0b0af889be
 for result in query:
 ...
 The full text index is stored in a property named __searchable_text_index.
+Specifying multiple indexes and properties to index
+---------------------------------------------------
+By default, one index is created with all string properties. You can define
+multiple indexes and specify which properties should be indexed for each by
+overriding SearchableProperties() method of model.SearchableModel, for example:
+class Article(search.SearchableModel):
+@classmethod
+def SearchableProperties(cls):
+return [['book', 'author'], ['book']]
+In this example, two indexes will be maintained - one that includes 'book' and
+'author' properties, and another one for 'book' property only. They will be
+stored in properties named __searchable_text_index_book_author and
+__searchable_text_index_book respectively. Note that the index that includes
+all properties will not be created unless added explicitly like this:
+@classmethod
+def SearchableProperties(cls):
+return [['book', 'author'], ['book'], search.ALL_PROPERTIES]
+The default return value of SearchableProperties() is [search.ALL_PROPERTIES]
+(one index, all properties).
+To search using a custom-defined index, pass its definition
+in 'properties' parameter of 'search':
+Article.all().search('Lem', properties=['book', 'author'])
+Note that the order of properties in the list matters.
+Adding indexes to  index.yaml
+-----------------------------
 In general, if you just want to provide full text search, you *don't* need to
 add any extra indexes to your index.yaml. However, if you want to use search()
 in a query *in addition to* an ancestor, filter, or sort order, you'll need to
 create an index in index.yaml with the __searchable_text_index property. For
 - name: __searchable_text_index
 - name: date
 direction: desc
 ...
+Similarly, if you created a custom index (see above), use the name of the
+property it's stored in, e.g. __searchable_text_index_book_author.
 Note that using SearchableModel will noticeable increase the latency of save()
 operations, since it writes an index row for each indexable word. This also
 means that the latency of save() will increase roughly with the size of the
 properties in a given entity. Caveat hacker!
 """
 from google.appengine.api import datastore
 from google.appengine.api import datastore_errors
 from google.appengine.api import datastore_types
 from google.appengine.ext import db
 from google.appengine.datastore import datastore_pb
+ALL_PROPERTIES = []
 class SearchableEntity(datastore.Entity):
 """A subclass of datastore.Entity that supports full text indexing.
 Automatically indexes all string and Text properties, using the datastore's
 'where', 'whether', 'which', 'while', 'who', 'whose', 'why', 'widely',
 'will', 'with', 'within', 'without', 'would', 'yet', 'you'])
 _word_delimiter_regex = re.compile('[' + re.escape(string.punctuation) + ']')
+_searchable_properties = [ALL_PROPERTIES]
 def __init__(self, kind_or_entity, word_delimiter_regex=None, *args,
 **kwargs):
 """Constructor. May be called as a copy constructor.
 If kind_or_entity is a datastore.Entity, copies it into this Entity.
 """
 self._word_delimiter_regex = word_delimiter_regex
 if isinstance(kind_or_entity, datastore.Entity):
 self._Entity__key = kind_or_entity._Entity__key
 self._Entity__unindexed_properties = frozenset(kind_or_entity.unindexed_properties())
+if isinstance(kind_or_entity, SearchableEntity):
+if getattr(kind_or_entity, '_searchable_properties', None) is not None:
+self._searchable_properties = kind_or_entity._searchable_properties
 self.update(kind_or_entity)
 else:
 super(SearchableEntity, self).__init__(kind_or_entity, *args, **kwargs)
 def _ToPb(self):
 """Rebuilds the full text index, then delegates to the superclass.
 Returns:
 entity_pb.Entity
 """
-if SearchableEntity._FULL_TEXT_INDEX_PROPERTY in self:
+for properties_to_index in self._searchable_properties:
-del self[SearchableEntity._FULL_TEXT_INDEX_PROPERTY]
+index_property_name = SearchableEntity.IndexPropertyName(properties_to_index)
+if index_property_name in self:
-index = set()
+del self[index_property_name]
-for (name, values) in self.items():
-if not isinstance(values, list):
-values = [values]
+if not properties_to_index:
-if (isinstance(values[0], basestring) and
+properties_to_index = self.keys()
-not isinstance(values[0], datastore_types.Blob)):
-for value in values:
+index = set()
-index.update(SearchableEntity._FullTextIndex(
+for name in properties_to_index:
-value, self._word_delimiter_regex))
+if not self.has_key(name):
+continue
-index_list = list(index)
-if index_list:
+values = self[name]
-self[SearchableEntity._FULL_TEXT_INDEX_PROPERTY] = index_list
+if not isinstance(values, list):
+values = [values]
+if (isinstance(values[0], basestring) and
+not isinstance(values[0], datastore_types.Blob)):
+for value in values:
+index.update(SearchableEntity._FullTextIndex(
+value, self._word_delimiter_regex))
+index_list = list(index)
+if index_list:
+self[index_property_name] = index_list
 return super(SearchableEntity, self)._ToPb()
 @classmethod
 def _FullTextIndex(cls, text, word_delimiter_regex=None):
 else:
 words = set()
 return words
+@classmethod
+def IndexPropertyName(cls, properties):
+"""Given index definition, returns the name of the property to put it in."""
+name = SearchableEntity._FULL_TEXT_INDEX_PROPERTY
+if properties:
+name += '_' + '_'.join(properties)
+return name
 class SearchableQuery(datastore.Query):
 """A subclass of datastore.Query that supports full text search.
 Only searches over entities that were created and stored using the
 SearchableEntity or SearchableModel classes.
 """
-def Search(self, search_query, word_delimiter_regex=None):
+def Search(self, search_query, word_delimiter_regex=None,
+properties=ALL_PROPERTIES):
 """Add a search query. This may be combined with filters.
 Note that keywords in the search query will be silently dropped if they
 are stop words or too short, ie if they wouldn't be indexed.
 SearchableQuery
 """
 datastore_types.ValidateString(search_query, 'search query')
 self._search_query = search_query
 self._word_delimiter_regex = word_delimiter_regex
+self._properties = properties
 return self
-def _ToPb(self, limit=None, offset=None):
+def _ToPb(self, *args, **kwds):
 """Adds filters for the search query, then delegates to the superclass.
-Raises BadFilterError if a filter on the index property already exists.
+Mimics Query._ToPb()'s signature. Raises BadFilterError if a filter on the
+index property already exists.
-Args:
-# an upper bound on the number of results returned by the query.
-limit: int
-# number of results that match the query to skip.  limit is applied
-# after the offset is fulfilled.
-offset: int
 Returns:
 datastore_pb.Query
 """
-if SearchableEntity._FULL_TEXT_INDEX_PROPERTY in self:
+properties = getattr(self, "_properties", ALL_PROPERTIES)
+index_property_name = SearchableEntity.IndexPropertyName(properties)
+if index_property_name in self:
 raise datastore_errors.BadFilterError(
-'%s is a reserved name.' % SearchableEntity._FULL_TEXT_INDEX_PROPERTY)
+'%s is a reserved name.' % index_property_name)
-pb = super(SearchableQuery, self)._ToPb(limit=limit, offset=offset)
+pb = super(SearchableQuery, self)._ToPb(*args, **kwds)
 if hasattr(self, '_search_query'):
 keywords = SearchableEntity._FullTextIndex(
 self._search_query, self._word_delimiter_regex)
 for keyword in keywords:
 filter = pb.add_filter()
 filter.set_op(datastore_pb.Query_Filter.EQUAL)
 prop = filter.add_property()
-prop.set_name(SearchableEntity._FULL_TEXT_INDEX_PROPERTY)
+prop.set_name(index_property_name)
 prop.set_multiple(len(keywords) > 1)
 prop.mutable_value().set_stringvalue(unicode(keyword).encode('utf-8'))
 return pb
 class SearchableModel(db.Model):
 """A subclass of db.Model that supports full text search and indexing.
 Automatically indexes all string-based properties. To search, use the all()
 method to get a SearchableModel.Query, then use its search() method.
+Override SearchableProperties() to define properties to index and/or multiple
+indexes (see the file's comment).
 """
+@classmethod
+def SearchableProperties(cls):
+return [ALL_PROPERTIES]
 class Query(db.Query):
 """A subclass of db.Query that supports full text search."""
 _search_query = None
+_properties = None
-def search(self, search_query):
+def search(self, search_query, properties=ALL_PROPERTIES):
 """Adds a full text search to this query.
 Args:
 search_query, a string containing the full text search query.
 Returns:
 self
 """
 self._search_query = search_query
+self._properties = properties
+if self._properties not in getattr(self, '_searchable_properties', [ALL_PROPERTIES]):
+raise datastore_errors.BadFilterError(
+'%s does not have a corresponding index. Please add it to'
+'the SEARCHABLE_PROPERTIES list' % self._properties)
 return self
 def _get_query(self):
 """Wraps db.Query._get_query() and injects SearchableQuery."""
 query = db.Query._get_query(self,
 _query_class=SearchableQuery,
 _multi_query_class=SearchableMultiQuery)
 if self._search_query:
-query.Search(self._search_query)
+query.Search(self._search_query, properties=self._properties)
 return query
 def _populate_internal_entity(self):
 """Wraps db.Model._populate_internal_entity() and injects
 SearchableEntity."""
-return db.Model._populate_internal_entity(self,
+entity = db.Model._populate_internal_entity(self,
 _entity_class=SearchableEntity)
+entity._searchable_properties = self.SearchableProperties()
+return entity
 @classmethod
 def from_entity(cls, entity):
 """Wraps db.Model.from_entity() and injects SearchableEntity."""
 if not isinstance(entity, SearchableEntity):
 return super(SearchableModel, cls).from_entity(entity)
 @classmethod
 def all(cls):
 """Returns a SearchableModel.Query for this kind."""
-return SearchableModel.Query(cls)
+query = SearchableModel.Query(cls)
+query._searchable_properties = cls.SearchableProperties()
+return query

changeset 2864	2e0b0af889be
parent 2309	be1b94099f2d