190 )) |
190 )) |
191 EXISTS_OPERATORS = set((datastore_pb.Query_Filter.EXISTS, |
191 EXISTS_OPERATORS = set((datastore_pb.Query_Filter.EXISTS, |
192 )) |
192 )) |
193 |
193 |
194 |
194 |
|
195 def Normalize(filters, orders): |
|
196 """ Normalizes filter and order query components. |
|
197 |
|
198 The resulting components have the same effect as the given components if used |
|
199 in a query. |
|
200 |
|
201 Returns: |
|
202 (filter, orders) the reduced set of filters and orders |
|
203 """ |
|
204 |
|
205 for f in filters: |
|
206 if f.op() == datastore_pb.Query_Filter.IN and f.property_size() == 1: |
|
207 f.set_op(datastore_pb.Query_Filter.EQUAL); |
|
208 |
|
209 eq_properties = set([f.property(0).name() for f in filters if f.op() == datastore_pb.Query_Filter.EQUAL]); |
|
210 |
|
211 remove_set = eq_properties.copy() |
|
212 new_orders = [] |
|
213 for o in orders: |
|
214 if o.property() not in remove_set: |
|
215 remove_set.add(o.property()) |
|
216 new_orders.append(o) |
|
217 orders = new_orders |
|
218 |
|
219 |
|
220 if datastore_types._KEY_SPECIAL_PROPERTY in eq_properties: |
|
221 orders = [] |
|
222 |
|
223 new_orders = [] |
|
224 for o in orders: |
|
225 if o.property() == datastore_types._KEY_SPECIAL_PROPERTY: |
|
226 new_orders.append(o) |
|
227 break |
|
228 new_orders.append(o) |
|
229 orders = new_orders |
|
230 |
|
231 return (filters, orders) |
|
232 |
|
233 |
|
234 def RemoveNativelySupportedComponents(filters, orders): |
|
235 """ Removes query components that are natively supported by the datastore. |
|
236 |
|
237 The resulting filters and orders should not be used in an actual query. |
|
238 |
|
239 Returns |
|
240 (filters, orders) the reduced set of filters and orders |
|
241 """ |
|
242 (filters, orders) = Normalize(filters, orders) |
|
243 |
|
244 has_key_desc_order = False |
|
245 if orders and orders[-1].property() == datastore_types._KEY_SPECIAL_PROPERTY: |
|
246 if orders[-1].direction() == ASCENDING: |
|
247 orders = orders[:-1] |
|
248 else: |
|
249 has_key_desc_order = True |
|
250 |
|
251 if not has_key_desc_order: |
|
252 for f in filters: |
|
253 if (f.op() in INEQUALITY_OPERATORS and |
|
254 f.property(0).name() != datastore_types._KEY_SPECIAL_PROPERTY): |
|
255 break |
|
256 else: |
|
257 filters = [f for f in filters |
|
258 if f.property(0).name() != datastore_types._KEY_SPECIAL_PROPERTY] |
|
259 |
|
260 return (filters, orders) |
|
261 |
|
262 |
195 def CompositeIndexForQuery(query): |
263 def CompositeIndexForQuery(query): |
196 """Return the composite index needed for a query. |
264 """Return the composite index needed for a query. |
197 |
265 |
198 A query is translated into a tuple, as follows: |
266 A query is translated into a tuple, as follows: |
199 |
267 |
211 - After that comes at most one (property, ASCENDING) pair for a |
279 - After that comes at most one (property, ASCENDING) pair for a |
212 Filter entry whose operator is on of the four inequalities. There |
280 Filter entry whose operator is on of the four inequalities. There |
213 can be at most one of these. |
281 can be at most one of these. |
214 |
282 |
215 - After that come all the (property, direction) pairs for the Order |
283 - After that come all the (property, direction) pairs for the Order |
216 entries, in the order given in the query. Exceptions: (a) if |
284 entries, in the order given in the query. Exceptions: |
217 there is a Filter entry with an inequality operator that matches |
285 (a) if there is a Filter entry with an inequality operator that matches |
218 the first Order entry, the first order pair is omitted (or, |
286 the first Order entry, the first order pair is omitted (or, |
219 equivalently, in this case the inequality pair is omitted); (b) if |
287 equivalently, in this case the inequality pair is omitted). |
220 an Order entry corresponds to an equality filter, it is ignored |
288 (b) if an Order entry corresponds to an equality filter, it is ignored |
221 (since there will only ever be one value returned). |
289 (since there will only ever be one value returned). |
|
290 (c) if there is an equality filter on __key__ all orders are dropped |
|
291 (since there will be at most one result returned). |
|
292 (d) if there is an order on __key__ all further orders are dropped (since |
|
293 keys are unique). |
|
294 (e) orders on __key__ ASCENDING are dropped (since this is supported |
|
295 natively by the datastore). |
222 |
296 |
223 - Finally, if there are Filter entries whose operator is EXISTS, and |
297 - Finally, if there are Filter entries whose operator is EXISTS, and |
224 whose property names are not already listed, they are added, with |
298 whose property names are not already listed, they are added, with |
225 the direction set to ASCENDING. |
299 the direction set to ASCENDING. |
226 |
300 |
269 for filter in filters: |
343 for filter in filters: |
270 assert filter.op() != datastore_pb.Query_Filter.IN, 'Filter.op()==IN' |
344 assert filter.op() != datastore_pb.Query_Filter.IN, 'Filter.op()==IN' |
271 nprops = len(filter.property_list()) |
345 nprops = len(filter.property_list()) |
272 assert nprops == 1, 'Filter has %s properties, expected 1' % nprops |
346 assert nprops == 1, 'Filter has %s properties, expected 1' % nprops |
273 |
347 |
274 if ancestor and not kind and not filters and not orders: |
348 if not kind: |
275 required = False |
349 required = False |
|
350 |
|
351 (filters, orders) = RemoveNativelySupportedComponents(filters, orders) |
276 |
352 |
277 eq_filters = [f for f in filters if f.op() in EQUALITY_OPERATORS] |
353 eq_filters = [f for f in filters if f.op() in EQUALITY_OPERATORS] |
278 ineq_filters = [f for f in filters if f.op() in INEQUALITY_OPERATORS] |
354 ineq_filters = [f for f in filters if f.op() in INEQUALITY_OPERATORS] |
279 exists_filters = [f for f in filters if f.op() in EXISTS_OPERATORS] |
355 exists_filters = [f for f in filters if f.op() in EXISTS_OPERATORS] |
280 assert (len(eq_filters) + len(ineq_filters) + |
356 assert (len(eq_filters) + len(ineq_filters) + |
281 len(exists_filters)) == len(filters), 'Not all filters used' |
357 len(exists_filters)) == len(filters), 'Not all filters used' |
282 |
358 |
283 if (kind and eq_filters and not ineq_filters and not exists_filters and |
359 if (kind and not ineq_filters and not exists_filters and |
284 not orders): |
360 not orders): |
285 names = set(f.property(0).name() for f in eq_filters) |
361 names = set(f.property(0).name() for f in eq_filters) |
286 if not names.intersection(datastore_types._SPECIAL_PROPERTIES): |
362 if not names.intersection(datastore_types._SPECIAL_PROPERTIES): |
287 required = False |
363 required = False |
288 |
364 |
289 ineq_property = None |
365 ineq_property = None |
290 if ineq_filters: |
366 if ineq_filters: |
291 ineq_property = ineq_filters[0].property(0).name() |
367 ineq_property = ineq_filters[0].property(0).name() |
292 for filter in ineq_filters: |
368 for filter in ineq_filters: |
293 assert filter.property(0).name() == ineq_property |
369 assert filter.property(0).name() == ineq_property |
294 |
|
295 new_orders = [] |
|
296 for order in orders: |
|
297 name = order.property() |
|
298 for filter in eq_filters: |
|
299 if filter.property(0).name() == name: |
|
300 break |
|
301 else: |
|
302 new_orders.append(order) |
|
303 orders = new_orders |
|
304 |
370 |
305 props = [] |
371 props = [] |
306 |
372 |
307 for f in eq_filters: |
373 for f in eq_filters: |
308 prop = f.property(0) |
374 prop = f.property(0) |