author | Daniel Diniz <ajaksu@gmail.com> |
Fri, 03 Jul 2009 12:10:24 +0200 | |
changeset 2500 | ecc16ffe174b |
parent 686 | df109be0567c |
child 3031 | 7678f72140e6 |
permissions | -rwxr-xr-x |
109
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
1 |
#!/usr/bin/env python |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
2 |
# |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
3 |
# Copyright 2007 Google Inc. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
4 |
# |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
5 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
6 |
# you may not use this file except in compliance with the License. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
7 |
# You may obtain a copy of the License at |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
8 |
# |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
9 |
# http://www.apache.org/licenses/LICENSE-2.0 |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
10 |
# |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
11 |
# Unless required by applicable law or agreed to in writing, software |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
12 |
# distributed under the License is distributed on an "AS IS" BASIS, |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
13 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
14 |
# See the License for the specific language governing permissions and |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
15 |
# limitations under the License. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
16 |
# |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
17 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
18 |
"""Utilities for generating and updating index.yaml.""" |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
19 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
20 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
21 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
22 |
import os |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
23 |
import logging |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
24 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
25 |
from google.appengine.api import apiproxy_stub_map |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
26 |
from google.appengine.api import datastore_admin |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
27 |
from google.appengine.api import yaml_errors |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
28 |
from google.appengine.datastore import datastore_index |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
29 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
30 |
import yaml |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
31 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
32 |
AUTO_MARKER = '\n# AUTOGENERATED\n' |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
33 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
34 |
AUTO_COMMENT = ''' |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
35 |
# This index.yaml is automatically updated whenever the dev_appserver |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
36 |
# detects that a new type of query is run. If you want to manage the |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
37 |
# index.yaml file manually, remove the above marker line (the line |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
38 |
# saying "# AUTOGENERATED"). If you want to manage some indexes |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
39 |
# manually, move them above the marker line. The index.yaml file is |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
40 |
# automatically uploaded to the admin console when you next deploy |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
41 |
# your application using appcfg.py. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
42 |
''' |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
43 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
44 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
45 |
def GenerateIndexFromHistory(query_history, |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
46 |
all_indexes=None, manual_indexes=None): |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
47 |
"""Generate most of the text for index.yaml from the query history. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
48 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
49 |
Args: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
50 |
query_history: Query history, a dict mapping query |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
51 |
all_indexes: Optional datastore_index.IndexDefinitions instance |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
52 |
representing all the indexes found in the input file. May be None. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
53 |
manual_indexes: Optional datastore_index.IndexDefinitions instance |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
54 |
containing indexes for which we should not generate output. May be None. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
55 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
56 |
Returns: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
57 |
A string representation that can safely be appended to an |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
58 |
existing index.yaml file. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
59 |
""" |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
60 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
61 |
all_keys = datastore_index.IndexDefinitionsToKeys(all_indexes) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
62 |
manual_keys = datastore_index.IndexDefinitionsToKeys(manual_indexes) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
63 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
64 |
indexes = dict((key, 0) for key in all_keys - manual_keys) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
65 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
66 |
for query, count in query_history.iteritems(): |
686
df109be0567c
Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents:
109
diff
changeset
|
67 |
required, kind, ancestor, props, num_eq_filters = datastore_index.CompositeIndexForQuery(query) |
df109be0567c
Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents:
109
diff
changeset
|
68 |
if required: |
df109be0567c
Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents:
109
diff
changeset
|
69 |
key = (kind, ancestor, props) |
109
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
70 |
if key not in manual_keys: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
71 |
if key in indexes: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
72 |
indexes[key] += count |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
73 |
else: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
74 |
indexes[key] = count |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
75 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
76 |
res = [] |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
77 |
for (kind, ancestor, props), count in sorted(indexes.iteritems()): |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
78 |
res.append('') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
79 |
if count == 0: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
80 |
message = '# Unused in query history -- copied from input.' |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
81 |
elif count == 1: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
82 |
message = '# Used once in query history.' |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
83 |
else: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
84 |
message = '# Used %d times in query history.' % count |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
85 |
res.append(message) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
86 |
res.append(datastore_index.IndexYamlForQuery(kind, ancestor, props)) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
87 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
88 |
res.append('') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
89 |
return '\n'.join(res) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
90 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
91 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
92 |
class IndexYamlUpdater(object): |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
93 |
"""Helper class for updating index.yaml. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
94 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
95 |
This class maintains some state about the query history and the |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
96 |
index.yaml file in order to minimize the number of times index.yaml |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
97 |
is actually overwritten. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
98 |
""" |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
99 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
100 |
index_yaml_is_manual = False |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
101 |
index_yaml_mtime = 0 |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
102 |
last_history_size = 0 |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
103 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
104 |
def __init__(self, root_path): |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
105 |
"""Constructor. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
106 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
107 |
Args: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
108 |
root_path: Path to the app's root directory. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
109 |
""" |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
110 |
self.root_path = root_path |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
111 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
112 |
def UpdateIndexYaml(self, openfile=open): |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
113 |
"""Update index.yaml. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
114 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
115 |
Args: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
116 |
openfile: Used for dependency injection. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
117 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
118 |
We only ever write to index.yaml if either: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
119 |
- it doesn't exist yet; or |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
120 |
- it contains an 'AUTOGENERATED' comment. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
121 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
122 |
All indexes *before* the AUTOGENERATED comment will be written |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
123 |
back unchanged. All indexes *after* the AUTOGENERATED comment |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
124 |
will be updated with the latest query counts (query counts are |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
125 |
reset by --clear_datastore). Indexes that aren't yet in the file |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
126 |
will be appended to the AUTOGENERATED section. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
127 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
128 |
We keep track of some data in order to avoid doing repetitive work: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
129 |
- if index.yaml is fully manual, we keep track of its mtime to |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
130 |
avoid parsing it over and over; |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
131 |
- we keep track of the number of keys in the history dict since |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
132 |
the last time we updated index.yaml (or decided there was |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
133 |
nothing to update). |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
134 |
""" |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
135 |
index_yaml_file = os.path.join(self.root_path, 'index.yaml') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
136 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
137 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
138 |
index_yaml_mtime = os.path.getmtime(index_yaml_file) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
139 |
except os.error: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
140 |
index_yaml_mtime = None |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
141 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
142 |
index_yaml_changed = (index_yaml_mtime != self.index_yaml_mtime) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
143 |
self.index_yaml_mtime = index_yaml_mtime |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
144 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
145 |
datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
146 |
query_history = datastore_stub.QueryHistory() |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
147 |
history_changed = (len(query_history) != self.last_history_size) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
148 |
self.last_history_size = len(query_history) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
149 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
150 |
if not (index_yaml_changed or history_changed): |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
151 |
logging.debug('No need to update index.yaml') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
152 |
return |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
153 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
154 |
if self.index_yaml_is_manual and not index_yaml_changed: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
155 |
logging.debug('Will not update manual index.yaml') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
156 |
return |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
157 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
158 |
if index_yaml_mtime is None: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
159 |
index_yaml_data = None |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
160 |
else: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
161 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
162 |
fh = open(index_yaml_file, 'r') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
163 |
except IOError: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
164 |
index_yaml_data = None |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
165 |
else: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
166 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
167 |
index_yaml_data = fh.read() |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
168 |
finally: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
169 |
fh.close() |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
170 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
171 |
self.index_yaml_is_manual = (index_yaml_data is not None and |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
172 |
AUTO_MARKER not in index_yaml_data) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
173 |
if self.index_yaml_is_manual: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
174 |
logging.info('Detected manual index.yaml, will not update') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
175 |
return |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
176 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
177 |
if index_yaml_data is None: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
178 |
all_indexes = None |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
179 |
else: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
180 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
181 |
all_indexes = datastore_index.ParseIndexDefinitions(index_yaml_data) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
182 |
except yaml_errors.EventListenerError, e: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
183 |
logging.error('Error parsing %s:\n%s', index_yaml_file, e) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
184 |
return |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
185 |
except Exception, err: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
186 |
logging.error('Error parsing %s:\n%s.%s: %s', index_yaml_file, |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
187 |
err.__class__.__module__, err.__class__.__name__, err) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
188 |
return |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
189 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
190 |
if index_yaml_data is None: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
191 |
manual_part, automatic_part = 'indexes:\n', '' |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
192 |
manual_indexes = None |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
193 |
else: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
194 |
manual_part, automatic_part = index_yaml_data.split(AUTO_MARKER, 1) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
195 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
196 |
manual_indexes = datastore_index.ParseIndexDefinitions(manual_part) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
197 |
except Exception, err: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
198 |
logging.error('Error parsing manual part of %s: %s', |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
199 |
index_yaml_file, err) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
200 |
return |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
201 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
202 |
automatic_part = GenerateIndexFromHistory(query_history, |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
203 |
all_indexes, manual_indexes) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
204 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
205 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
206 |
fh = openfile(index_yaml_file, 'w') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
207 |
except IOError, err: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
208 |
logging.error('Can\'t write index.yaml: %s', err) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
209 |
return |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
210 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
211 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
212 |
logging.info('Updating %s', index_yaml_file) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
213 |
fh.write(manual_part) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
214 |
fh.write(AUTO_MARKER) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
215 |
fh.write(AUTO_COMMENT) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
216 |
fh.write(automatic_part) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
217 |
finally: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
218 |
fh.close() |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
219 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
220 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
221 |
self.index_yaml_mtime = os.path.getmtime(index_yaml_file) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
222 |
except os.error, err: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
223 |
logging.error('Can\'t stat index.yaml we just wrote: %s', err) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
224 |
self.index_yaml_mtime = None |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
225 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
226 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
227 |
def SetupIndexes(app_id, root_path): |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
228 |
"""Ensure that the set of existing composite indexes matches index.yaml. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
229 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
230 |
Note: this is similar to the algorithm used by the admin console for |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
231 |
the same purpose. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
232 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
233 |
Args: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
234 |
app_id: Application ID being served. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
235 |
root_path: Path to the root of the application. |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
236 |
""" |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
237 |
index_yaml_file = os.path.join(root_path, 'index.yaml') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
238 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
239 |
fh = open(index_yaml_file, 'r') |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
240 |
except IOError: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
241 |
index_yaml_data = None |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
242 |
else: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
243 |
try: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
244 |
index_yaml_data = fh.read() |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
245 |
finally: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
246 |
fh.close() |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
247 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
248 |
indexes = [] |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
249 |
if index_yaml_data is not None: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
250 |
index_defs = datastore_index.ParseIndexDefinitions(index_yaml_data) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
251 |
if index_defs is not None: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
252 |
indexes = index_defs.indexes |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
253 |
if indexes is None: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
254 |
indexes = [] |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
255 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
256 |
requested_indexes = datastore_admin.IndexDefinitionsToProtos(app_id, indexes) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
257 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
258 |
existing_indexes = datastore_admin.GetIndices(app_id) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
259 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
260 |
requested = dict((x.definition().Encode(), x) for x in requested_indexes) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
261 |
existing = dict((x.definition().Encode(), x) for x in existing_indexes) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
262 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
263 |
created = 0 |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
264 |
for key, index in requested.iteritems(): |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
265 |
if key not in existing: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
266 |
datastore_admin.CreateIndex(index) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
267 |
created += 1 |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
268 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
269 |
deleted = 0 |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
270 |
for key, index in existing.iteritems(): |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
271 |
if key not in requested: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
272 |
datastore_admin.DeleteIndex(index) |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
273 |
deleted += 1 |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
274 |
|
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
275 |
if created or deleted: |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
276 |
logging.info("Created %d and deleted %d index(es); total %d", |
620f9b141567
Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
277 |
created, deleted, len(requested)) |