# HG changeset patch # User Pawel Solyga # Date 1234441836 0 # Node ID a7766286a7bec44f3c3ba91cc5cde2cfc5f2d285 # Parent 5c931bd3dc1ec61dba8a7242c55c4957325a7d31 Load /Users/solydzajs/Downloads/google_appengine into trunk/thirdparty/google_appengine. diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/RELEASE_NOTES --- a/thirdparty/google_appengine/RELEASE_NOTES Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/RELEASE_NOTES Thu Feb 12 12:30:36 2009 +0000 @@ -3,6 +3,45 @@ App Engine SDK - Release Notes +Version 1.1.9 - February 2, 2009 +================================ + + - HTTP Request and Response limit raised to 10MB from 1MB. + Note that API call limits remain at 1MB. + http://code.google.com/p/googleappengine/issues/detail?id=78 + - urllib and urllib2 now available, implemented using urlfetch. + Also adds additional stubs which may enable other modules. + http://code.google.com/p/googleappengine/issues/detail?id=61 + http://code.google.com/p/googleappengine/issues/detail?id=68 + http://code.google.com/p/googleappengine/issues/detail?id=572 + http://code.google.com/p/googleappengine/issues/detail?id=821 + - Early release of a new data bulk upload tool, bulkloader.py + http://code.google.com/appengine/docs/python/tools/uploadingdata.html + - New remote_api for datastore at google.appengine.ext.remote_api + - Single property descending indexes are automatically generated. + - Added db.Query support for IN and != operators. + http://code.google.com/p/googleappengine/issues/detail?id=751 + - Fixed issue where gql date/time parsing could not handle Unicode strings. + - Fixed issue with db model instance key() returning the wrong key for + unsaved instances with parent as key + http://code.google.com/p/googleappengine/issues/detail?id=883 + - New run_in_transaction_custom_retries method for datastore. + - Fixed issue with relative dev_appserver datastore and history paths. + http://code.google.com/p/googleappengine/issues/detail?id=845 + - Static files and skipped files are not readable in dev_appserver, to match + the behavior on App Engine. + http://code.google.com/p/googleappengine/issues/detail?id=550 + - Images API allows multiple transforms of the same type in one request. A + limit of 10 total transforms per request has been added. + - PIL import will work with both PIL.Image and Image. + http://code.google.com/p/googleappengine/issues/detail?id=929 + - Fixed an issue with sending email in dev_appserver when the application + code changed. + http://code.google.com/p/googleappengine/issues/detail?id=182 + - Memcache counters (incr/decr) do nothing on non positive integers to match + the behavior on App Engine. + http://code.google.com/p/googleappengine/issues/detail?id=918 + Version 1.1.8 - January 7, 2008 ================================= - Skip_files RegexStr validator allows lists to for regex-ors. diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/VERSION --- a/thirdparty/google_appengine/VERSION Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/VERSION Thu Feb 12 12:30:36 2009 +0000 @@ -1,3 +1,3 @@ -release: "1.1.8" -timestamp: 1231809440 +release: "1.1.9" +timestamp: 1232676672 api_versions: ['1'] diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/appcfg.py --- a/thirdparty/google_appengine/appcfg.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/appcfg.py Thu Feb 12 12:30:36 2009 +0000 @@ -48,12 +48,12 @@ "dev_appserver.py" : "dev_appserver_main.py" } -def run_file(file_path, globals_): +def run_file(file_path, globals_, script_dir=SCRIPT_DIR): """Execute the file at the specified path with the passed-in globals.""" sys.path = EXTRA_PATHS + sys.path script_name = os.path.basename(file_path) script_name = SCRIPT_EXCEPTIONS.get(script_name, script_name) - script_path = os.path.join(SCRIPT_DIR, script_name) + script_path = os.path.join(script_dir, script_name) execfile(script_path, globals_) if __name__ == '__main__': diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/bulkload_client.py --- a/thirdparty/google_appengine/bulkload_client.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/bulkload_client.py Thu Feb 12 12:30:36 2009 +0000 @@ -48,12 +48,12 @@ "dev_appserver.py" : "dev_appserver_main.py" } -def run_file(file_path, globals_): +def run_file(file_path, globals_, script_dir=SCRIPT_DIR): """Execute the file at the specified path with the passed-in globals.""" sys.path = EXTRA_PATHS + sys.path script_name = os.path.basename(file_path) script_name = SCRIPT_EXCEPTIONS.get(script_name, script_name) - script_path = os.path.join(SCRIPT_DIR, script_name) + script_path = os.path.join(script_dir, script_name) execfile(script_path, globals_) if __name__ == '__main__': diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/bulkloader.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/bulkloader.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Convenience wrapper for starting an appengine tool.""" + + +import os +import sys + +if not hasattr(sys, 'version_info'): + sys.stderr.write('Very old versions of Python are not supported. Please ' + 'use version 2.5 or greater.\n') + sys.exit(1) +version_tuple = tuple(sys.version_info[:2]) +if version_tuple < (2, 4): + sys.stderr.write('Error: Python %d.%d is not supported. Please use ' + 'version 2.5 or greater.\n' % version_tuple) + sys.exit(1) +if version_tuple == (2, 4): + sys.stderr.write('Warning: Python 2.4 is not supported; this program may ' + 'break. Please use version 2.5 or greater.\n') + +DIR_PATH = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) +SCRIPT_DIR = os.path.join(DIR_PATH, 'google', 'appengine', 'tools') + +EXTRA_PATHS = [ + DIR_PATH, + os.path.join(DIR_PATH, 'lib', 'antlr3'), + os.path.join(DIR_PATH, 'lib', 'django'), + os.path.join(DIR_PATH, 'lib', 'webob'), + os.path.join(DIR_PATH, 'lib', 'yaml', 'lib'), +] + +SCRIPT_EXCEPTIONS = { + "dev_appserver.py" : "dev_appserver_main.py" +} + +def run_file(file_path, globals_, script_dir=SCRIPT_DIR): + """Execute the file at the specified path with the passed-in globals.""" + sys.path = EXTRA_PATHS + sys.path + script_name = os.path.basename(file_path) + script_name = SCRIPT_EXCEPTIONS.get(script_name, script_name) + script_path = os.path.join(script_dir, script_name) + execfile(script_path, globals_) + +if __name__ == '__main__': + run_file(__file__, globals()) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/dev_appserver.py --- a/thirdparty/google_appengine/dev_appserver.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/dev_appserver.py Thu Feb 12 12:30:36 2009 +0000 @@ -48,12 +48,12 @@ "dev_appserver.py" : "dev_appserver_main.py" } -def run_file(file_path, globals_): +def run_file(file_path, globals_, script_dir=SCRIPT_DIR): """Execute the file at the specified path with the passed-in globals.""" sys.path = EXTRA_PATHS + sys.path script_name = os.path.basename(file_path) script_name = SCRIPT_EXCEPTIONS.get(script_name, script_name) - script_path = os.path.join(SCRIPT_DIR, script_name) + script_path = os.path.join(script_dir, script_name) execfile(script_path, globals_) if __name__ == '__main__': diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/api_base_pb.py --- a/thirdparty/google_appengine/google/appengine/api/api_base_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/api_base_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -36,8 +36,9 @@ self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -116,8 +117,9 @@ self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = 0 + if self.has_value_: + self.has_value_ = 0 + self.value_ = 0 def has_value(self): return self.has_value_ @@ -196,8 +198,9 @@ self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = 0 + if self.has_value_: + self.has_value_ = 0 + self.value_ = 0 def has_value(self): return self.has_value_ @@ -276,8 +279,9 @@ self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = 0 + if self.has_value_: + self.has_value_ = 0 + self.value_ = 0 def has_value(self): return self.has_value_ @@ -355,8 +359,9 @@ self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = 0.0 + if self.has_value_: + self.has_value_ = 0 + self.value_ = 0.0 def has_value(self): return self.has_value_ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/apiproxy_rpc.py --- a/thirdparty/google_appengine/google/appengine/api/apiproxy_rpc.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/apiproxy_rpc.py Thu Feb 12 12:30:36 2009 +0000 @@ -37,7 +37,7 @@ FINISHING = 2 def __init__(self, package=None, call=None, request=None, response=None, - callback=None, stub=None): + callback=None, deadline=None, stub=None): """Constructor for the RPC object. All arguments are optional, and simply set members on the class. @@ -49,6 +49,8 @@ request: ProtocolMessage instance, appropriate for the arguments response: ProtocolMessage instance, appropriate for the response callback: callable, called when call is complete + deadline: A double specifying the deadline for this call as the number of + seconds from the current time. Ignored if non-positive. stub: APIProxyStub instance, used in default _WaitImpl to do real call """ self.__exception = None @@ -60,10 +62,11 @@ self.request = request self.response = response self.callback = callback + self.deadline = deadline self.stub = stub def MakeCall(self, package=None, call=None, request=None, response=None, - callback=None): + callback=None, deadline=None): """Makes an asynchronous (i.e. non-blocking) API call within the specified package for the specified call method. @@ -81,6 +84,7 @@ self.call = call or self.call self.request = request or self.request self.response = response or self.response + self.deadline = deadline or self.deadline assert self.__state is RPC.IDLE, ('RPC for %s.%s has already been started' % (self.package, self.call)) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/appinfo.py --- a/thirdparty/google_appengine/google/appengine/api/appinfo.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/appinfo.py Thu Feb 12 12:30:36 2009 +0000 @@ -56,6 +56,8 @@ RUNTIME_RE_STRING = r'[a-z]{1,30}' +API_VERSION_RE_STRING = r'[\w.]{1,32}' + HANDLER_STATIC_FILES = 'static_files' HANDLER_STATIC_DIR = 'static_dir' HANDLER_SCRIPT = 'script' @@ -307,7 +309,7 @@ RUNTIME: RUNTIME_RE_STRING, - API_VERSION: validation.Options('1', 'beta'), + API_VERSION: API_VERSION_RE_STRING, HANDLERS: validation.Optional(validation.Repeated(URLMap)), DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX), SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/capabilities/capability_service_pb.py --- a/thirdparty/google_appengine/google/appengine/api/capabilities/capability_service_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/capabilities/capability_service_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -39,8 +39,9 @@ self.package_ = x def clear_package(self): - self.has_package_ = 0 - self.package_ = "" + if self.has_package_: + self.has_package_ = 0 + self.package_ = "" def has_package(self): return self.has_package_ @@ -216,8 +217,9 @@ self.summary_status_ = x def clear_summary_status(self): - self.has_summary_status_ = 0 - self.summary_status_ = 0 + if self.has_summary_status_: + self.has_summary_status_ = 0 + self.summary_status_ = 0 def has_summary_status(self): return self.has_summary_status_ @@ -228,8 +230,9 @@ self.time_until_scheduled_ = x def clear_time_until_scheduled(self): - self.has_time_until_scheduled_ = 0 - self.time_until_scheduled_ = 0 + if self.has_time_until_scheduled_: + self.has_time_until_scheduled_ = 0 + self.time_until_scheduled_ = 0 def has_time_until_scheduled(self): return self.has_time_until_scheduled_ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/croninfo.py --- a/thirdparty/google_appengine/google/appengine/api/croninfo.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/croninfo.py Thu Feb 12 12:30:36 2009 +0000 @@ -23,6 +23,15 @@ +import logging +import sys +import traceback + +try: + import pytz +except ImportError: + pytz = None + from google.appengine.cron import groc from google.appengine.api import validation from google.appengine.api import yaml_builder @@ -30,10 +39,7 @@ from google.appengine.api import yaml_object _URL_REGEX = r'^/.*$' - - _TIMEZONE_REGEX = r'^.{0,100}$' - _DESCRIPTION_REGEX = r'^.{0,499}$' @@ -55,6 +61,31 @@ return value +class TimezoneValidator(validation.Validator): + """Checks that a timezone can be correctly parsed and is known.""" + + def Validate(self, value): + """Validates a timezone.""" + if value is None: + return + if not isinstance(value, basestring): + raise TypeError('timezone must be a string, not \'%r\'' % type(value)) + if pytz is None: + return value + try: + pytz.timezone(value) + except pytz.UnknownTimeZoneError: + raise validation.ValidationError('timezone \'%s\' is unknown' % value) + except IOError: + return value + except: + e, v, t = sys.exc_info() + logging.warning("pytz raised an unexpected error: %s.\n" % (v) + + "Traceback:\n" + "\n".join(traceback.format_tb(t))) + raise + return value + + CRON = 'cron' URL = 'url' @@ -73,7 +104,7 @@ ATTRIBUTES = { URL: _URL_REGEX, SCHEDULE: GrocValidator(), - TIMEZONE: validation.Optional(_TIMEZONE_REGEX), + TIMEZONE: TimezoneValidator(), DESCRIPTION: validation.Optional(_DESCRIPTION_REGEX) } diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/datastore.py --- a/thirdparty/google_appengine/google/appengine/api/datastore.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/datastore.py Thu Feb 12 12:30:36 2009 +0000 @@ -31,6 +31,8 @@ +import heapq +import itertools import logging import re import string @@ -47,7 +49,9 @@ from google.appengine.runtime import apiproxy_errors from google.appengine.datastore import entity_pb -TRANSACTION_RETRIES = 3 +MAX_ALLOWABLE_QUERIES = 30 + +DEFAULT_TRANSACTION_RETRIES = 3 _MAX_INDEXED_PROPERTIES = 5000 @@ -488,7 +492,7 @@ if isinstance(sample, list): sample = values[0] - if isinstance(sample, (datastore_types.Blob, datastore_types.Text)): + if isinstance(sample, datastore_types._RAW_PROPERTY_TYPES): pb.raw_property_list().extend(properties) else: pb.property_list().extend(properties) @@ -1072,12 +1076,9 @@ values = list(values) elif not isinstance(values, list): values = [values] - if isinstance(values[0], datastore_types.Blob): + if isinstance(values[0], datastore_types._RAW_PROPERTY_TYPES): raise datastore_errors.BadValueError( - 'Filtering on Blob properties is not supported.') - if isinstance(values[0], datastore_types.Text): - raise datastore_errors.BadValueError( - 'Filtering on Text properties is not supported.') + 'Filtering on %s properties is not supported.' % typename(values[0])) if operator in self.INEQUALITY_OPERATORS: if self.__inequality_prop and property != self.__inequality_prop: @@ -1165,6 +1166,306 @@ return pb +class MultiQuery(Query): + """Class representing a query which requires multiple datastore queries. + + This class is actually a subclass of datastore.Query as it is intended to act + like a normal Query object (supporting the same interface). + """ + + def __init__(self, bound_queries, orderings): + if len(bound_queries) > MAX_ALLOWABLE_QUERIES: + raise datastore_errors.BadArgumentError( + 'Cannot satisfy query -- too many subqueries (max: %d, got %d).' + ' Probable cause: too many IN/!= filters in query.' % + (MAX_ALLOWABLE_QUERIES, len(bound_queries))) + self.__bound_queries = bound_queries + self.__orderings = orderings + + def __str__(self): + res = 'MultiQuery: ' + for query in self.__bound_queries: + res = '%s %s' % (res, str(query)) + return res + + def Get(self, limit, offset=0): + """Get results of the query with a limit on the number of results. + + Args: + limit: maximum number of values to return. + offset: offset requested -- if nonzero, this will override the offset in + the original query + + Returns: + A list of entities with at most "limit" entries (less if the query + completes before reading limit values). + """ + count = 1 + result = [] + + iterator = self.Run() + + try: + for i in xrange(offset): + val = iterator.next() + except StopIteration: + pass + + try: + while count <= limit: + val = iterator.next() + result.append(val) + count += 1 + except StopIteration: + pass + return result + + class SortOrderEntity(object): + """Allow entity comparisons using provided orderings. + + The iterator passed to the constructor is eventually consumed via + calls to GetNext(), which generate new SortOrderEntity s with the + same orderings. + """ + + def __init__(self, entity_iterator, orderings): + """Ctor. + + Args: + entity_iterator: an iterator of entities which will be wrapped. + orderings: an iterable of (identifier, order) pairs. order + should be either Query.ASCENDING or Query.DESCENDING. + """ + self.__entity_iterator = entity_iterator + self.__entity = None + self.__min_max_value_cache = {} + try: + self.__entity = entity_iterator.next() + except StopIteration: + pass + else: + self.__orderings = orderings + + def __str__(self): + return str(self.__entity) + + def GetEntity(self): + """Gets the wrapped entity.""" + return self.__entity + + def GetNext(self): + """Wrap and return the next entity. + + The entity is retrieved from the iterator given at construction time. + """ + return MultiQuery.SortOrderEntity(self.__entity_iterator, + self.__orderings) + + def CmpProperties(self, that): + """Compare two entities and return their relative order. + + Compares self to that based on the current sort orderings and the + key orders between them. Returns negative, 0, or positive depending on + whether self is less, equal to, or greater than that. This + comparison returns as if all values were to be placed in ascending order + (highest value last). Only uses the sort orderings to compare (ignores + keys). + + Args: + that: SortOrderEntity + + Returns: + Negative if self < that + Zero if self == that + Positive if self > that + """ + if not self.__entity: + return cmp(self.__entity, that.__entity) + + for (identifier, order) in self.__orderings: + value1 = self.__GetValueForId(self, identifier, order) + value2 = self.__GetValueForId(that, identifier, order) + + result = cmp(value1, value2) + if order == Query.DESCENDING: + result = -result + if result: + return result + return 0 + + def __GetValueForId(self, sort_order_entity, identifier, sort_order): + value = sort_order_entity.__entity[identifier] + entity_key = sort_order_entity.__entity.key() + if (entity_key, identifier) in self.__min_max_value_cache: + value = self.__min_max_value_cache[(entity_key, identifier)] + elif isinstance(value, list): + if sort_order == Query.DESCENDING: + value = min(value) + else: + value = max(value) + self.__min_max_value_cache[(entity_key, identifier)] = value + + return value + + def __cmp__(self, that): + """Compare self to that w.r.t. values defined in the sort order. + + Compare an entity with another, using sort-order first, then the key + order to break ties. This can be used in a heap to have faster min-value + lookup. + + Args: + that: other entity to compare to + Returns: + negative: if self is less than that in sort order + zero: if self is equal to that in sort order + positive: if self is greater than that in sort order + """ + property_compare = self.CmpProperties(that) + if property_compare: + return property_compare + else: + return cmp(self.__entity.key(), that.__entity.key()) + + def Run(self): + """Return an iterable output with all results in order.""" + results = [] + count = 1 + log_level = logging.DEBUG - 1 + for bound_query in self.__bound_queries: + logging.log(log_level, 'Running query #%i' % count) + results.append(bound_query.Run()) + count += 1 + + def IterateResults(results): + """Iterator function to return all results in sorted order. + + Iterate over the array of results, yielding the next element, in + sorted order. This function is destructive (results will be empty + when the operation is complete). + + Args: + results: list of result iterators to merge and iterate through + + Yields: + The next result in sorted order. + """ + result_heap = [] + for result in results: + heap_value = MultiQuery.SortOrderEntity(result, self.__orderings) + if heap_value.GetEntity(): + heapq.heappush(result_heap, heap_value) + + used_keys = set() + + while result_heap: + top_result = heapq.heappop(result_heap) + + results_to_push = [] + if top_result.GetEntity().key() not in used_keys: + yield top_result.GetEntity() + else: + pass + + used_keys.add(top_result.GetEntity().key()) + + results_to_push = [] + while result_heap: + next = heapq.heappop(result_heap) + if cmp(top_result, next): + results_to_push.append(next) + break + else: + results_to_push.append(next.GetNext()) + results_to_push.append(top_result.GetNext()) + + for popped_result in results_to_push: + if popped_result.GetEntity(): + heapq.heappush(result_heap, popped_result) + + return IterateResults(results) + + def Count(self, limit=None): + """Return the number of matched entities for this query. + + Will return the de-duplicated count of results. Will call the more + efficient Get() function if a limit is given. + + Args: + limit: maximum number of entries to count (for any result > limit, return + limit). + Returns: + count of the number of entries returned. + """ + if limit is None: + count = 0 + for i in self.Run(): + count += 1 + return count + else: + return len(self.Get(limit)) + + def __setitem__(self, query_filter, value): + """Add a new filter by setting it on all subqueries. + + If any of the setting operations raise an exception, the ones + that succeeded are undone and the exception is propagated + upward. + + Args: + query_filter: a string of the form "property operand". + value: the value that the given property is compared against. + """ + saved_items = [] + for index, query in enumerate(self.__bound_queries): + saved_items.append(query.get(query_filter, None)) + try: + query[query_filter] = value + except: + for q, old_value in itertools.izip(self.__bound_queries[:index], + saved_items): + if old_value is not None: + q[query_filter] = old_value + else: + del q[query_filter] + raise + + def __delitem__(self, query_filter): + """Delete a filter by deleting it from all subqueries. + + If a KeyError is raised during the attempt, it is ignored, unless + every subquery raised a KeyError. If any other exception is + raised, any deletes will be rolled back. + + Args: + query_filter: the filter to delete. + + Raises: + KeyError: No subquery had an entry containing query_filter. + """ + subquery_count = len(self.__bound_queries) + keyerror_count = 0 + saved_items = [] + for index, query in enumerate(self.__bound_queries): + try: + saved_items.append(query.get(query_filter, None)) + del query[query_filter] + except KeyError: + keyerror_count += 1 + except: + for q, old_value in itertools.izip(self.__bound_queries[:index], + saved_items): + if old_value is not None: + q[query_filter] = old_value + raise + + if keyerror_count == subquery_count: + raise KeyError(query_filter) + + def __iter__(self): + return iter(self.__bound_queries) + + class Iterator(object): """An iterator over the results of a datastore query. @@ -1331,8 +1632,29 @@ self.modified_keys.update(keys) +def RunInTransaction(function, *args, **kwargs): + """Runs a function inside a datastore transaction. -def RunInTransaction(function, *args, **kwargs): + Runs the user-provided function inside transaction, retries default + number of times. + + Args: + # a function to be run inside the transaction + function: callable + # positional arguments to pass to the function + args: variable number of any type + + Returns: + the function's return value, if any + + Raises: + TransactionFailedError, if the transaction could not be committed. + """ + return RunInTransactionCustomRetries( + DEFAULT_TRANSACTION_RETRIES, function, *args, **kwargs) + + +def RunInTransactionCustomRetries(retries, function, *args, **kwargs): """Runs a function inside a datastore transaction. Runs the user-provided function inside a full-featured, ACID datastore @@ -1387,6 +1709,8 @@ Nested transactions are not supported. Args: + # number of retries + retries: integer # a function to be run inside the transaction function: callable # positional arguments to pass to the function @@ -1403,6 +1727,10 @@ raise datastore_errors.BadRequestError( 'Nested transactions are not supported.') + if retries < 0: + raise datastore_errors.BadRequestError( + 'Number of retries should be non-negative number.') + tx_key = None try: @@ -1410,7 +1738,7 @@ tx = _Transaction() _txes[tx_key] = tx - for i in range(0, TRANSACTION_RETRIES + 1): + for i in range(0, retries + 1): tx.modified_keys.clear() try: @@ -1436,7 +1764,7 @@ if tx.handle: try: - resp = api_base_pb.VoidProto() + resp = datastore_pb.CommitResponse() apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Commit', tx.handle, resp) except apiproxy_errors.ApplicationError, err: @@ -1544,7 +1872,7 @@ """Walks the stack to find a RunInTransaction() call. Returns: - # this is the RunInTransaction() frame record, if found + # this is the RunInTransactionCustomRetries() frame record, if found frame record or None """ frame = sys._getframe() @@ -1553,7 +1881,7 @@ frame = frame.f_back.f_back while frame: if (frame.f_code.co_filename == filename and - frame.f_code.co_name == 'RunInTransaction'): + frame.f_code.co_name == 'RunInTransactionCustomRetries'): return frame frame = frame.f_back diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/datastore_file_stub.py --- a/thirdparty/google_appengine/google/appengine/api/datastore_file_stub.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/datastore_file_stub.py Thu Feb 12 12:30:36 2009 +0000 @@ -128,6 +128,18 @@ users.User: entity_pb.PropertyValue.kUserValueGroup, } + WRITE_ONLY = entity_pb.CompositeIndex.WRITE_ONLY + READ_WRITE = entity_pb.CompositeIndex.READ_WRITE + DELETED = entity_pb.CompositeIndex.DELETED + ERROR = entity_pb.CompositeIndex.ERROR + + _INDEX_STATE_TRANSITIONS = { + WRITE_ONLY: frozenset((READ_WRITE, DELETED, ERROR)), + READ_WRITE: frozenset((DELETED,)), + ERROR: frozenset((DELETED,)), + DELETED: frozenset((ERROR,)), + } + def __init__(self, app_id, datastore_file, @@ -794,6 +806,7 @@ for name, value_pb in props.items(): prop_pb = kind_pb.add_property() prop_pb.set_name(name) + prop_pb.set_multiple(False) prop_pb.mutable_value().CopyFrom(value_pb) kinds.append(kind_pb) @@ -838,7 +851,8 @@ if not stored_index: raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, "Index doesn't exist.") - elif index.state() != stored_index.state() + 1: + elif (index.state() != stored_index.state() and + index.state() not in self._INDEX_STATE_TRANSITIONS[stored_index.state()]): raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, "cannot move index state from %s to %s" % diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/datastore_types.py --- a/thirdparty/google_appengine/google/appengine/api/datastore_types.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/datastore_types.py Thu Feb 12 12:30:36 2009 +0000 @@ -942,15 +942,7 @@ users.User, ]) -_RAW_PROPERTY_TYPES = frozenset([ - Blob, - Text, -]) - -_STRING_TYPES = frozenset([ - str, - unicode, -]) +_RAW_PROPERTY_TYPES = (Blob, Text) def ValidatePropertyInteger(name, value): """Raises an exception if the supplied integer is invalid. @@ -1143,7 +1135,7 @@ def DatetimeToTimestamp(value): - """Converts a datetime.datetime to seconds since the epoch, as a float. + """Converts a datetime.datetime to microseconds since the epoch, as a float. Args: value: datetime.datetime diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/images/__init__.py --- a/thirdparty/google_appengine/google/appengine/api/images/__init__.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/images/__init__.py Thu Feb 12 12:30:36 2009 +0000 @@ -42,6 +42,8 @@ OUTPUT_ENCODING_TYPES = frozenset([JPEG, PNG]) +MAX_TRANSFORMS_PER_REQUEST = 10 + class Error(Exception): """Base error class for this module.""" @@ -84,29 +86,19 @@ self._image_data = image_data self._transforms = [] - self._transform_map = {} self._width = None self._height = None - def _check_transform_limits(self, transform): + def _check_transform_limits(self): """Ensure some simple limits on the number of transforms allowed. - Args: - transform: images_service_pb.ImagesServiceTransform, enum of the - trasnform called. - Raises: - BadRequestError if the transform has already been requested for the image. + BadRequestError if MAX_TRANSFORMS_PER_REQUEST transforms have already been + requested for this image """ - if not images_service_pb.ImagesServiceTransform.Type_Name(transform): - raise BadRequestError("'%s' is not a valid transform." % transform) - - if transform in self._transform_map: - transform_name = images_service_pb.ImagesServiceTransform.Type_Name( - transform) - raise BadRequestError("A '%s' transform has already been " - "requested on this image." % transform_name) - self._transform_map[transform] = True + if len(self._transforms) >= MAX_TRANSFORMS_PER_REQUEST: + raise BadRequestError("%d transforms have already been requested on this " + "image." % MAX_TRANSFORMS_PER_REQUEST) def _update_dimensions(self): """Updates the width and height fields of the image. @@ -172,14 +164,14 @@ if (offset < size and ord(self._image_data[offset]) & 0xF0 == 0xC0 and ord(self._image_data[offset]) != 0xC4): offset += 4 - if offset + 4 < size: + if offset + 4 <= size: self._height, self._width = struct.unpack( ">HH", self._image_data[offset:offset + 4]) break else: raise BadImageError("Corrupt JPEG format") - elif offset + 2 <= size: + elif offset + 3 <= size: offset += 1 offset += struct.unpack(">H", self._image_data[offset:offset + 2])[0] else: @@ -199,7 +191,7 @@ else: endianness = ">" ifd_offset = struct.unpack(endianness + "I", self._image_data[4:8])[0] - if ifd_offset < size + 14: + if ifd_offset + 14 <= size: ifd_size = struct.unpack( endianness + "H", self._image_data[ifd_offset:ifd_offset + 2])[0] @@ -291,7 +283,8 @@ Raises: TypeError when width or height is not either 'int' or 'long' types. BadRequestError when there is something wrong with the given height or - width or if a Resize has already been requested on this image. + width or if MAX_TRANSFORMS_PER_REQUEST transforms have already been + requested on this image. """ if (not isinstance(width, (int, long)) or not isinstance(height, (int, long))): @@ -305,8 +298,7 @@ if width > 4000 or height > 4000: raise BadRequestError("Both width and height must be < 4000.") - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.RESIZE) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_width(width) @@ -323,7 +315,7 @@ Raises: TypeError when degrees is not either 'int' or 'long' types. BadRequestError when there is something wrong with the given degrees or - if a Rotate trasnform has already been requested. + if MAX_TRANSFORMS_PER_REQUEST transforms have already been requested. """ if not isinstance(degrees, (int, long)): raise TypeError("Degrees must be integers.") @@ -333,8 +325,7 @@ degrees = degrees % 360 - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.ROTATE) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_rotate(degrees) @@ -345,11 +336,10 @@ """Flip the image horizontally. Raises: - BadRequestError if a HorizontalFlip has already been requested on the - image. + BadRequestError if MAX_TRANSFORMS_PER_REQUEST transforms have already been + requested on the image. """ - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.HORIZONTAL_FLIP) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_horizontal_flip(True) @@ -360,11 +350,10 @@ """Flip the image vertically. Raises: - BadRequestError if a HorizontalFlip has already been requested on the - image. + BadRequestError if MAX_TRANSFORMS_PER_REQUEST transforms have already been + requested on the image. """ - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.VERTICAL_FLIP) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_vertical_flip(True) @@ -405,7 +394,8 @@ Raises: TypeError if the args are not of type 'float'. BadRequestError when there is something wrong with the given bounding box - or if there has already been a crop transform requested for this image. + or if MAX_TRANSFORMS_PER_REQUEST transforms have already been requested + for this image. """ self._validate_crop_arg(left_x, "left_x") self._validate_crop_arg(top_y, "top_y") @@ -417,7 +407,7 @@ if top_y >= bottom_y: raise BadRequestError("top_y must be less than bottom_y") - self._check_transform_limits(images_service_pb.ImagesServiceTransform.CROP) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_crop_left_x(left_x) @@ -433,11 +423,10 @@ This is similar to the "I'm Feeling Lucky" button in Picasa. Raises: - BadRequestError if this transform has already been requested for this - image. + BadRequestError if MAX_TRANSFORMS_PER_REQUEST transforms have already + been requested for this image. """ - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.IM_FEELING_LUCKY) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_autolevels(True) @@ -504,7 +493,6 @@ self._image_data = response.image().content() self._transforms = [] - self._transform_map.clear() self._width = None self._height = None return self._image_data @@ -540,7 +528,7 @@ Raises: TypeError when width or height not either 'int' or 'long' types. BadRequestError when there is something wrong with the given height or - width or if a Resize has already been requested on this image. + width. Error when something went wrong with the call. See Image.ExecuteTransforms for more details. """ @@ -559,8 +547,7 @@ Raises: TypeError when degrees is not either 'int' or 'long' types. - BadRequestError when there is something wrong with the given degrees or - if a Rotate trasnform has already been requested. + BadRequestError when there is something wrong with the given degrees. Error when something went wrong with the call. See Image.ExecuteTransforms for more details. """ @@ -619,8 +606,7 @@ Raises: TypeError if the args are not of type 'float'. - BadRequestError when there is something wrong with the given bounding box - or if there has already been a crop transform requested for this image. + BadRequestError when there is something wrong with the given bounding box. Error when something went wrong with the call. See Image.ExecuteTransforms for more details. """ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/images/images_service_pb.py --- a/thirdparty/google_appengine/google/appengine/api/images/images_service_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/images/images_service_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -192,8 +192,9 @@ self.width_ = x def clear_width(self): - self.has_width_ = 0 - self.width_ = 0 + if self.has_width_: + self.has_width_ = 0 + self.width_ = 0 def has_width(self): return self.has_width_ @@ -204,8 +205,9 @@ self.height_ = x def clear_height(self): - self.has_height_ = 0 - self.height_ = 0 + if self.has_height_: + self.has_height_ = 0 + self.height_ = 0 def has_height(self): return self.has_height_ @@ -216,8 +218,9 @@ self.rotate_ = x def clear_rotate(self): - self.has_rotate_ = 0 - self.rotate_ = 0 + if self.has_rotate_: + self.has_rotate_ = 0 + self.rotate_ = 0 def has_rotate(self): return self.has_rotate_ @@ -228,8 +231,9 @@ self.horizontal_flip_ = x def clear_horizontal_flip(self): - self.has_horizontal_flip_ = 0 - self.horizontal_flip_ = 0 + if self.has_horizontal_flip_: + self.has_horizontal_flip_ = 0 + self.horizontal_flip_ = 0 def has_horizontal_flip(self): return self.has_horizontal_flip_ @@ -240,8 +244,9 @@ self.vertical_flip_ = x def clear_vertical_flip(self): - self.has_vertical_flip_ = 0 - self.vertical_flip_ = 0 + if self.has_vertical_flip_: + self.has_vertical_flip_ = 0 + self.vertical_flip_ = 0 def has_vertical_flip(self): return self.has_vertical_flip_ @@ -252,8 +257,9 @@ self.crop_left_x_ = x def clear_crop_left_x(self): - self.has_crop_left_x_ = 0 - self.crop_left_x_ = 0.0 + if self.has_crop_left_x_: + self.has_crop_left_x_ = 0 + self.crop_left_x_ = 0.0 def has_crop_left_x(self): return self.has_crop_left_x_ @@ -264,8 +270,9 @@ self.crop_top_y_ = x def clear_crop_top_y(self): - self.has_crop_top_y_ = 0 - self.crop_top_y_ = 0.0 + if self.has_crop_top_y_: + self.has_crop_top_y_ = 0 + self.crop_top_y_ = 0.0 def has_crop_top_y(self): return self.has_crop_top_y_ @@ -276,8 +283,9 @@ self.crop_right_x_ = x def clear_crop_right_x(self): - self.has_crop_right_x_ = 0 - self.crop_right_x_ = 1.0 + if self.has_crop_right_x_: + self.has_crop_right_x_ = 0 + self.crop_right_x_ = 1.0 def has_crop_right_x(self): return self.has_crop_right_x_ @@ -288,8 +296,9 @@ self.crop_bottom_y_ = x def clear_crop_bottom_y(self): - self.has_crop_bottom_y_ = 0 - self.crop_bottom_y_ = 1.0 + if self.has_crop_bottom_y_: + self.has_crop_bottom_y_ = 0 + self.crop_bottom_y_ = 1.0 def has_crop_bottom_y(self): return self.has_crop_bottom_y_ @@ -300,8 +309,9 @@ self.autolevels_ = x def clear_autolevels(self): - self.has_autolevels_ = 0 - self.autolevels_ = 0 + if self.has_autolevels_: + self.has_autolevels_ = 0 + self.autolevels_ = 0 def has_autolevels(self): return self.has_autolevels_ @@ -521,8 +531,9 @@ self.content_ = x def clear_content(self): - self.has_content_ = 0 - self.content_ = "" + if self.has_content_: + self.has_content_ = 0 + self.content_ = "" def has_content(self): return self.has_content_ @@ -613,8 +624,9 @@ self.mime_type_ = x def clear_mime_type(self): - self.has_mime_type_ = 0 - self.mime_type_ = 0 + if self.has_mime_type_: + self.has_mime_type_ = 0 + self.mime_type_ = 0 def has_mime_type(self): return self.has_mime_type_ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/images/images_stub.py --- a/thirdparty/google_appengine/google/appengine/api/images/images_stub.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/images/images_stub.py Thu Feb 12 12:30:36 2009 +0000 @@ -22,9 +22,13 @@ import logging import StringIO -import PIL -from PIL import _imaging -from PIL import Image +try: + import PIL + from PIL import _imaging + from PIL import Image +except ImportError: + import _imaging + import Image from google.appengine.api import apiproxy_stub from google.appengine.api import images @@ -246,24 +250,6 @@ return image.crop(box) - def _CheckTransformCount(self, transform_map, req_transform): - """Check that the requested transform hasn't already been set in map. - - Args: - transform_map: {images_service_pb.ImagesServiceTransform: boolean}, map - to use to determine if the requested transform has been called. - req_transform: images_service_pb.ImagesServiceTransform, the requested - transform. - - Raises: - BadRequestError if we are passed more than one of the same type of - transform. - """ - if req_transform in transform_map: - raise apiproxy_errors.ApplicationError( - images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) - transform_map[req_transform] = True - def _ProcessTransforms(self, image, transforms): """Execute PIL operations based on transform values. @@ -279,56 +265,29 @@ transform. """ new_image = image - transform_map = {} + if len(transforms) > images.MAX_TRANSFORMS_PER_REQUEST: + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) for transform in transforms: if transform.has_width() or transform.has_height(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.RESIZE - ) - new_image = self._Resize(new_image, transform) elif transform.has_rotate(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.ROTATE - ) - new_image = self._Rotate(new_image, transform) elif transform.has_horizontal_flip(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.HORIZONTAL_FLIP - ) - new_image = new_image.transpose(Image.FLIP_LEFT_RIGHT) elif transform.has_vertical_flip(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.VERTICAL_FLIP - ) - new_image = new_image.transpose(Image.FLIP_TOP_BOTTOM) elif (transform.has_crop_left_x() or transform.has_crop_top_y() or transform.has_crop_right_x() or transform.has_crop_bottom_y()): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.CROP - ) - new_image = self._Crop(new_image, transform) elif transform.has_autolevels(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.IM_FEELING_LUCKY - ) logging.info("I'm Feeling Lucky autolevels will be visible once this " "application is deployed.") else: diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/mail_service_pb.py --- a/thirdparty/google_appengine/google/appengine/api/mail_service_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/mail_service_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -107,8 +107,9 @@ self.filename_ = x def clear_filename(self): - self.has_filename_ = 0 - self.filename_ = "" + if self.has_filename_: + self.has_filename_ = 0 + self.filename_ = "" def has_filename(self): return self.has_filename_ @@ -119,8 +120,9 @@ self.data_ = x def clear_data(self): - self.has_data_ = 0 - self.data_ = "" + if self.has_data_: + self.has_data_ = 0 + self.data_ = "" def has_data(self): return self.has_data_ @@ -230,8 +232,9 @@ self.sender_ = x def clear_sender(self): - self.has_sender_ = 0 - self.sender_ = "" + if self.has_sender_: + self.has_sender_ = 0 + self.sender_ = "" def has_sender(self): return self.has_sender_ @@ -242,8 +245,9 @@ self.replyto_ = x def clear_replyto(self): - self.has_replyto_ = 0 - self.replyto_ = "" + if self.has_replyto_: + self.has_replyto_ = 0 + self.replyto_ = "" def has_replyto(self): return self.has_replyto_ @@ -299,8 +303,9 @@ self.subject_ = x def clear_subject(self): - self.has_subject_ = 0 - self.subject_ = "" + if self.has_subject_: + self.has_subject_ = 0 + self.subject_ = "" def has_subject(self): return self.has_subject_ @@ -311,8 +316,9 @@ self.textbody_ = x def clear_textbody(self): - self.has_textbody_ = 0 - self.textbody_ = "" + if self.has_textbody_: + self.has_textbody_ = 0 + self.textbody_ = "" def has_textbody(self): return self.has_textbody_ @@ -323,8 +329,9 @@ self.htmlbody_ = x def clear_htmlbody(self): - self.has_htmlbody_ = 0 - self.htmlbody_ = "" + if self.has_htmlbody_: + self.has_htmlbody_ = 0 + self.htmlbody_ = "" def has_htmlbody(self): return self.has_htmlbody_ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/mail_stub.py --- a/thirdparty/google_appengine/google/appengine/api/mail_stub.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/mail_stub.py Thu Feb 12 12:30:36 2009 +0000 @@ -196,6 +196,8 @@ if self._smtp_host and self._enable_sendmail: log('Both SMTP and sendmail are enabled. Ignoring sendmail.') + import email + mime_message = mail.MailMessageToMIMEMessage(request) if self._smtp_host: self._SendSMTP(mime_message, smtp_lib) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/memcache/memcache_service_pb.py --- a/thirdparty/google_appengine/google/appengine/api/memcache/memcache_service_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/memcache/memcache_service_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -189,8 +189,9 @@ self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -201,8 +202,9 @@ self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -213,8 +215,9 @@ self.flags_ = x def clear_flags(self): - self.has_flags_ = 0 - self.flags_ = 0 + if self.has_flags_: + self.has_flags_ = 0 + self.flags_ = 0 def has_flags(self): return self.has_flags_ @@ -418,8 +421,9 @@ self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -430,8 +434,9 @@ self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -442,8 +447,9 @@ self.flags_ = x def clear_flags(self): - self.has_flags_ = 0 - self.flags_ = 0 + if self.has_flags_: + self.has_flags_ = 0 + self.flags_ = 0 def has_flags(self): return self.has_flags_ @@ -454,8 +460,9 @@ self.set_policy_ = x def clear_set_policy(self): - self.has_set_policy_ = 0 - self.set_policy_ = 1 + if self.has_set_policy_: + self.has_set_policy_ = 0 + self.set_policy_ = 1 def has_set_policy(self): return self.has_set_policy_ @@ -466,8 +473,9 @@ self.expiration_time_ = x def clear_expiration_time(self): - self.has_expiration_time_ = 0 - self.expiration_time_ = 0 + if self.has_expiration_time_: + self.has_expiration_time_ = 0 + self.expiration_time_ = 0 def has_expiration_time(self): return self.has_expiration_time_ @@ -811,8 +819,9 @@ self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -823,8 +832,9 @@ self.delete_time_ = x def clear_delete_time(self): - self.has_delete_time_ = 0 - self.delete_time_ = 0 + if self.has_delete_time_: + self.has_delete_time_ = 0 + self.delete_time_ = 0 def has_delete_time(self): return self.has_delete_time_ @@ -1115,8 +1125,9 @@ self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -1127,8 +1138,9 @@ self.delta_ = x def clear_delta(self): - self.has_delta_ = 0 - self.delta_ = 1 + if self.has_delta_: + self.has_delta_ = 0 + self.delta_ = 1 def has_delta(self): return self.has_delta_ @@ -1139,8 +1151,9 @@ self.direction_ = x def clear_direction(self): - self.has_direction_ = 0 - self.direction_ = 1 + if self.has_direction_: + self.has_direction_ = 0 + self.direction_ = 1 def has_direction(self): return self.has_direction_ @@ -1251,8 +1264,9 @@ self.new_value_ = x def clear_new_value(self): - self.has_new_value_ = 0 - self.new_value_ = 0 + if self.has_new_value_: + self.has_new_value_ = 0 + self.new_value_ = 0 def has_new_value(self): return self.has_new_value_ @@ -1488,8 +1502,9 @@ self.hits_ = x def clear_hits(self): - self.has_hits_ = 0 - self.hits_ = 0 + if self.has_hits_: + self.has_hits_ = 0 + self.hits_ = 0 def has_hits(self): return self.has_hits_ @@ -1500,8 +1515,9 @@ self.misses_ = x def clear_misses(self): - self.has_misses_ = 0 - self.misses_ = 0 + if self.has_misses_: + self.has_misses_ = 0 + self.misses_ = 0 def has_misses(self): return self.has_misses_ @@ -1512,8 +1528,9 @@ self.byte_hits_ = x def clear_byte_hits(self): - self.has_byte_hits_ = 0 - self.byte_hits_ = 0 + if self.has_byte_hits_: + self.has_byte_hits_ = 0 + self.byte_hits_ = 0 def has_byte_hits(self): return self.has_byte_hits_ @@ -1524,8 +1541,9 @@ self.items_ = x def clear_items(self): - self.has_items_ = 0 - self.items_ = 0 + if self.has_items_: + self.has_items_ = 0 + self.items_ = 0 def has_items(self): return self.has_items_ @@ -1536,8 +1554,9 @@ self.bytes_ = x def clear_bytes(self): - self.has_bytes_ = 0 - self.bytes_ = 0 + if self.has_bytes_: + self.has_bytes_ = 0 + self.bytes_ = 0 def has_bytes(self): return self.has_bytes_ @@ -1548,8 +1567,9 @@ self.oldest_item_age_ = x def clear_oldest_item_age(self): - self.has_oldest_item_age_ = 0 - self.oldest_item_age_ = 0 + if self.has_oldest_item_age_: + self.has_oldest_item_age_ = 0 + self.oldest_item_age_ = 0 def has_oldest_item_age(self): return self.has_oldest_item_age_ @@ -1728,8 +1748,9 @@ def mutable_stats(self): self.has_stats_ = 1; return self.stats() def clear_stats(self): - self.has_stats_ = 0; - if self.stats_ is not None: self.stats_.Clear() + if self.has_stats_: + self.has_stats_ = 0; + if self.stats_ is not None: self.stats_.Clear() def has_stats(self): return self.has_stats_ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/memcache/memcache_stub.py --- a/thirdparty/google_appengine/google/appengine/api/memcache/memcache_stub.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/memcache/memcache_stub.py Thu Feb 12 12:30:36 2009 +0000 @@ -223,9 +223,11 @@ try: old_value = long(entry.value) + if old_value < 0: + raise ValueError except ValueError, e: logging.error('Increment/decrement failed: Could not interpret ' - 'value for key = "%s" as an integer.', key) + 'value for key = "%s" as an unsigned integer.', key) return delta = request.delta() diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/urlfetch_service_pb.py --- a/thirdparty/google_appengine/google/appengine/api/urlfetch_service_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/urlfetch_service_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -109,8 +109,9 @@ self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -121,8 +122,9 @@ self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -227,8 +229,9 @@ self.method_ = x def clear_method(self): - self.has_method_ = 0 - self.method_ = 0 + if self.has_method_: + self.has_method_ = 0 + self.method_ = 0 def has_method(self): return self.has_method_ @@ -239,8 +242,9 @@ self.url_ = x def clear_url(self): - self.has_url_ = 0 - self.url_ = "" + if self.has_url_: + self.has_url_ = 0 + self.url_ = "" def has_url(self): return self.has_url_ @@ -267,8 +271,9 @@ self.payload_ = x def clear_payload(self): - self.has_payload_ = 0 - self.payload_ = "" + if self.has_payload_: + self.has_payload_ = 0 + self.payload_ = "" def has_payload(self): return self.has_payload_ @@ -279,8 +284,9 @@ self.followredirects_ = x def clear_followredirects(self): - self.has_followredirects_ = 0 - self.followredirects_ = 1 + if self.has_followredirects_: + self.has_followredirects_ = 0 + self.followredirects_ = 1 def has_followredirects(self): return self.has_followredirects_ @@ -448,8 +454,9 @@ self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -460,8 +467,9 @@ self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -546,8 +554,9 @@ self.content_ = x def clear_content(self): - self.has_content_ = 0 - self.content_ = "" + if self.has_content_: + self.has_content_ = 0 + self.content_ = "" def has_content(self): return self.has_content_ @@ -558,8 +567,9 @@ self.statuscode_ = x def clear_statuscode(self): - self.has_statuscode_ = 0 - self.statuscode_ = 0 + if self.has_statuscode_: + self.has_statuscode_ = 0 + self.statuscode_ = 0 def has_statuscode(self): return self.has_statuscode_ @@ -586,8 +596,9 @@ self.contentwastruncated_ = x def clear_contentwastruncated(self): - self.has_contentwastruncated_ = 0 - self.contentwastruncated_ = 0 + if self.has_contentwastruncated_: + self.has_contentwastruncated_ = 0 + self.contentwastruncated_ = 0 def has_contentwastruncated(self): return self.has_contentwastruncated_ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/user_service_pb.py --- a/thirdparty/google_appengine/google/appengine/api/user_service_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/user_service_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -27,10 +27,12 @@ OK = 0 REDIRECT_URL_TOO_LONG = 1 + NOT_ALLOWED = 2 _ErrorCode_NAMES = { 0: "OK", 1: "REDIRECT_URL_TOO_LONG", + 2: "NOT_ALLOWED", } def ErrorCode_Name(cls, x): return cls._ErrorCode_NAMES.get(x, "") diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/api/users.py --- a/thirdparty/google_appengine/google/appengine/api/users.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/api/users.py Thu Feb 12 12:30:36 2009 +0000 @@ -20,9 +20,9 @@ Classes defined here: User: object representing a user. Error: base exception type - BadRequestError: UserService exception UserNotFoundError: UserService exception - BackendError: UserService exception + RedirectTooLongError: UserService exception + NotAllowedError: UserService exception """ @@ -50,6 +50,10 @@ """Raised by UserService calls if the generated redirect URL was too long. """ +class NotAllowedError(Error): + """Raised by UserService calls if the requested redirect URL is not allowed. + """ + class User(object): """A user. @@ -147,6 +151,9 @@ if (e.application_error == user_service_pb.UserServiceError.REDIRECT_URL_TOO_LONG): raise RedirectTooLongError + elif (e.application_error == + user_service_pb.UserServiceError.NOT_ALLOWED): + raise NotAllowedError else: raise e return resp.value() diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/base/capabilities_pb.py --- a/thirdparty/google_appengine/google/appengine/base/capabilities_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/base/capabilities_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -59,8 +59,9 @@ def mutable_default_config(self): self.has_default_config_ = 1; return self.default_config() def clear_default_config(self): - self.has_default_config_ = 0; - if self.default_config_ is not None: self.default_config_.Clear() + if self.has_default_config_: + self.has_default_config_ = 0; + if self.default_config_ is not None: self.default_config_.Clear() def has_default_config(self): return self.has_default_config_ @@ -203,8 +204,9 @@ self.package_ = x def clear_package(self): - self.has_package_ = 0 - self.package_ = "" + if self.has_package_: + self.has_package_ = 0 + self.package_ = "" def has_package(self): return self.has_package_ @@ -215,8 +217,9 @@ self.capability_ = x def clear_capability(self): - self.has_capability_ = 0 - self.capability_ = "" + if self.has_capability_: + self.has_capability_ = 0 + self.capability_ = "" def has_capability(self): return self.has_capability_ @@ -227,8 +230,9 @@ self.status_ = x def clear_status(self): - self.has_status_ = 0 - self.status_ = 4 + if self.has_status_: + self.has_status_ = 0 + self.status_ = 4 def has_status(self): return self.has_status_ @@ -239,8 +243,9 @@ self.scheduled_time_ = x def clear_scheduled_time(self): - self.has_scheduled_time_ = 0 - self.scheduled_time_ = "" + if self.has_scheduled_time_: + self.has_scheduled_time_ = 0 + self.scheduled_time_ = "" def has_scheduled_time(self): return self.has_scheduled_time_ @@ -251,8 +256,9 @@ self.internal_message_ = x def clear_internal_message(self): - self.has_internal_message_ = 0 - self.internal_message_ = "" + if self.has_internal_message_: + self.has_internal_message_ = 0 + self.internal_message_ = "" def has_internal_message(self): return self.has_internal_message_ @@ -263,8 +269,9 @@ self.admin_message_ = x def clear_admin_message(self): - self.has_admin_message_ = 0 - self.admin_message_ = "" + if self.has_admin_message_: + self.has_admin_message_ = 0 + self.admin_message_ = "" def has_admin_message(self): return self.has_admin_message_ @@ -275,8 +282,9 @@ self.error_message_ = x def clear_error_message(self): - self.has_error_message_ = 0 - self.error_message_ = "" + if self.has_error_message_: + self.has_error_message_ = 0 + self.error_message_ = "" def has_error_message(self): return self.has_error_message_ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/cron/groctimespecification.py --- a/thirdparty/google_appengine/google/appengine/cron/groctimespecification.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/cron/groctimespecification.py Thu Feb 12 12:30:36 2009 +0000 @@ -22,25 +22,35 @@ module takes a parsed schedule (produced by Antlr) and creates objects that can produce times that match this schedule. -A parsed schedule is one of two types - an Interval, and a Specific Time. +A parsed schedule is one of two types - an Interval or a Specific Time. See the class docstrings for more. Extensions to be considered: allowing a comma separated list of times to run allowing the user to specify particular days of the month to run - """ import calendar import datetime +try: + import pytz +except ImportError: + pytz = None + import groc HOURS = 'hours' MINUTES = 'minutes' +try: + from pytz import NonExistentTimeError +except ImportError: + class NonExistentTimeError(Exception): + pass + def GrocTimeSpecification(schedule): """Factory function. @@ -53,7 +63,6 @@ Returns: a TimeSpecification instance """ - parser = groc.CreateParser(schedule) parser.timespec() @@ -71,7 +80,7 @@ """Returns the next n times that match the schedule, starting at time start. Arguments: - start: a datetime to start from. Matches will start from after this time + start: a datetime to start from. Matches will start from after this time. n: the number of matching times to return Returns: @@ -89,7 +98,7 @@ Must be implemented in subclasses. Arguments: - start: a datetime to start with. Matches will start from this time + start: a datetime to start with. Matches will start from this time. Returns: a datetime object @@ -100,13 +109,14 @@ class IntervalTimeSpecification(TimeSpecification): """A time specification for a given interval. - An Interval type spec runs at the given fixed interval. They have two + An Interval type spec runs at the given fixed interval. It has two attributes: period - the type of interval, either "hours" or "minutes" interval - the number of units of type period. + timezone - the timezone for this specification. Not used in this class. """ - def __init__(self, interval, period): + def __init__(self, interval, period, timezone=None): super(IntervalTimeSpecification, self).__init__(self) self.interval = interval self.period = period @@ -115,7 +125,7 @@ """Returns the next match after time 't'. Arguments: - t: a datetime to start from. Matches will start from after this time + t: a datetime to start from. Matches will start from after this time. Returns: a datetime object @@ -129,47 +139,55 @@ class SpecificTimeSpecification(TimeSpecification): """Specific time specification. - A Specific interval is more complex, but define a certain time to run, on - given days. They have the following attributes: + A Specific interval is more complex, but defines a certain time to run and + the days that it should run. It has the following attributes: time - the time of day to run, as "HH:MM" ordinals - first, second, third &c, as a set of integers in 1..5 - months - the months that this is valid, as a set of integers in 1..12 - weekdays - the days of the week to run this, 0=Sunday, 6=Saturday. + months - the months that this should run, as a set of integers in 1..12 + weekdays - the days of the week that this should run, as a set of integers, + 0=Sunday, 6=Saturday + timezone - the optional timezone as a string for this specification. + Defaults to UTC - valid entries are things like Australia/Victoria + or PST8PDT. - The specific time interval can be quite complex. A schedule could look like + A specific time schedule can be quite complex. A schedule could look like this: "1st,third sat,sun of jan,feb,mar 09:15" - In this case, ordinals would be [1,3], weekdays [0,6], months [1,2,3] and time - would be "09:15". + In this case, ordinals would be {1,3}, weekdays {0,6}, months {1,2,3} and + time would be "09:15". """ + timezone = None + def __init__(self, ordinals=None, weekdays=None, months=None, monthdays=None, - timestr='00:00'): + timestr='00:00', timezone=None): super(SpecificTimeSpecification, self).__init__(self) - if weekdays and monthdays: + if weekdays is not None and monthdays is not None: raise ValueError("can't supply both monthdays and weekdays") if ordinals is None: self.ordinals = set(range(1, 6)) else: - self.ordinals = ordinals + self.ordinals = set(ordinals) if weekdays is None: self.weekdays = set(range(7)) else: - self.weekdays = weekdays + self.weekdays = set(weekdays) if months is None: self.months = set(range(1, 13)) else: - self.months = months + self.months = set(months) if monthdays is None: self.monthdays = set() else: - self.monthdays = monthdays + self.monthdays = set(monthdays) hourstr, minutestr = timestr.split(':') self.time = datetime.time(int(hourstr), int(minutestr)) + if timezone and pytz is not None: + self.timezone = pytz.timezone(timezone) def _MatchingDays(self, year, month): """Returns matching days for the given year and month. @@ -225,33 +243,53 @@ """Returns the next time that matches the schedule after time start. Arguments: - start: a datetime to start with. Matches will start after this time + start: a UTC datetime to start from. Matches will start after this time Returns: a datetime object """ start_time = start + if self.timezone and pytz is not None: + if not start_time.tzinfo: + start_time = pytz.utc.localize(start_time) + start_time = start_time.astimezone(self.timezone) + start_time = start_time.replace(tzinfo=None) if self.months: - months = self._NextMonthGenerator(start.month, self.months) + months = self._NextMonthGenerator(start_time.month, self.months) while True: month, yearwraps = months.next() - candidate = start_time.replace(day=1, month=month, + candidate_month = start_time.replace(day=1, month=month, year=start_time.year + yearwraps) if self.monthdays: - _, last_day = calendar.monthrange(candidate.year, candidate.month) - day_matches = sorted([x for x in self.monthdays if x <= last_day]) + _, last_day = calendar.monthrange(candidate_month.year, + candidate_month.month) + day_matches = sorted(x for x in self.monthdays if x <= last_day) else: - day_matches = self._MatchingDays(candidate.year, month) + day_matches = self._MatchingDays(candidate_month.year, month) - if ((candidate.year, candidate.month) + if ((candidate_month.year, candidate_month.month) == (start_time.year, start_time.month)): day_matches = [x for x in day_matches if x >= start_time.day] - if day_matches and day_matches[0] == start_time.day: - if start_time.time() >= self.time: - day_matches.pop(0) - if not day_matches: - continue - out = candidate.replace(day=day_matches[0], hour=self.time.hour, - minute=self.time.minute, second=0, microsecond=0) - return out + while (day_matches and day_matches[0] == start_time.day + and start_time.time() >= self.time): + day_matches.pop(0) + while day_matches: + out = candidate_month.replace(day=day_matches[0], hour=self.time.hour, + + + minute=self.time.minute, second=0, + microsecond=0) + if self.timezone and pytz is not None: + try: + out = self.timezone.localize(out) + except (NonExistentTimeError, IndexError): + for _ in range(24): + out = out.replace(minute=1) + datetime.timedelta(minutes=60) + try: + out = self.timezone.localize(out) + except (NonExistentTimeError, IndexError): + continue + break + out = out.astimezone(pytz.utc) + return out diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/datastore/datastore_index.py --- a/thirdparty/google_appengine/google/appengine/datastore/datastore_index.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/datastore/datastore_index.py Thu Feb 12 12:30:36 2009 +0000 @@ -328,8 +328,7 @@ else: props.append((prop_name, ASCENDING)) - if (kind and not ancestor and - (not props or (len(props) == 1 and props[0][1] == ASCENDING))): + if kind and not ancestor and len(props) <= 1: required = False if props: diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/datastore/datastore_pb.py --- a/thirdparty/google_appengine/google/appengine/datastore/datastore_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/datastore/datastore_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -44,8 +44,9 @@ self.handle_ = x def clear_handle(self): - self.has_handle_ = 0 - self.handle_ = 0 + if self.has_handle_: + self.has_handle_ = 0 + self.handle_ = 0 def has_handle(self): return self.has_handle_ @@ -146,8 +147,9 @@ self.op_ = x def clear_op(self): - self.has_op_ = 0 - self.op_ = 0 + if self.has_op_: + self.has_op_ = 0 + self.op_ = 0 def has_op(self): return self.has_op_ @@ -269,8 +271,9 @@ self.property_ = x def clear_property(self): - self.has_property_ = 0 - self.property_ = "" + if self.has_property_: + self.has_property_ = 0 + self.property_ = "" def has_property(self): return self.has_property_ @@ -281,8 +284,9 @@ self.direction_ = x def clear_direction(self): - self.has_direction_ = 0 - self.direction_ = 1 + if self.has_direction_: + self.has_direction_ = 0 + self.direction_ = 1 def has_direction(self): return self.has_direction_ @@ -391,8 +395,9 @@ self.app_ = x def clear_app(self): - self.has_app_ = 0 - self.app_ = "" + if self.has_app_: + self.has_app_ = 0 + self.app_ = "" def has_app(self): return self.has_app_ @@ -403,8 +408,9 @@ self.kind_ = x def clear_kind(self): - self.has_kind_ = 0 - self.kind_ = "" + if self.has_kind_: + self.has_kind_ = 0 + self.kind_ = "" def has_kind(self): return self.has_kind_ @@ -420,8 +426,9 @@ def mutable_ancestor(self): self.has_ancestor_ = 1; return self.ancestor() def clear_ancestor(self): - self.has_ancestor_ = 0; - if self.ancestor_ is not None: self.ancestor_.Clear() + if self.has_ancestor_: + self.has_ancestor_ = 0; + if self.ancestor_ is not None: self.ancestor_.Clear() def has_ancestor(self): return self.has_ancestor_ @@ -448,8 +455,9 @@ self.search_query_ = x def clear_search_query(self): - self.has_search_query_ = 0 - self.search_query_ = "" + if self.has_search_query_: + self.has_search_query_ = 0 + self.search_query_ = "" def has_search_query(self): return self.has_search_query_ @@ -476,8 +484,9 @@ self.hint_ = x def clear_hint(self): - self.has_hint_ = 0 - self.hint_ = 0 + if self.has_hint_: + self.has_hint_ = 0 + self.hint_ = 0 def has_hint(self): return self.has_hint_ @@ -488,8 +497,9 @@ self.offset_ = x def clear_offset(self): - self.has_offset_ = 0 - self.offset_ = 0 + if self.has_offset_: + self.has_offset_ = 0 + self.offset_ = 0 def has_offset(self): return self.has_offset_ @@ -500,8 +510,9 @@ self.limit_ = x def clear_limit(self): - self.has_limit_ = 0 - self.limit_ = 0 + if self.has_limit_: + self.has_limit_ = 0 + self.limit_ = 0 def has_limit(self): return self.has_limit_ @@ -528,8 +539,9 @@ self.require_perfect_plan_ = x def clear_require_perfect_plan(self): - self.has_require_perfect_plan_ = 0 - self.require_perfect_plan_ = 0 + if self.has_require_perfect_plan_: + self.has_require_perfect_plan_ = 0 + self.require_perfect_plan_ = 0 def has_require_perfect_plan(self): return self.has_require_perfect_plan_ @@ -851,8 +863,9 @@ self.native_ancestor_ = x def clear_native_ancestor(self): - self.has_native_ancestor_ = 0 - self.native_ancestor_ = 0 + if self.has_native_ancestor_: + self.has_native_ancestor_ = 0 + self.native_ancestor_ = 0 def has_native_ancestor(self): return self.has_native_ancestor_ @@ -879,8 +892,9 @@ self.native_offset_ = x def clear_native_offset(self): - self.has_native_offset_ = 0 - self.native_offset_ = 0 + if self.has_native_offset_: + self.has_native_offset_ = 0 + self.native_offset_ = 0 def has_native_offset(self): return self.has_native_offset_ @@ -891,8 +905,9 @@ self.native_limit_ = x def clear_native_limit(self): - self.has_native_limit_ = 0 - self.native_limit_ = 0 + if self.has_native_limit_: + self.has_native_limit_ = 0 + self.native_limit_ = 0 def has_native_limit(self): return self.has_native_limit_ @@ -1031,8 +1046,9 @@ self.cursor_ = x def clear_cursor(self): - self.has_cursor_ = 0 - self.cursor_ = 0 + if self.has_cursor_: + self.has_cursor_ = 0 + self.cursor_ = 0 def has_cursor(self): return self.has_cursor_ @@ -1178,8 +1194,9 @@ self.index_writes_ = x def clear_index_writes(self): - self.has_index_writes_ = 0 - self.index_writes_ = 0 + if self.has_index_writes_: + self.has_index_writes_ = 0 + self.index_writes_ = 0 def has_index_writes(self): return self.has_index_writes_ @@ -1278,8 +1295,9 @@ def mutable_transaction(self): self.has_transaction_ = 1; return self.transaction() def clear_transaction(self): - self.has_transaction_ = 0; - if self.transaction_ is not None: self.transaction_.Clear() + if self.has_transaction_: + self.has_transaction_ = 0; + if self.transaction_ is not None: self.transaction_.Clear() def has_transaction(self): return self.has_transaction_ @@ -1400,8 +1418,9 @@ def mutable_entity(self): self.has_entity_ = 1; return self.entity() def clear_entity(self): - self.has_entity_ = 0; - if self.entity_ is not None: self.entity_.Clear() + if self.has_entity_: + self.has_entity_ = 0; + if self.entity_ is not None: self.entity_.Clear() def has_entity(self): return self.has_entity_ @@ -1556,6 +1575,8 @@ class PutRequest(ProtocolBuffer.ProtocolMessage): has_transaction_ = 0 transaction_ = None + has_trusted_ = 0 + trusted_ = 0 def __init__(self, contents=None): self.entity_ = [] @@ -1591,8 +1612,9 @@ def mutable_transaction(self): self.has_transaction_ = 1; return self.transaction() def clear_transaction(self): - self.has_transaction_ = 0; - if self.transaction_ is not None: self.transaction_.Clear() + if self.has_transaction_: + self.has_transaction_ = 0; + if self.transaction_ is not None: self.transaction_.Clear() def has_transaction(self): return self.has_transaction_ @@ -1612,12 +1634,26 @@ def clear_composite_index(self): self.composite_index_ = [] + def trusted(self): return self.trusted_ + + def set_trusted(self, x): + self.has_trusted_ = 1 + self.trusted_ = x + + def clear_trusted(self): + if self.has_trusted_: + self.has_trusted_ = 0 + self.trusted_ = 0 + + def has_trusted(self): return self.has_trusted_ + def MergeFrom(self, x): assert x is not self for i in xrange(x.entity_size()): self.add_entity().CopyFrom(x.entity(i)) if (x.has_transaction()): self.mutable_transaction().MergeFrom(x.transaction()) for i in xrange(x.composite_index_size()): self.add_composite_index().CopyFrom(x.composite_index(i)) + if (x.has_trusted()): self.set_trusted(x.trusted()) def Equals(self, x): if x is self: return 1 @@ -1629,6 +1665,8 @@ if len(self.composite_index_) != len(x.composite_index_): return 0 for e1, e2 in zip(self.composite_index_, x.composite_index_): if e1 != e2: return 0 + if self.has_trusted_ != x.has_trusted_: return 0 + if self.has_trusted_ and self.trusted_ != x.trusted_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -1647,12 +1685,14 @@ if (self.has_transaction_): n += 1 + self.lengthString(self.transaction_.ByteSize()) n += 1 * len(self.composite_index_) for i in xrange(len(self.composite_index_)): n += self.lengthString(self.composite_index_[i].ByteSize()) + if (self.has_trusted_): n += 2 return n + 0 def Clear(self): self.clear_entity() self.clear_transaction() self.clear_composite_index() + self.clear_trusted() def OutputUnchecked(self, out): for i in xrange(len(self.entity_)): @@ -1667,6 +1707,9 @@ out.putVarInt32(26) out.putVarInt32(self.composite_index_[i].ByteSize()) self.composite_index_[i].OutputUnchecked(out) + if (self.has_trusted_): + out.putVarInt32(32) + out.putBoolean(self.trusted_) def TryMerge(self, d): while d.avail() > 0: @@ -1689,6 +1732,9 @@ d.skip(length) self.add_composite_index().TryMerge(tmp) continue + if tt == 32: + self.set_trusted(d.getBoolean()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -1715,17 +1761,20 @@ res+=e.__str__(prefix + " ", printElemNumber) res+=prefix+">\n" cnt+=1 + if self.has_trusted_: res+=prefix+("trusted: %s\n" % self.DebugFormatBool(self.trusted_)) return res kentity = 1 ktransaction = 2 kcomposite_index = 3 + ktrusted = 4 _TEXT = ( "ErrorCode", "entity", "transaction", "composite_index", + "trusted", ) _TYPES = ( @@ -1736,6 +1785,8 @@ ProtocolBuffer.Encoder.STRING, + ProtocolBuffer.Encoder.NUMERIC, + ) _STYLE = """""" @@ -1777,8 +1828,9 @@ def mutable_cost(self): self.has_cost_ = 1; return self.cost() def clear_cost(self): - self.has_cost_ = 0; - if self.cost_ is not None: self.cost_.Clear() + if self.has_cost_: + self.has_cost_ = 0; + if self.cost_ is not None: self.cost_.Clear() def has_cost(self): return self.has_cost_ @@ -1882,6 +1934,8 @@ class DeleteRequest(ProtocolBuffer.ProtocolMessage): has_transaction_ = 0 transaction_ = None + has_trusted_ = 0 + trusted_ = 0 def __init__(self, contents=None): self.key_ = [] @@ -1916,16 +1970,31 @@ def mutable_transaction(self): self.has_transaction_ = 1; return self.transaction() def clear_transaction(self): - self.has_transaction_ = 0; - if self.transaction_ is not None: self.transaction_.Clear() + if self.has_transaction_: + self.has_transaction_ = 0; + if self.transaction_ is not None: self.transaction_.Clear() def has_transaction(self): return self.has_transaction_ + def trusted(self): return self.trusted_ + + def set_trusted(self, x): + self.has_trusted_ = 1 + self.trusted_ = x + + def clear_trusted(self): + if self.has_trusted_: + self.has_trusted_ = 0 + self.trusted_ = 0 + + def has_trusted(self): return self.has_trusted_ + def MergeFrom(self, x): assert x is not self for i in xrange(x.key_size()): self.add_key().CopyFrom(x.key(i)) if (x.has_transaction()): self.mutable_transaction().MergeFrom(x.transaction()) + if (x.has_trusted()): self.set_trusted(x.trusted()) def Equals(self, x): if x is self: return 1 @@ -1934,6 +2003,8 @@ if e1 != e2: return 0 if self.has_transaction_ != x.has_transaction_: return 0 if self.has_transaction_ and self.transaction_ != x.transaction_: return 0 + if self.has_trusted_ != x.has_trusted_: return 0 + if self.has_trusted_ and self.trusted_ != x.trusted_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -1948,13 +2019,18 @@ n += 1 * len(self.key_) for i in xrange(len(self.key_)): n += self.lengthString(self.key_[i].ByteSize()) if (self.has_transaction_): n += 1 + self.lengthString(self.transaction_.ByteSize()) + if (self.has_trusted_): n += 2 return n + 0 def Clear(self): self.clear_key() self.clear_transaction() + self.clear_trusted() def OutputUnchecked(self, out): + if (self.has_trusted_): + out.putVarInt32(32) + out.putBoolean(self.trusted_) if (self.has_transaction_): out.putVarInt32(42) out.putVarInt32(self.transaction_.ByteSize()) @@ -1967,6 +2043,9 @@ def TryMerge(self, d): while d.avail() > 0: tt = d.getVarInt32() + if tt == 32: + self.set_trusted(d.getBoolean()) + continue if tt == 42: length = d.getVarInt32() tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) @@ -1997,17 +2076,19 @@ res+=prefix+"transaction <\n" res+=self.transaction_.__str__(prefix + " ", printElemNumber) res+=prefix+">\n" + if self.has_trusted_: res+=prefix+("trusted: %s\n" % self.DebugFormatBool(self.trusted_)) return res kkey = 6 ktransaction = 5 + ktrusted = 4 _TEXT = ( "ErrorCode", None, None, None, - None, + "trusted", "transaction", "key", ) @@ -2020,7 +2101,7 @@ ProtocolBuffer.Encoder.MAX_TYPE, - ProtocolBuffer.Encoder.MAX_TYPE, + ProtocolBuffer.Encoder.NUMERIC, ProtocolBuffer.Encoder.STRING, @@ -2050,8 +2131,9 @@ def mutable_cost(self): self.has_cost_ = 1; return self.cost() def clear_cost(self): - self.has_cost_ = 0; - if self.cost_ is not None: self.cost_.Clear() + if self.has_cost_: + self.has_cost_ = 0; + if self.cost_ is not None: self.cost_.Clear() def has_cost(self): return self.has_cost_ @@ -2145,8 +2227,9 @@ self.count_ = x def clear_count(self): - self.has_count_ = 0 - self.count_ = 1 + if self.has_count_: + self.has_count_ = 0 + self.count_ = 1 def has_count(self): return self.has_count_ @@ -2258,8 +2341,9 @@ def mutable_cursor(self): self.has_cursor_ = 1; return self.cursor() def clear_cursor(self): - self.has_cursor_ = 0; - if self.cursor_ is not None: self.cursor_.Clear() + if self.has_cursor_: + self.has_cursor_ = 0; + if self.cursor_ is not None: self.cursor_.Clear() def has_cursor(self): return self.has_cursor_ @@ -2286,8 +2370,9 @@ self.more_results_ = x def clear_more_results(self): - self.has_more_results_ = 0 - self.more_results_ = 0 + if self.has_more_results_: + self.has_more_results_ = 0 + self.more_results_ = 0 def has_more_results(self): return self.has_more_results_ @@ -2596,5 +2681,97 @@ _STYLE = """""" _STYLE_CONTENT_TYPE = """""" - -__all__ = ['Transaction','Query','Query_Filter','Query_Order','QueryExplanation','Cursor','Error','Cost','GetRequest','GetResponse','GetResponse_Entity','PutRequest','PutResponse','DeleteRequest','DeleteResponse','NextRequest','QueryResult','Schema','CompositeIndices'] +class CommitResponse(ProtocolBuffer.ProtocolMessage): + has_cost_ = 0 + cost_ = None + + def __init__(self, contents=None): + self.lazy_init_lock_ = thread.allocate_lock() + if contents is not None: self.MergeFromString(contents) + + def cost(self): + if self.cost_ is None: + self.lazy_init_lock_.acquire() + try: + if self.cost_ is None: self.cost_ = Cost() + finally: + self.lazy_init_lock_.release() + return self.cost_ + + def mutable_cost(self): self.has_cost_ = 1; return self.cost() + + def clear_cost(self): + if self.has_cost_: + self.has_cost_ = 0; + if self.cost_ is not None: self.cost_.Clear() + + def has_cost(self): return self.has_cost_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_cost()): self.mutable_cost().MergeFrom(x.cost()) + + def Equals(self, x): + if x is self: return 1 + if self.has_cost_ != x.has_cost_: return 0 + if self.has_cost_ and self.cost_ != x.cost_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (self.has_cost_ and not self.cost_.IsInitialized(debug_strs)): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + if (self.has_cost_): n += 1 + self.lengthString(self.cost_.ByteSize()) + return n + 0 + + def Clear(self): + self.clear_cost() + + def OutputUnchecked(self, out): + if (self.has_cost_): + out.putVarInt32(10) + out.putVarInt32(self.cost_.ByteSize()) + self.cost_.OutputUnchecked(out) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_cost().TryMerge(tmp) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_cost_: + res+=prefix+"cost <\n" + res+=self.cost_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + return res + + kcost = 1 + + _TEXT = ( + "ErrorCode", + "cost", + ) + + _TYPES = ( + ProtocolBuffer.Encoder.NUMERIC, + ProtocolBuffer.Encoder.STRING, + + ) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" + +__all__ = ['Transaction','Query','Query_Filter','Query_Order','QueryExplanation','Cursor','Error','Cost','GetRequest','GetResponse','GetResponse_Entity','PutRequest','PutResponse','DeleteRequest','DeleteResponse','NextRequest','QueryResult','Schema','CompositeIndices','CommitResponse'] diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/datastore/entity_pb.py --- a/thirdparty/google_appengine/google/appengine/datastore/entity_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/datastore/entity_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -40,8 +40,9 @@ self.type_ = x def clear_type(self): - self.has_type_ = 0 - self.type_ = "" + if self.has_type_: + self.has_type_ = 0 + self.type_ = "" def has_type(self): return self.has_type_ @@ -52,8 +53,9 @@ self.id_ = x def clear_id(self): - self.has_id_ = 0 - self.id_ = 0 + if self.has_id_: + self.has_id_ = 0 + self.id_ = 0 def has_id(self): return self.has_id_ @@ -64,8 +66,9 @@ self.name_ = x def clear_name(self): - self.has_name_ = 0 - self.name_ = "" + if self.has_name_: + self.has_name_ = 0 + self.name_ = "" def has_name(self): return self.has_name_ @@ -156,8 +159,9 @@ self.x_ = x def clear_x(self): - self.has_x_ = 0 - self.x_ = 0.0 + if self.has_x_: + self.has_x_ = 0 + self.x_ = 0.0 def has_x(self): return self.has_x_ @@ -168,8 +172,9 @@ self.y_ = x def clear_y(self): - self.has_y_ = 0 - self.y_ = 0.0 + if self.has_y_: + self.has_y_ = 0 + self.y_ = 0.0 def has_y(self): return self.has_y_ @@ -253,8 +258,9 @@ self.email_ = x def clear_email(self): - self.has_email_ = 0 - self.email_ = "" + if self.has_email_: + self.has_email_ = 0 + self.email_ = "" def has_email(self): return self.has_email_ @@ -265,8 +271,9 @@ self.auth_domain_ = x def clear_auth_domain(self): - self.has_auth_domain_ = 0 - self.auth_domain_ = "" + if self.has_auth_domain_: + self.has_auth_domain_ = 0 + self.auth_domain_ = "" def has_auth_domain(self): return self.has_auth_domain_ @@ -277,8 +284,9 @@ self.nickname_ = x def clear_nickname(self): - self.has_nickname_ = 0 - self.nickname_ = "" + if self.has_nickname_: + self.has_nickname_ = 0 + self.nickname_ = "" def has_nickname(self): return self.has_nickname_ @@ -289,8 +297,9 @@ self.gaiaid_ = x def clear_gaiaid(self): - self.has_gaiaid_ = 0 - self.gaiaid_ = 0 + if self.has_gaiaid_: + self.has_gaiaid_ = 0 + self.gaiaid_ = 0 def has_gaiaid(self): return self.has_gaiaid_ @@ -398,8 +407,9 @@ self.app_ = x def clear_app(self): - self.has_app_ = 0 - self.app_ = "" + if self.has_app_: + self.has_app_ = 0 + self.app_ = "" def has_app(self): return self.has_app_ @@ -517,8 +527,9 @@ self.int64value_ = x def clear_int64value(self): - self.has_int64value_ = 0 - self.int64value_ = 0 + if self.has_int64value_: + self.has_int64value_ = 0 + self.int64value_ = 0 def has_int64value(self): return self.has_int64value_ @@ -529,8 +540,9 @@ self.booleanvalue_ = x def clear_booleanvalue(self): - self.has_booleanvalue_ = 0 - self.booleanvalue_ = 0 + if self.has_booleanvalue_: + self.has_booleanvalue_ = 0 + self.booleanvalue_ = 0 def has_booleanvalue(self): return self.has_booleanvalue_ @@ -541,8 +553,9 @@ self.stringvalue_ = x def clear_stringvalue(self): - self.has_stringvalue_ = 0 - self.stringvalue_ = "" + if self.has_stringvalue_: + self.has_stringvalue_ = 0 + self.stringvalue_ = "" def has_stringvalue(self): return self.has_stringvalue_ @@ -553,8 +566,9 @@ self.doublevalue_ = x def clear_doublevalue(self): - self.has_doublevalue_ = 0 - self.doublevalue_ = 0.0 + if self.has_doublevalue_: + self.has_doublevalue_ = 0 + self.doublevalue_ = 0.0 def has_doublevalue(self): return self.has_doublevalue_ @@ -570,8 +584,9 @@ def mutable_pointvalue(self): self.has_pointvalue_ = 1; return self.pointvalue() def clear_pointvalue(self): - self.has_pointvalue_ = 0; - if self.pointvalue_ is not None: self.pointvalue_.Clear() + if self.has_pointvalue_: + self.has_pointvalue_ = 0; + if self.pointvalue_ is not None: self.pointvalue_.Clear() def has_pointvalue(self): return self.has_pointvalue_ @@ -587,8 +602,9 @@ def mutable_uservalue(self): self.has_uservalue_ = 1; return self.uservalue() def clear_uservalue(self): - self.has_uservalue_ = 0; - if self.uservalue_ is not None: self.uservalue_.Clear() + if self.has_uservalue_: + self.has_uservalue_ = 0; + if self.uservalue_ is not None: self.uservalue_.Clear() def has_uservalue(self): return self.has_uservalue_ @@ -604,8 +620,9 @@ def mutable_referencevalue(self): self.has_referencevalue_ = 1; return self.referencevalue() def clear_referencevalue(self): - self.has_referencevalue_ = 0; - if self.referencevalue_ is not None: self.referencevalue_.Clear() + if self.has_referencevalue_: + self.has_referencevalue_ = 0; + if self.referencevalue_ is not None: self.referencevalue_.Clear() def has_referencevalue(self): return self.has_referencevalue_ @@ -884,8 +901,9 @@ self.meaning_ = x def clear_meaning(self): - self.has_meaning_ = 0 - self.meaning_ = 0 + if self.has_meaning_: + self.has_meaning_ = 0 + self.meaning_ = 0 def has_meaning(self): return self.has_meaning_ @@ -896,8 +914,9 @@ self.meaning_uri_ = x def clear_meaning_uri(self): - self.has_meaning_uri_ = 0 - self.meaning_uri_ = "" + if self.has_meaning_uri_: + self.has_meaning_uri_ = 0 + self.meaning_uri_ = "" def has_meaning_uri(self): return self.has_meaning_uri_ @@ -908,8 +927,9 @@ self.name_ = x def clear_name(self): - self.has_name_ = 0 - self.name_ = "" + if self.has_name_: + self.has_name_ = 0 + self.name_ = "" def has_name(self): return self.has_name_ @@ -928,8 +948,9 @@ self.multiple_ = x def clear_multiple(self): - self.has_multiple_ = 0 - self.multiple_ = 0 + if self.has_multiple_: + self.has_multiple_ = 0 + self.multiple_ = 0 def has_multiple(self): return self.has_multiple_ @@ -1087,8 +1108,9 @@ self.type_ = x def clear_type(self): - self.has_type_ = 0 - self.type_ = "" + if self.has_type_: + self.has_type_ = 0 + self.type_ = "" def has_type(self): return self.has_type_ @@ -1099,8 +1121,9 @@ self.id_ = x def clear_id(self): - self.has_id_ = 0 - self.id_ = 0 + if self.has_id_: + self.has_id_ = 0 + self.id_ = 0 def has_id(self): return self.has_id_ @@ -1111,8 +1134,9 @@ self.name_ = x def clear_name(self): - self.has_name_ = 0 - self.name_ = "" + if self.has_name_: + self.has_name_ = 0 + self.name_ = "" def has_name(self): return self.has_name_ @@ -1307,8 +1331,9 @@ self.app_ = x def clear_app(self): - self.has_app_ = 0 - self.app_ = "" + if self.has_app_: + self.has_app_ = 0 + self.app_ = "" def has_app(self): return self.has_app_ @@ -1464,8 +1489,9 @@ self.email_ = x def clear_email(self): - self.has_email_ = 0 - self.email_ = "" + if self.has_email_: + self.has_email_ = 0 + self.email_ = "" def has_email(self): return self.has_email_ @@ -1476,8 +1502,9 @@ self.auth_domain_ = x def clear_auth_domain(self): - self.has_auth_domain_ = 0 - self.auth_domain_ = "" + if self.has_auth_domain_: + self.has_auth_domain_ = 0 + self.auth_domain_ = "" def has_auth_domain(self): return self.has_auth_domain_ @@ -1488,8 +1515,9 @@ self.nickname_ = x def clear_nickname(self): - self.has_nickname_ = 0 - self.nickname_ = "" + if self.has_nickname_: + self.has_nickname_ = 0 + self.nickname_ = "" def has_nickname(self): return self.has_nickname_ @@ -1500,8 +1528,9 @@ self.gaiaid_ = x def clear_gaiaid(self): - self.has_gaiaid_ = 0 - self.gaiaid_ = 0 + if self.has_gaiaid_: + self.has_gaiaid_ = 0 + self.gaiaid_ = 0 def has_gaiaid(self): return self.has_gaiaid_ @@ -1680,8 +1709,9 @@ def mutable_owner(self): self.has_owner_ = 1; return self.owner() def clear_owner(self): - self.has_owner_ = 0; - if self.owner_ is not None: self.owner_.Clear() + if self.has_owner_: + self.has_owner_ = 0; + if self.owner_ is not None: self.owner_.Clear() def has_owner(self): return self.has_owner_ @@ -1692,8 +1722,9 @@ self.kind_ = x def clear_kind(self): - self.has_kind_ = 0 - self.kind_ = 0 + if self.has_kind_: + self.has_kind_ = 0 + self.kind_ = 0 def has_kind(self): return self.has_kind_ @@ -1704,8 +1735,9 @@ self.kind_uri_ = x def clear_kind_uri(self): - self.has_kind_uri_ = 0 - self.kind_uri_ = "" + if self.has_kind_uri_: + self.has_kind_uri_ = 0 + self.kind_uri_ = "" def has_kind_uri(self): return self.has_kind_uri_ @@ -2000,8 +2032,9 @@ self.index_id_ = x def clear_index_id(self): - self.has_index_id_ = 0 - self.index_id_ = 0 + if self.has_index_id_: + self.has_index_id_ = 0 + self.index_id_ = 0 def has_index_id(self): return self.has_index_id_ @@ -2132,8 +2165,9 @@ self.name_ = x def clear_name(self): - self.has_name_ = 0 - self.name_ = "" + if self.has_name_: + self.has_name_ = 0 + self.name_ = "" def has_name(self): return self.has_name_ @@ -2144,8 +2178,9 @@ self.direction_ = x def clear_direction(self): - self.has_direction_ = 0 - self.direction_ = 1 + if self.has_direction_: + self.has_direction_ = 0 + self.direction_ = 1 def has_direction(self): return self.has_direction_ @@ -2225,8 +2260,9 @@ self.entity_type_ = x def clear_entity_type(self): - self.has_entity_type_ = 0 - self.entity_type_ = "" + if self.has_entity_type_: + self.has_entity_type_ = 0 + self.entity_type_ = "" def has_entity_type(self): return self.has_entity_type_ @@ -2237,8 +2273,9 @@ self.ancestor_ = x def clear_ancestor(self): - self.has_ancestor_ = 0 - self.ancestor_ = 0 + if self.has_ancestor_: + self.has_ancestor_ = 0 + self.ancestor_ = 0 def has_ancestor(self): return self.has_ancestor_ @@ -2409,8 +2446,9 @@ self.app_id_ = x def clear_app_id(self): - self.has_app_id_ = 0 - self.app_id_ = "" + if self.has_app_id_: + self.has_app_id_ = 0 + self.app_id_ = "" def has_app_id(self): return self.has_app_id_ @@ -2421,8 +2459,9 @@ self.id_ = x def clear_id(self): - self.has_id_ = 0 - self.id_ = 0 + if self.has_id_: + self.has_id_ = 0 + self.id_ = 0 def has_id(self): return self.has_id_ @@ -2441,8 +2480,9 @@ self.state_ = x def clear_state(self): - self.has_state_ = 0 - self.state_ = 0 + if self.has_state_: + self.has_state_ = 0 + self.state_ = 0 def has_state(self): return self.has_state_ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/dist/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/dist/__init__.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Specify the modules for which a stub exists.""" + + +__all__ = ['ftplib', 'httplib', 'py_imp', 'neo_cgi', 'select', 'socket', + 'subprocess', 'tempfile'] diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/dist/ftplib.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/dist/ftplib.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/dist/httplib.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/dist/httplib.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,385 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Copyright 2008 Python Software Foundation, Ian Bicking, and Google.""" + +import mimetools +import StringIO +import sys + + +CONTINUE = 100 +SWITCHING_PROTOCOLS = 101 +PROCESSING = 102 +OK = 200 +CREATED = 201 +ACCEPTED = 202 +NON_AUTHORITATIVE_INFORMATION = 203 +NO_CONTENT = 204 +RESET_CONTENT = 205 +PARTIAL_CONTENT = 206 +MULTI_STATUS = 207 +IM_USED = 226 +MULTIPLE_CHOICES = 300 +MOVED_PERMANENTLY = 301 +FOUND = 302 +SEE_OTHER = 303 +NOT_MODIFIED = 304 +USE_PROXY = 305 +TEMPORARY_REDIRECT = 307 +BAD_REQUEST = 400 +UNAUTHORIZED = 401 +PAYMENT_REQUIRED = 402 +FORBIDDEN = 403 +NOT_FOUND = 404 +METHOD_NOT_ALLOWED = 405 +NOT_ACCEPTABLE = 406 +PROXY_AUTHENTICATION_REQUIRED = 407 +REQUEST_TIMEOUT = 408 +CONFLICT = 409 +GONE = 410 +LENGTH_REQUIRED = 411 +PRECONDITION_FAILED = 412 +REQUEST_ENTITY_TOO_LARGE = 413 +REQUEST_URI_TOO_LONG = 414 +UNSUPPORTED_MEDIA_TYPE = 415 +REQUESTED_RANGE_NOT_SATISFIABLE = 416 +EXPECTATION_FAILED = 417 +UNPROCESSABLE_ENTITY = 422 +LOCKED = 423 +FAILED_DEPENDENCY = 424 +UPGRADE_REQUIRED = 426 +INTERNAL_SERVER_ERROR = 500 +NOT_IMPLEMENTED = 501 +BAD_GATEWAY = 502 +SERVICE_UNAVAILABLE = 503 +GATEWAY_TIMEOUT = 504 +HTTP_VERSION_NOT_SUPPORTED = 505 +INSUFFICIENT_STORAGE = 507 +NOT_EXTENDED = 510 + +responses = { + 100: 'Continue', + 101: 'Switching Protocols', + + 200: 'OK', + 201: 'Created', + 202: 'Accepted', + 203: 'Non-Authoritative Information', + 204: 'No Content', + 205: 'Reset Content', + 206: 'Partial Content', + + 300: 'Multiple Choices', + 301: 'Moved Permanently', + 302: 'Found', + 303: 'See Other', + 304: 'Not Modified', + 305: 'Use Proxy', + 306: '(Unused)', + 307: 'Temporary Redirect', + + 400: 'Bad Request', + 401: 'Unauthorized', + 402: 'Payment Required', + 403: 'Forbidden', + 404: 'Not Found', + 405: 'Method Not Allowed', + 406: 'Not Acceptable', + 407: 'Proxy Authentication Required', + 408: 'Request Timeout', + 409: 'Conflict', + 410: 'Gone', + 411: 'Length Required', + 412: 'Precondition Failed', + 413: 'Request Entity Too Large', + 414: 'Request-URI Too Long', + 415: 'Unsupported Media Type', + 416: 'Requested Range Not Satisfiable', + 417: 'Expectation Failed', + + 500: 'Internal Server Error', + 501: 'Not Implemented', + 502: 'Bad Gateway', + 503: 'Service Unavailable', + 504: 'Gateway Timeout', + 505: 'HTTP Version Not Supported', +} + +HTTP_PORT = 80 +HTTPS_PORT = 443 + + + + + +class HTTPConnection: + + + protocol = 'http' + default_port = HTTP_PORT + _allow_truncated = True + _follow_redirects = False + + def __init__(self, host, port=None, strict=False, timeout=None): + from google.appengine.api import urlfetch + self._fetch = urlfetch.fetch + self._method_map = { + 'GET': urlfetch.GET, + 'POST': urlfetch.POST, + 'HEAD': urlfetch.HEAD, + 'PUT': urlfetch.PUT, + 'DELETE': urlfetch.DELETE, + } + self.host = host + self.port = port + self._method = self._url = None + self._body = '' + self.headers = [] + + def connect(self): + pass + + def request(self, method, url, body=None, headers=None): + self._method = method + self._url = url + try: + self._body = body.read() + except AttributeError: + self._body = body + if headers is None: + headers = [] + elif hasattr(headers, 'items'): + headers = headers.items() + self.headers = headers + + def putrequest(self, request, selector, skip_host=False, skip_accept_encoding=False): + self._method = request + self._url = selector + + def putheader(self, header, *lines): + line = '\r\n\t'.join(lines) + self.headers.append((header, line)) + + def endheaders(self): + pass + + def set_debuglevel(self, level=None): + pass + + def send(self, data): + self._body += data + + def getresponse(self): + if self.port and self.port != self.default_port: + host = '%s:%s' % (self.host, self.port) + else: + host = self.host + url = '%s://%s%s' % (self.protocol, host, self._url) + headers = dict(self.headers) + + try: + method = self._method_map[self._method.upper()] + except KeyError: + raise ValueError("%r is an unrecognized HTTP method" % self._method) + + response = self._fetch(url, self._body, method, headers, + self._allow_truncated, self._follow_redirects) + return HTTPResponse(response) + + def close(self): + pass + + +class HTTPSConnection(HTTPConnection): + + protocol = 'https' + default_port = HTTPS_PORT + + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=False, timeout=None): + if key_file is not None or cert_file is not None: + raise NotImplementedError( + "key_file and cert_file arguments are not implemented") + HTTPConnection.__init__(self, host, port=port, strict=strict, + timeout=timeout) + + +class HTTPResponse(object): + + def __init__(self, fetch_response): + self._fetch_response = fetch_response + self.fp = StringIO.StringIO(fetch_response.content) + + def __getattr__(self, attr): + return getattr(self.fp, attr) + + def getheader(self, name, default=None): + return self._fetch_response.headers.get(name, default) + + def getheaders(self): + return self._fetch_response.headers.items() + + @property + def msg(self): + msg = mimetools.Message(StringIO.StringIO('')) + for name, value in self._fetch_response.headers.items(): + msg[name] = value + return msg + + version = 11 + + @property + def status(self): + return self._fetch_response.status_code + + @property + def reason(self): + return responses.get(self._fetch_response.status_code, 'Unknown') + + +class HTTP: + "Compatibility class with httplib.py from 1.5." + + _http_vsn = 11 + _http_vsn_str = 'HTTP/1.1' + + debuglevel = 0 + + _connection_class = HTTPConnection + + def __init__(self, host='', port=None, strict=None): + "Provide a default host, since the superclass requires one." + + if port == 0: + port = None + + self._setup(self._connection_class(host, port, strict)) + + def _setup(self, conn): + self._conn = conn + + self.send = conn.send + self.putrequest = conn.putrequest + self.endheaders = conn.endheaders + self.set_debuglevel = conn.set_debuglevel + + conn._http_vsn = self._http_vsn + conn._http_vsn_str = self._http_vsn_str + + self.file = None + + def connect(self, host=None, port=None): + "Accept arguments to set the host/port, since the superclass doesn't." + self.__init__(host, port) + + def getfile(self): + "Provide a getfile, since the superclass' does not use this concept." + return self.file + + def putheader(self, header, *values): + "The superclass allows only one value argument." + self._conn.putheader(header, '\r\n\t'.join(values)) + + def getreply(self): + """Compat definition since superclass does not define it. + + Returns a tuple consisting of: + - server status code (e.g. '200' if all goes well) + - server "reason" corresponding to status code + - any RFC822 headers in the response from the server + """ + response = self._conn.getresponse() + + self.headers = response.msg + self.file = response.fp + return response.status, response.reason, response.msg + + def close(self): + self._conn.close() + + self.file = None + + +class HTTPS(HTTP): + """Compatibility with 1.5 httplib interface + + Python 1.5.2 did not have an HTTPS class, but it defined an + interface for sending http requests that is also useful for + https. + """ + + _connection_class = HTTPSConnection + + def __init__(self, host='', port=None, key_file=None, cert_file=None, + strict=None): + if key_file is not None or cert_file is not None: + raise NotImplementedError( + "key_file and cert_file arguments are not implemented") + + + if port == 0: + port = None + self._setup(self._connection_class(host, port, key_file, + cert_file, strict)) + + self.key_file = key_file + self.cert_file = cert_file + + +class HTTPException(Exception): + pass + +class NotConnected(HTTPException): + pass + +class InvalidURL(HTTPException): + pass + +class UnknownProtocol(HTTPException): + def __init__(self, version): + self.version = version + HTTPException.__init__(self, version) + +class UnknownTransferEncoding(HTTPException): + pass + +class UnimplementedFileMode(HTTPException): + pass + +class IncompleteRead(HTTPException): + def __init__(self, partial): + self.partial = partial + HTTPException.__init__(self, partial) + +class ImproperConnectionState(HTTPException): + pass + +class CannotSendRequest(ImproperConnectionState): + pass + +class CannotSendHeader(ImproperConnectionState): + pass + +class ResponseNotReady(ImproperConnectionState): + pass + +class BadStatusLine(HTTPException): + def __init__(self, line): + self.line = line + HTTPException.__init__(self, line) + +error = HTTPException diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/dist/neo_cgi.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/dist/neo_cgi.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/dist/py_imp.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/dist/py_imp.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Stub replacement for Python's imp module.""" + + +import os +import sys + + +PY_SOURCE, PY_COMPILED, C_EXTENSION = 1, 2, 3 +PKG_DIRECTORY, C_BUILTIN, PY_FROZEN = 5, 6, 7 + + +def get_magic(): + return '\0\0\0\0' + + +def get_suffixes(): + return [('.py', 'U', PY_SOURCE)] + + +def new_module(name): + return type(sys.modules[__name__])(name) + + +def lock_held(): + """Return False since threading is not supported.""" + return False + +def acquire_lock(): + """Acquiring the lock is a no-op since no threading is supported.""" + pass + +def release_lock(): + """There is no lock to release since acquiring is a no-op when there is no + threading.""" + pass + + +def is_builtin(name): + return name in sys.builtin_module_names + + +def is_frozen(name): + return False + + +class NullImporter(object): + + def __init__(self, path_string): + if not path_string: + raise ImportError("empty pathname") + elif os.path.isdir(path_string): + raise ImportError("existing directory") + + def find_module(self, fullname): + return None diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/dist/select.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/dist/select.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/dist/socket.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/dist/socket.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +AF_INET = None +SOCK_STREAM = None +SOCK_DGRAM = None + +_GLOBAL_DEFAULT_TIMEOUT = object() + + +class error(OSError): + pass + +class herror(error): + pass + +class gaierror(error): + pass + +class timeout(error): + pass + + +def _fileobject(fp, mode='rb', bufsize=-1, close=False): + """Assuming that the argument is a StringIO or file instance.""" + if not hasattr(fp, 'fileno'): + fp.fileno = lambda: None + return fp diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/dist/subprocess.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/dist/subprocess.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/dist/tempfile.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/dist/tempfile.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Temporary files. + +This module is a replacement for the stock tempfile module in Python, +and provides only in-memory temporary files as implemented by +cStringIO. The only functionality provided is the TemporaryFile +function. +""" + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +__all__ = [ + "TemporaryFile", + + "NamedTemporaryFile", "mkstemp", "mkdtemp", "mktemp", + "TMP_MAX", "gettempprefix", "tempdir", "gettempdir", +] + +TMP_MAX = 10000 + +template = "tmp" + +tempdir = None + +def TemporaryFile(mode='w+b', bufsize=-1, suffix="", + prefix=template, dir=None): + """Create and return a temporary file. + Arguments: + 'prefix', 'suffix', 'dir', 'mode', 'bufsize' are all ignored. + + Returns an object with a file-like interface. The file is in memory + only, and does not exist on disk. + """ + + return StringIO() + +def PlaceHolder(*args, **kwargs): + raise NotImplementedError("Only tempfile.TemporaryFile is available for use") + +NamedTemporaryFile = PlaceHolder +mkstemp = PlaceHolder +mkdtemp = PlaceHolder +mktemp = PlaceHolder +gettempprefix = PlaceHolder +tempdir = PlaceHolder +gettempdir = PlaceHolder diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/ext/bulkload/__init__.py --- a/thirdparty/google_appengine/google/appengine/ext/bulkload/__init__.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/ext/bulkload/__init__.py Thu Feb 12 12:30:36 2009 +0000 @@ -105,17 +105,12 @@ import csv import httplib import os -import sys import traceback -import types -import struct -import zlib import google import wsgiref.handlers from google.appengine.api import datastore -from google.appengine.api import datastore_types from google.appengine.ext import webapp from google.appengine.ext.bulkload import constants @@ -299,13 +294,8 @@ """ Handle a POST. Reads CSV data, converts to entities, and stores them. """ self.response.headers['Content-Type'] = 'text/plain' - version = self.request.headers.get('GAE-Uploader-Version', '0') - if version == '1': - kind = self.request.headers.get('GAE-Uploader-Kind') - response, output = self.LoadV1(kind, self.request.body) - else: - response, output = self.Load(self.request.get(constants.KIND_PARAM), - self.request.get(constants.CSV_PARAM)) + response, output = self.Load(self.request.get(constants.KIND_PARAM), + self.request.get(constants.CSV_PARAM)) self.response.set_status(response) self.response.out.write(output) @@ -422,69 +412,6 @@ return self.LoadEntities(self.IterRows(reader), loader) - def IterRowsV1(self, data): - """Yields a tuple of columns for each row in the uploaded data. - - Args: - data: a string containing the unzipped v1 format data to load. - - """ - column_count, = struct.unpack_from('!i', data) - offset = 4 - - lengths_format = '!%di' % (column_count,) - - while offset < len(data): - id_num = struct.unpack_from('!i', data, offset=offset) - offset += 4 - - value_lengths = struct.unpack_from(lengths_format, data, offset=offset) - offset += 4 * column_count - - columns = struct.unpack_from(''.join('%ds' % length - for length in value_lengths), data, - offset=offset) - offset += sum(value_lengths) - - yield (id_num, columns) - - - def LoadV1(self, kind, data): - """Parses version-1 format data, converts to entities, and stores them. - - On error, fails fast. Returns a "bad request" HTTP response code and - includes the traceback in the output. - - Args: - kind: a string containing the entity kind that this loader handles - data: a string containing the (v1 format) data to load - - Returns: - tuple (response code, output) where: - response code: integer HTTP response code to return - output: string containing the HTTP response body - """ - Validate(kind, basestring) - Validate(data, basestring) - output = [] - - try: - loader = Loader.RegisteredLoaders()[kind] - except KeyError: - output.append('Error: no Loader defined for kind %s.' % kind) - return (httplib.BAD_REQUEST, ''.join(output)) - - try: - data = zlib.decompress(data) - except: - stacktrace = traceback.format_exc() - output.append('Error: Could not decompress data\n%s' % stacktrace) - return (httplib.BAD_REQUEST, ''.join(output)) - - key_format = 'i%010d' - return self.LoadEntities(self.IterRowsV1(data), - loader, - key_format=key_format) def main(*loaders): """Starts bulk upload. diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/ext/db/__init__.py --- a/thirdparty/google_appengine/google/appengine/ext/db/__init__.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/ext/db/__init__.py Thu Feb 12 12:30:36 2009 +0000 @@ -77,10 +77,13 @@ +import copy import datetime import logging +import re import time import urlparse +import warnings from google.appengine.api import datastore from google.appengine.api import datastore_errors @@ -187,6 +190,11 @@ _ALLOWED_EXPANDO_PROPERTY_TYPES = set(_ALLOWED_PROPERTY_TYPES) _ALLOWED_EXPANDO_PROPERTY_TYPES.update((list, tuple, type(None))) +_OPERATORS = ['<', '<=', '>', '>=', '=', '==', '!=', 'in'] +_FILTER_REGEX = re.compile( + '^\s*([^\s]+)(\s+(%s)\s*)?$' % '|'.join(_OPERATORS), + re.IGNORECASE | re.UNICODE) + def class_for_kind(kind): """Return base-class responsible for implementing kind. @@ -527,12 +535,12 @@ **kwds): """Creates a new instance of this model. - To create a new entity, you instantiate a model and then call save(), + To create a new entity, you instantiate a model and then call put(), which saves the entity to the datastore: person = Person() person.name = 'Bret' - person.save() + person.put() You can initialize properties in the model in the constructor with keyword arguments: @@ -595,7 +603,7 @@ This property is only available if this entity is already stored in the datastore, so it is available if this entity was fetched returned from a - query, or after save() is called the first time for new entities. + query, or after put() is called the first time for new entities. Returns: Datastore key of persisted entity. @@ -606,7 +614,11 @@ if self.is_saved(): return self._entity.key() elif self._key_name: - parent = self._parent and self._parent.key() + if self._parent_key: + parent_key = self._parent_key + elif self._parent: + parent_key = self._parent.key() + parent = self._parent_key or (self._parent and self._parent.key()) return Key.from_path(self.kind(), self._key_name, parent=parent) else: raise NotSavedError() @@ -1143,7 +1155,7 @@ 1 - Because it is not possible for the datastore to know what kind of property to store on an undefined expando value, setting a property to - None is the same as deleting it form the expando. + None is the same as deleting it from the expando. 2 - Persistent variables on Expando must not begin with '_'. These variables considered to be 'protected' in Python, and are used @@ -1524,16 +1536,71 @@ model_class: Model class to build query for. """ super(Query, self).__init__(model_class) - self.__query_set = {} + self.__query_sets = [{}] self.__orderings = [] self.__ancestor = None - def _get_query(self, _query_class=datastore.Query): - query = _query_class(self._model_class.kind(), self.__query_set) - if self.__ancestor is not None: - query.Ancestor(self.__ancestor) - query.Order(*self.__orderings) - return query + def _get_query(self, + _query_class=datastore.Query, + _multi_query_class=datastore.MultiQuery): + queries = [] + for query_set in self.__query_sets: + query = _query_class(self._model_class.kind(), query_set) + if self.__ancestor is not None: + query.Ancestor(self.__ancestor) + queries.append(query) + + if (_query_class != datastore.Query and + _multi_query_class == datastore.MultiQuery): + warnings.warn( + 'Custom _query_class specified without corresponding custom' + ' _query_multi_class. Things will break if you use queries with' + ' the "IN" or "!=" operators.', RuntimeWarning) + if len(queries) > 1: + raise datastore_errors.BadArgumentError( + 'Query requires multiple subqueries to satisfy. If _query_class' + ' is overridden, _multi_query_class must also be overridden.') + elif (_query_class == datastore.Query and + _multi_query_class != datastore.MultiQuery): + raise BadArgumentError('_query_class must also be overridden if' + ' _multi_query_class is overridden.') + + if len(queries) == 1: + queries[0].Order(*self.__orderings) + return queries[0] + else: + return _multi_query_class(queries, self.__orderings) + + def __filter_disjunction(self, operations, values): + """Add a disjunction of several filters and several values to the query. + + This is implemented by duplicating queries and combining the + results later. + + Args: + operations: a string or list of strings. Each string contains a + property name and an operator to filter by. The operators + themselves must not require multiple queries to evaluate + (currently, this means that 'in' and '!=' are invalid). + + values: a value or list of filter values, normalized by + _normalize_query_parameter. + """ + if not isinstance(operations, (list, tuple)): + operations = [operations] + if not isinstance(values, (list, tuple)): + values = [values] + + new_query_sets = [] + for operation in operations: + if operation.lower().endswith('in') or operation.endswith('!='): + raise BadQueryError('Cannot use "in" or "!=" in a disjunction.') + for query_set in self.__query_sets: + for value in values: + new_query_set = copy.copy(query_set) + datastore._AddOrAppend(new_query_set, operation, value) + new_query_sets.append(new_query_set) + self.__query_sets = new_query_sets def filter(self, property_operator, value): """Add filter to query. @@ -1545,11 +1612,29 @@ Returns: Self to support method chaining. """ - if isinstance(value, (list, tuple)): - raise BadValueError('Filtering on lists is not supported') - - value = _normalize_query_parameter(value) - datastore._AddOrAppend(self.__query_set, property_operator, value) + match = _FILTER_REGEX.match(property_operator) + prop = match.group(1) + if match.group(3) is not None: + operator = match.group(3) + else: + operator = '==' + + if operator.lower() == 'in': + if not isinstance(value, (list, tuple)): + raise BadValueError('Argument to the "in" operator must be a list') + values = [_normalize_query_parameter(v) for v in value] + self.__filter_disjunction(prop + ' =', values) + else: + if isinstance(value, (list, tuple)): + raise BadValueError('Filtering on lists is not supported') + if operator == '!=': + self.__filter_disjunction([prop + ' <', prop + ' >'], + _normalize_query_parameter(value)) + else: + value = _normalize_query_parameter(value) + for query_set in self.__query_sets: + datastore._AddOrAppend(query_set, property_operator, value) + return self def order(self, property): @@ -2359,8 +2444,11 @@ Returns: validated list appropriate to save in the datastore. """ - return self.validate_list_contents( + value = self.validate_list_contents( super(ListProperty, self).get_value_for_datastore(model_instance)) + if self.validator: + self.validator(value) + return value class StringListProperty(ListProperty): @@ -2622,5 +2710,7 @@ run_in_transaction = datastore.RunInTransaction +run_in_transaction_custom_retries = datastore.RunInTransactionCustomRetries RunInTransaction = run_in_transaction +RunInTransactionCustomRetries = run_in_transaction_custom_retries diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/ext/gql/__init__.py --- a/thirdparty/google_appengine/google/appengine/ext/gql/__init__.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/ext/gql/__init__.py Thu Feb 12 12:30:36 2009 +0000 @@ -28,7 +28,6 @@ import calendar import datetime -import heapq import logging import re import time @@ -38,6 +37,7 @@ from google.appengine.api import datastore_types from google.appengine.api import users +MultiQuery = datastore.MultiQuery LOG_LEVEL = logging.DEBUG - 1 @@ -161,7 +161,7 @@ \S+ """, re.VERBOSE | re.IGNORECASE) - MAX_ALLOWABLE_QUERIES = 30 + MAX_ALLOWABLE_QUERIES = datastore.MAX_ALLOWABLE_QUERIES __ANCESTOR = -1 @@ -347,37 +347,113 @@ else: return users.User(email=values[0], _auth_domain=self.__auth_domain) + def __EncodeIfNeeded(self, value): + """Simple helper function to create an str from possibly unicode strings. + Args: + value: input string (should pass as an instance of str or unicode). + """ + if isinstance(value, unicode): + return value.encode('utf8') + else: + return value + def __CastDate(self, values): - """Cast date values to DATETIME() class using ISO string or tuple inputs.""" + """Cast DATE values (year/month/day) from input (to datetime.datetime). + + Casts DATE input values formulated as ISO string or time tuple inputs. + + Args: + values: either a single string with ISO time representation or 3 + integer valued date tuple (year, month, day). + + Returns: + datetime.datetime value parsed from the input values. + """ + + if len(values) == 1: + value = self.__EncodeIfNeeded(values[0]) + if isinstance(value, str): + try: + time_tuple = time.strptime(value, '%Y-%m-%d')[0:6] + except ValueError, err: + self.__CastError('DATE', values, err) + else: + self.__CastError('DATE', values, 'Single input value not a string') + elif len(values) == 3: + time_tuple = (values[0], values[1], values[2], 0, 0, 0) + else: + self.__CastError('DATE', values, + 'function takes 1 string or 3 integer values') + try: - if len(values) == 1 and isinstance(values[0], str): - time_tuple = time.strptime(values[0], '%Y-%m-%d') - return datetime.datetime(*time_tuple[0:6]) - else: - return datetime.datetime(values[0], values[1], values[2], 0, 0, 0) + return datetime.datetime(*time_tuple) except ValueError, err: self.__CastError('DATE', values, err) def __CastTime(self, values): - """Cast time values to DATETIME() class using ISO string or tuple inputs.""" + """Cast TIME values (hour/min/sec) from input (to datetime.datetime). + + Casts TIME input values formulated as ISO string or time tuple inputs. + + Args: + values: either a single string with ISO time representation or 1-4 + integer valued time tuple (hour), (hour, minute), + (hour, minute, second), (hour, minute, second, microsec). + + Returns: + datetime.datetime value parsed from the input values. + """ + if len(values) == 1: + value = self.__EncodeIfNeeded(values[0]) + if isinstance(value, str): + try: + time_tuple = time.strptime(value, '%H:%M:%S') + except ValueError, err: + self.__CastError('TIME', values, err) + time_tuple = (1970, 1, 1) + time_tuple[3:] + time_tuple = time_tuple[0:6] + elif isinstance(value, int): + time_tuple = (1970, 1, 1, value) + else: + self.__CastError('TIME', values, + 'Single input value not a string or integer hour') + elif len(values) <= 4: + time_tuple = (1970, 1, 1) + tuple(values) + else: + self.__CastError('TIME', values, err) + try: - if len(values) == 1 and isinstance(values[0], str): - time_tuple = time.strptime(values[0], '%H:%M:%S') - time_tuple = (1970, 1, 1) + time_tuple[3:] - return datetime.datetime(*time_tuple[0:6]) - else: - return datetime.datetime(1970, 1, 1, *values) + return datetime.datetime(*time_tuple) except ValueError, err: self.__CastError('TIME', values, err) def __CastDatetime(self, values): - """Cast values to DATETIME() class using ISO string or tuple inputs.""" + """Cast DATETIME values (string or tuple) from input (to datetime.datetime). + + Casts DATETIME input values formulated as ISO string or datetime tuple + inputs. + + Args: + values: either a single string with ISO representation or 3-7 + integer valued time tuple (year, month, day, ...). + + Returns: + datetime.datetime value parsed from the input values. + """ + if len(values) == 1: + value = self.__EncodeIfNeeded(values[0]) + if isinstance(value, str): + try: + time_tuple = time.strptime(str(value), '%Y-%m-%d %H:%M:%S')[0:6] + except ValueError, err: + self.__CastError('DATETIME', values, err) + else: + self.__CastError('DATETIME', values, 'Single input value not a string') + else: + time_tuple = values + try: - if len(values) == 1 and isinstance(values[0], str): - time_tuple = time.strptime(values[0], '%Y-%m-%d %H:%M:%S') - return datetime.datetime(*time_tuple[0:6]) - else: - return datetime.datetime(*values) + return datetime.datetime(*time_tuple) except ValueError, err: self.__CastError('DATETIME', values, err) @@ -1062,220 +1138,3 @@ def __repr__(self): return 'Literal(%s)' % repr(self.__value) - - -class MultiQuery(datastore.Query): - """Class representing a GQL query requiring multiple datastore queries. - - This class is actually a subclass of datastore.Query as it is intended to act - like a normal Query object (supporting the same interface). - """ - - def __init__(self, bound_queries, orderings): - self.__bound_queries = bound_queries - self.__orderings = orderings - - def __str__(self): - res = 'MultiQuery: ' - for query in self.__bound_queries: - res = '%s %s' % (res, str(query)) - return res - - def Get(self, limit, offset=0): - """Get results of the query with a limit on the number of results. - - Args: - limit: maximum number of values to return. - offset: offset requested -- if nonzero, this will override the offset in - the original query - - Returns: - A list of entities with at most "limit" entries (less if the query - completes before reading limit values). - """ - count = 1 - result = [] - - iterator = self.Run() - - try: - for i in xrange(offset): - val = iterator.next() - except StopIteration: - pass - - try: - while count <= limit: - val = iterator.next() - result.append(val) - count += 1 - except StopIteration: - pass - return result - - class SortOrderEntity(object): - def __init__(self, entity_iterator, orderings): - self.__entity_iterator = entity_iterator - self.__entity = None - self.__min_max_value_cache = {} - try: - self.__entity = entity_iterator.next() - except StopIteration: - pass - else: - self.__orderings = orderings - - def __str__(self): - return str(self.__entity) - - def GetEntity(self): - return self.__entity - - def GetNext(self): - return MultiQuery.SortOrderEntity(self.__entity_iterator, - self.__orderings) - - def CmpProperties(self, that): - """Compare two entities and return their relative order. - - Compares self to that based on the current sort orderings and the - key orders between them. Returns negative, 0, or positive depending on - whether self is less, equal to, or greater than that. This - comparison returns as if all values were to be placed in ascending order - (highest value last). Only uses the sort orderings to compare (ignores - keys). - - Args: - self: SortOrderEntity - that: SortOrderEntity - - Returns: - Negative if self < that - Zero if self == that - Positive if self > that - """ - if not self.__entity: - return cmp(self.__entity, that.__entity) - - for (identifier, order) in self.__orderings: - value1 = self.__GetValueForId(self, identifier, order) - value2 = self.__GetValueForId(that, identifier, order) - - result = cmp(value1, value2) - if order == datastore.Query.DESCENDING: - result = -result - if result: - return result - return 0 - - def __GetValueForId(self, sort_order_entity, identifier, sort_order): - value = sort_order_entity.__entity[identifier] - entity_key = sort_order_entity.__entity.key() - if self.__min_max_value_cache.has_key((entity_key, identifier)): - value = self.__min_max_value_cache[(entity_key, identifier)] - elif isinstance(value, list): - if sort_order == datastore.Query.DESCENDING: - value = min(value) - else: - value = max(value) - self.__min_max_value_cache[(entity_key, identifier)] = value - - return value - - def __cmp__(self, that): - """Compare self to that w.r.t. values defined in the sort order. - - Compare an entity with another, using sort-order first, then the key - order to break ties. This can be used in a heap to have faster min-value - lookup. - - Args: - that: other entity to compare to - Returns: - negative: if self is less than that in sort order - zero: if self is equal to that in sort order - positive: if self is greater than that in sort order - """ - property_compare = self.CmpProperties(that) - if property_compare: - return property_compare - else: - return cmp(self.__entity.key(), that.__entity.key()) - - def Run(self): - """Return an iterable output with all results in order.""" - results = [] - count = 1 - for bound_query in self.__bound_queries: - logging.log(LOG_LEVEL, 'Running query #%i' % count) - results.append(bound_query.Run()) - count += 1 - - def IterateResults(results): - """Iterator function to return all results in sorted order. - - Iterate over the array of results, yielding the next element, in - sorted order. This function is destructive (results will be empty - when the operation is complete). - - Args: - results: list of result iterators to merge and iterate through - - Yields: - The next result in sorted order. - """ - result_heap = [] - for result in results: - heap_value = MultiQuery.SortOrderEntity(result, self.__orderings) - if heap_value.GetEntity(): - heapq.heappush(result_heap, heap_value) - - used_keys = set() - - while result_heap: - top_result = heapq.heappop(result_heap) - - results_to_push = [] - if top_result.GetEntity().key() not in used_keys: - yield top_result.GetEntity() - else: - pass - - used_keys.add(top_result.GetEntity().key()) - - results_to_push = [] - while result_heap: - next = heapq.heappop(result_heap) - if cmp(top_result, next): - results_to_push.append(next) - break - else: - results_to_push.append(next.GetNext()) - results_to_push.append(top_result.GetNext()) - - for popped_result in results_to_push: - if popped_result.GetEntity(): - heapq.heappush(result_heap, popped_result) - - return IterateResults(results) - - def Count(self, limit=None): - """Return the number of matched entities for this query. - - Will return the de-duplicated count of results. Will call the more - efficient Get() function if a limit is given. - - Args: - limit: maximum number of entries to count (for any result > limit, return - limit). - Returns: - count of the number of entries returned. - """ - if limit is None: - count = 0 - for value in self.Run(): - count += 1 - return count - else: - return len(self.Get(limit)) - diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/ext/remote_api/__init__.py --- a/thirdparty/google_appengine/google/appengine/ext/remote_api/__init__.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/ext/remote_api/__init__.py Thu Feb 12 12:30:36 2009 +0000 @@ -14,385 +14,3 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -"""An apiproxy stub that calls a remote handler via HTTP. - -This allows easy remote access to the App Engine datastore, and potentially any -of the other App Engine APIs, using the same interface you use when accessing -the service locally. - -An example Python script: ---- -from google.appengine.ext import db -from google.appengine.ext import remote_api -from myapp import models -import getpass - -def auth_func(): - return (raw_input('Username:'), getpass.getpass('Password:')) - -remote_api.ConfigureRemoteDatastore('my-app', '/remote_api', auth_func) - -# Now you can access the remote datastore just as if your code was running on -# App Engine! - -houses = models.House.all().fetch(100) -for a_house in q: - a_house.doors += 1 -db.put(houses) ---- - -A few caveats: -- Where possible, avoid iterating over queries directly. Fetching as many - results as you will need is faster and more efficient. -- If you need to iterate, consider instead fetching items in batches with a sort - order and constructing a new query starting from where the previous one left - off. The __key__ pseudo-property can be used as a sort key for this purpose, - and does not even require a custom index if you are iterating over all - entities of a given type. -- Likewise, it's a good idea to put entities in batches. Instead of calling put - for each individual entity, accumulate them and put them in batches using - db.put(), if you can. -- Requests and responses are still limited to 1MB each, so if you have large - entities or try and fetch or put many of them at once, your requests may fail. -""" - - - - - -import os -import pickle -import sha -import sys -import thread -import threading -from google.appengine.api import apiproxy_stub_map -from google.appengine.datastore import datastore_pb -from google.appengine.ext.remote_api import remote_api_pb -from google.appengine.runtime import apiproxy_errors -from google.appengine.tools import appengine_rpc - - -def GetUserAgent(): - """Determines the value of the 'User-agent' header to use for HTTP requests. - - Returns: - String containing the 'user-agent' header value, which includes the SDK - version, the platform information, and the version of Python; - e.g., "remote_api/1.0.1 Darwin/9.2.0 Python/2.5.2". - """ - product_tokens = [] - - product_tokens.append("Google-remote_api/1.0") - - product_tokens.append(appengine_rpc.GetPlatformToken()) - - python_version = ".".join(str(i) for i in sys.version_info) - product_tokens.append("Python/%s" % python_version) - - return " ".join(product_tokens) - - -def GetSourceName(): - return "Google-remote_api-1.0" - - -class TransactionData(object): - """Encapsulates data about an individual transaction.""" - - def __init__(self, thread_id): - self.thread_id = thread_id - self.preconditions = {} - self.entities = {} - - -class RemoteStub(object): - """A stub for calling services on a remote server over HTTP. - - You can use this to stub out any service that the remote server supports. - """ - - def __init__(self, server, path): - """Constructs a new RemoteStub that communicates with the specified server. - - Args: - server: An instance of a subclass of - google.appengine.tools.appengine_rpc.AbstractRpcServer. - path: The path to the handler this stub should send requests to. - """ - self._server = server - self._path = path - - def MakeSyncCall(self, service, call, request, response): - request_pb = remote_api_pb.Request() - request_pb.set_service_name(service) - request_pb.set_method(call) - request_pb.mutable_request().set_contents(request.Encode()) - - response_pb = remote_api_pb.Response() - response_pb.ParseFromString(self._server.Send(self._path, - request_pb.Encode())) - - if response_pb.has_exception(): - raise pickle.loads(response_pb.exception().contents()) - else: - response.ParseFromString(response_pb.response().contents()) - - -class RemoteDatastoreStub(RemoteStub): - """A specialised stub for accessing the App Engine datastore remotely. - - A specialised stub is required because there are some datastore operations - that preserve state between calls. This stub makes queries possible. - Transactions on the remote datastore are unfortunately still impossible. - """ - - def __init__(self, server, path): - super(RemoteDatastoreStub, self).__init__(server, path) - self.__queries = {} - self.__transactions = {} - - self.__next_local_cursor = 1 - self.__local_cursor_lock = threading.Lock() - self.__next_local_tx = 1 - self.__local_tx_lock = threading.Lock() - - def MakeSyncCall(self, service, call, request, response): - assert service == 'datastore_v3' - - explanation = [] - assert request.IsInitialized(explanation), explanation - - handler = getattr(self, '_Dynamic_' + call, None) - if handler: - handler(request, response) - else: - super(RemoteDatastoreStub, self).MakeSyncCall(service, call, request, - response) - - assert response.IsInitialized(explanation), explanation - - def _Dynamic_RunQuery(self, query, query_result): - self.__local_cursor_lock.acquire() - try: - cursor_id = self.__next_local_cursor - self.__next_local_cursor += 1 - finally: - self.__local_cursor_lock.release() - self.__queries[cursor_id] = query - - query_result.mutable_cursor().set_cursor(cursor_id) - query_result.set_more_results(True) - - def _Dynamic_Next(self, next_request, query_result): - cursor = next_request.cursor().cursor() - if cursor not in self.__queries: - raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, - 'Cursor %d not found' % cursor) - query = self.__queries[cursor] - - if query is None: - query_result.set_more_results(False) - return - - request = datastore_pb.Query() - request.CopyFrom(query) - if request.has_limit(): - request.set_limit(min(request.limit(), next_request.count())) - else: - request.set_limit(next_request.count()) - - super(RemoteDatastoreStub, self).MakeSyncCall( - 'remote_datastore', 'RunQuery', request, query_result) - - query.set_offset(query.offset() + query_result.result_size()) - if query.has_limit(): - query.set_limit(query.limit() - query_result.result_size()) - if not query_result.more_results(): - self.__queries[cursor] = None - - def _Dynamic_Get(self, get_request, get_response): - txid = None - if get_request.has_transaction(): - txid = get_request.transaction().handle() - txdata = self.__transactions[txid] - assert (txdata.thread_id == thread.get_ident(), - "Transactions are single-threaded.") - - keys = [(k, k.Encode()) for k in get_request.key_list()] - - new_request = datastore_pb.GetRequest() - for key, enckey in keys: - if enckey not in txdata.entities: - new_request.add_key().CopyFrom(key) - else: - new_request = get_request - - if new_request.key_size() > 0: - super(RemoteDatastoreStub, self).MakeSyncCall( - 'datastore_v3', 'Get', new_request, get_response) - - if txid is not None: - newkeys = new_request.key_list() - entities = get_response.entity_list() - for key, entity in zip(newkeys, entities): - entity_hash = None - if entity.has_entity(): - entity_hash = sha.new(entity.entity().Encode()).digest() - txdata.preconditions[key.Encode()] = (key, entity_hash) - - new_response = datastore_pb.GetResponse() - it = iter(get_response.entity_list()) - for key, enckey in keys: - if enckey in txdata.entities: - cached_entity = txdata.entities[enckey][1] - if cached_entity: - new_response.add_entity().mutable_entity().CopyFrom(cached_entity) - else: - new_response.add_entity() - else: - new_entity = it.next() - if new_entity.has_entity(): - assert new_entity.entity().key() == key - new_response.add_entity().CopyFrom(new_entity) - else: - new_response.add_entity() - get_response.CopyFrom(new_response) - - def _Dynamic_Put(self, put_request, put_response): - if put_request.has_transaction(): - entities = put_request.entity_list() - - requires_id = lambda x: x.id() == 0 and not x.has_name() - new_ents = [e for e in entities - if requires_id(e.key().path().element_list()[-1])] - id_request = remote_api_pb.PutRequest() - if new_ents: - for ent in new_ents: - e = id_request.add_entity() - e.mutable_key().CopyFrom(ent.key()) - e.mutable_entity_group() - id_response = datastore_pb.PutResponse() - super(RemoteDatastoreStub, self).MakeSyncCall( - 'remote_datastore', 'GetIDs', id_request, id_response) - assert id_request.entity_size() == id_response.key_size() - for key, ent in zip(id_response.key_list(), new_ents): - ent.mutable_key().CopyFrom(key) - ent.mutable_entity_group().add_element().CopyFrom( - key.path().element(0)) - - txid = put_request.transaction().handle() - txdata = self.__transactions[txid] - assert (txdata.thread_id == thread.get_ident(), - "Transactions are single-threaded.") - for entity in entities: - txdata.entities[entity.key().Encode()] = (entity.key(), entity) - put_response.add_key().CopyFrom(entity.key()) - else: - super(RemoteDatastoreStub, self).MakeSyncCall( - 'datastore_v3', 'Put', put_request, put_response) - - def _Dynamic_Delete(self, delete_request, response): - if delete_request.has_transaction(): - txid = delete_request.transaction().handle() - txdata = self.__transactions[txid] - assert (txdata.thread_id == thread.get_ident(), - "Transactions are single-threaded.") - for key in delete_request.key_list(): - txdata.entities[key.Encode()] = (key, None) - else: - super(RemoteDatastoreStub, self).MakeSyncCall( - 'datastore_v3', 'Delete', delete_request, response) - - def _Dynamic_BeginTransaction(self, request, transaction): - self.__local_tx_lock.acquire() - try: - txid = self.__next_local_tx - self.__transactions[txid] = TransactionData(thread.get_ident()) - self.__next_local_tx += 1 - finally: - self.__local_tx_lock.release() - transaction.set_handle(txid) - - def _Dynamic_Commit(self, transaction, transaction_response): - txid = transaction.handle() - if txid not in self.__transactions: - raise apiproxy_errors.ApplicationError( - datastore_pb.Error.BAD_REQUEST, - 'Transaction %d not found.' % (txid,)) - - txdata = self.__transactions[txid] - assert (txdata.thread_id == thread.get_ident(), - "Transactions are single-threaded.") - del self.__transactions[txid] - - tx = remote_api_pb.TransactionRequest() - for key, hash in txdata.preconditions.values(): - precond = tx.add_precondition() - precond.mutable_key().CopyFrom(key) - if hash: - precond.set_hash(hash) - - puts = tx.mutable_puts() - deletes = tx.mutable_deletes() - for key, entity in txdata.entities.values(): - if entity: - puts.add_entity().CopyFrom(entity) - else: - deletes.add_key().CopyFrom(key) - - super(RemoteDatastoreStub, self).MakeSyncCall( - 'remote_datastore', 'Transaction', - tx, datastore_pb.PutResponse()) - - def _Dynamic_Rollback(self, transaction, transaction_response): - txid = transaction.handle() - self.__local_tx_lock.acquire() - try: - if txid not in self.__transactions: - raise apiproxy_errors.ApplicationError( - datastore_pb.Error.BAD_REQUEST, - 'Transaction %d not found.' % (txid,)) - - assert (txdata[txid].thread_id == thread.get_ident(), - "Transactions are single-threaded.") - del self.__transactions[txid] - finally: - self.__local_tx_lock.release() - - def _Dynamic_CreateIndex(self, index, id_response): - raise apiproxy_errors.CapabilityDisabledError( - 'The remote datastore does not support index manipulation.') - - def _Dynamic_UpdateIndex(self, index, void): - raise apiproxy_errors.CapabilityDisabledError( - 'The remote datastore does not support index manipulation.') - - def _Dynamic_DeleteIndex(self, index, void): - raise apiproxy_errors.CapabilityDisabledError( - 'The remote datastore does not support index manipulation.') - - -def ConfigureRemoteDatastore(app_id, path, auth_func, servername=None): - """Does necessary setup to allow easy remote access to an AppEngine datastore. - - Args: - app_id: The app_id of your app, as declared in app.yaml. - path: The path to the remote_api handler for your app - (for example, '/remote_api'). - auth_func: A function that takes no arguments and returns a - (username, password) tuple. This will be called if your application - requires authentication to access the remote_api handler (it should!) - and you do not already have a valid auth cookie. - servername: The hostname your app is deployed on. Defaults to - .appspot.com. - """ - if not servername: - servername = '%s.appspot.com' % (app_id,) - os.environ['APPLICATION_ID'] = app_id - apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() - server = appengine_rpc.HttpRpcServer(servername, auth_func, GetUserAgent(), - GetSourceName()) - stub = RemoteDatastoreStub(server, path) - apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', stub) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/ext/remote_api/handler.py --- a/thirdparty/google_appengine/google/appengine/ext/remote_api/handler.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/ext/remote_api/handler.py Thu Feb 12 12:30:36 2009 +0000 @@ -209,7 +209,7 @@ response.mutable_response().set_contents(response_data.Encode()) self.response.set_status(200) except Exception, e: - self.response.set_status(500) + self.response.set_status(200) response.mutable_exception().set_contents(pickle.dumps(e)) self.response.out.write(response.Encode()) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/ext/remote_api/remote_api_pb.py --- a/thirdparty/google_appengine/google/appengine/ext/remote_api/remote_api_pb.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/ext/remote_api/remote_api_pb.py Thu Feb 12 12:30:36 2009 +0000 @@ -44,8 +44,9 @@ self.service_name_ = x def clear_service_name(self): - self.has_service_name_ = 0 - self.service_name_ = "" + if self.has_service_name_: + self.has_service_name_ = 0 + self.service_name_ = "" def has_service_name(self): return self.has_service_name_ @@ -56,8 +57,9 @@ self.method_ = x def clear_method(self): - self.has_method_ = 0 - self.method_ = "" + if self.has_method_: + self.has_method_ = 0 + self.method_ = "" def has_method(self): return self.has_method_ @@ -201,8 +203,9 @@ def mutable_response(self): self.has_response_ = 1; return self.response() def clear_response(self): - self.has_response_ = 0; - if self.response_ is not None: self.response_.Clear() + if self.has_response_: + self.has_response_ = 0; + if self.response_ is not None: self.response_.Clear() def has_response(self): return self.has_response_ @@ -218,8 +221,9 @@ def mutable_exception(self): self.has_exception_ = 1; return self.exception() def clear_exception(self): - self.has_exception_ = 0; - if self.exception_ is not None: self.exception_.Clear() + if self.has_exception_: + self.has_exception_ = 0; + if self.exception_ is not None: self.exception_.Clear() def has_exception(self): return self.has_exception_ @@ -337,8 +341,9 @@ self.hash_ = x def clear_hash(self): - self.has_hash_ = 0 - self.hash_ = "" + if self.has_hash_: + self.has_hash_ = 0 + self.hash_ = "" def has_hash(self): return self.has_hash_ @@ -448,8 +453,9 @@ def mutable_puts(self): self.has_puts_ = 1; return self.puts() def clear_puts(self): - self.has_puts_ = 0; - if self.puts_ is not None: self.puts_.Clear() + if self.has_puts_: + self.has_puts_ = 0; + if self.puts_ is not None: self.puts_.Clear() def has_puts(self): return self.has_puts_ @@ -465,8 +471,9 @@ def mutable_deletes(self): self.has_deletes_ = 1; return self.deletes() def clear_deletes(self): - self.has_deletes_ = 0; - if self.deletes_ is not None: self.deletes_.Clear() + if self.has_deletes_: + self.has_deletes_ = 0; + if self.deletes_ is not None: self.deletes_.Clear() def has_deletes(self): return self.has_deletes_ diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/ext/remote_api/remote_api_stub.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/ext/remote_api/remote_api_stub.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,403 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""An apiproxy stub that calls a remote handler via HTTP. + +This allows easy remote access to the App Engine datastore, and potentially any +of the other App Engine APIs, using the same interface you use when accessing +the service locally. + +An example Python script: +--- +from google.appengine.ext import db +from google.appengine.ext.remote_api import remote_api_stub +from myapp import models +import getpass + +def auth_func(): + return (raw_input('Username:'), getpass.getpass('Password:')) + +remote_api_stub.ConfigureRemoteDatastore('my-app', '/remote_api', auth_func) + +# Now you can access the remote datastore just as if your code was running on +# App Engine! + +houses = models.House.all().fetch(100) +for a_house in q: + a_house.doors += 1 +db.put(houses) +--- + +A few caveats: +- Where possible, avoid iterating over queries directly. Fetching as many + results as you will need is faster and more efficient. +- If you need to iterate, consider instead fetching items in batches with a sort + order and constructing a new query starting from where the previous one left + off. The __key__ pseudo-property can be used as a sort key for this purpose, + and does not even require a custom index if you are iterating over all + entities of a given type. +- Likewise, it's a good idea to put entities in batches. Instead of calling put + for each individual entity, accumulate them and put them in batches using + db.put(), if you can. +- Requests and responses are still limited to 1MB each, so if you have large + entities or try and fetch or put many of them at once, your requests may fail. +""" + + + + + +import os +import pickle +import sha +import sys +import thread +import threading +from google.appengine.api import apiproxy_stub_map +from google.appengine.datastore import datastore_pb +from google.appengine.ext.remote_api import remote_api_pb +from google.appengine.runtime import apiproxy_errors +from google.appengine.tools import appengine_rpc + + +def GetUserAgent(): + """Determines the value of the 'User-agent' header to use for HTTP requests. + + Returns: + String containing the 'user-agent' header value, which includes the SDK + version, the platform information, and the version of Python; + e.g., "remote_api/1.0.1 Darwin/9.2.0 Python/2.5.2". + """ + product_tokens = [] + + product_tokens.append("Google-remote_api/1.0") + + product_tokens.append(appengine_rpc.GetPlatformToken()) + + python_version = ".".join(str(i) for i in sys.version_info) + product_tokens.append("Python/%s" % python_version) + + return " ".join(product_tokens) + + +def GetSourceName(): + return "Google-remote_api-1.0" + + +class TransactionData(object): + """Encapsulates data about an individual transaction.""" + + def __init__(self, thread_id): + self.thread_id = thread_id + self.preconditions = {} + self.entities = {} + + +class RemoteStub(object): + """A stub for calling services on a remote server over HTTP. + + You can use this to stub out any service that the remote server supports. + """ + + def __init__(self, server, path): + """Constructs a new RemoteStub that communicates with the specified server. + + Args: + server: An instance of a subclass of + google.appengine.tools.appengine_rpc.AbstractRpcServer. + path: The path to the handler this stub should send requests to. + """ + self._server = server + self._path = path + + def MakeSyncCall(self, service, call, request, response): + request_pb = remote_api_pb.Request() + request_pb.set_service_name(service) + request_pb.set_method(call) + request_pb.mutable_request().set_contents(request.Encode()) + + response_pb = remote_api_pb.Response() + response_pb.ParseFromString(self._server.Send(self._path, + request_pb.Encode())) + + if response_pb.has_exception(): + raise pickle.loads(response_pb.exception().contents()) + else: + response.ParseFromString(response_pb.response().contents()) + + +class RemoteDatastoreStub(RemoteStub): + """A specialised stub for accessing the App Engine datastore remotely. + + A specialised stub is required because there are some datastore operations + that preserve state between calls. This stub makes queries possible. + Transactions on the remote datastore are unfortunately still impossible. + """ + + def __init__(self, server, path): + super(RemoteDatastoreStub, self).__init__(server, path) + self.__queries = {} + self.__transactions = {} + + self.__next_local_cursor = 1 + self.__local_cursor_lock = threading.Lock() + self.__next_local_tx = 1 + self.__local_tx_lock = threading.Lock() + + def MakeSyncCall(self, service, call, request, response): + assert service == 'datastore_v3' + + explanation = [] + assert request.IsInitialized(explanation), explanation + + handler = getattr(self, '_Dynamic_' + call, None) + if handler: + handler(request, response) + else: + super(RemoteDatastoreStub, self).MakeSyncCall(service, call, request, + response) + + assert response.IsInitialized(explanation), explanation + + def _Dynamic_RunQuery(self, query, query_result): + self.__local_cursor_lock.acquire() + try: + cursor_id = self.__next_local_cursor + self.__next_local_cursor += 1 + finally: + self.__local_cursor_lock.release() + self.__queries[cursor_id] = query + + query_result.mutable_cursor().set_cursor(cursor_id) + query_result.set_more_results(True) + + def _Dynamic_Next(self, next_request, query_result): + cursor = next_request.cursor().cursor() + if cursor not in self.__queries: + raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, + 'Cursor %d not found' % cursor) + query = self.__queries[cursor] + + if query is None: + query_result.set_more_results(False) + return + + request = datastore_pb.Query() + request.CopyFrom(query) + if request.has_limit(): + request.set_limit(min(request.limit(), next_request.count())) + else: + request.set_limit(next_request.count()) + + super(RemoteDatastoreStub, self).MakeSyncCall( + 'remote_datastore', 'RunQuery', request, query_result) + + query.set_offset(query.offset() + query_result.result_size()) + if query.has_limit(): + query.set_limit(query.limit() - query_result.result_size()) + if not query_result.more_results(): + self.__queries[cursor] = None + + def _Dynamic_Get(self, get_request, get_response): + txid = None + if get_request.has_transaction(): + txid = get_request.transaction().handle() + txdata = self.__transactions[txid] + assert (txdata.thread_id == thread.get_ident(), + "Transactions are single-threaded.") + + keys = [(k, k.Encode()) for k in get_request.key_list()] + + new_request = datastore_pb.GetRequest() + for key, enckey in keys: + if enckey not in txdata.entities: + new_request.add_key().CopyFrom(key) + else: + new_request = get_request + + if new_request.key_size() > 0: + super(RemoteDatastoreStub, self).MakeSyncCall( + 'datastore_v3', 'Get', new_request, get_response) + + if txid is not None: + newkeys = new_request.key_list() + entities = get_response.entity_list() + for key, entity in zip(newkeys, entities): + entity_hash = None + if entity.has_entity(): + entity_hash = sha.new(entity.entity().Encode()).digest() + txdata.preconditions[key.Encode()] = (key, entity_hash) + + new_response = datastore_pb.GetResponse() + it = iter(get_response.entity_list()) + for key, enckey in keys: + if enckey in txdata.entities: + cached_entity = txdata.entities[enckey][1] + if cached_entity: + new_response.add_entity().mutable_entity().CopyFrom(cached_entity) + else: + new_response.add_entity() + else: + new_entity = it.next() + if new_entity.has_entity(): + assert new_entity.entity().key() == key + new_response.add_entity().CopyFrom(new_entity) + else: + new_response.add_entity() + get_response.CopyFrom(new_response) + + def _Dynamic_Put(self, put_request, put_response): + if put_request.has_transaction(): + entities = put_request.entity_list() + + requires_id = lambda x: x.id() == 0 and not x.has_name() + new_ents = [e for e in entities + if requires_id(e.key().path().element_list()[-1])] + id_request = remote_api_pb.PutRequest() + if new_ents: + for ent in new_ents: + e = id_request.add_entity() + e.mutable_key().CopyFrom(ent.key()) + e.mutable_entity_group() + id_response = datastore_pb.PutResponse() + super(RemoteDatastoreStub, self).MakeSyncCall( + 'remote_datastore', 'GetIDs', id_request, id_response) + assert id_request.entity_size() == id_response.key_size() + for key, ent in zip(id_response.key_list(), new_ents): + ent.mutable_key().CopyFrom(key) + ent.mutable_entity_group().add_element().CopyFrom( + key.path().element(0)) + + txid = put_request.transaction().handle() + txdata = self.__transactions[txid] + assert (txdata.thread_id == thread.get_ident(), + "Transactions are single-threaded.") + for entity in entities: + txdata.entities[entity.key().Encode()] = (entity.key(), entity) + put_response.add_key().CopyFrom(entity.key()) + else: + super(RemoteDatastoreStub, self).MakeSyncCall( + 'datastore_v3', 'Put', put_request, put_response) + + def _Dynamic_Delete(self, delete_request, response): + if delete_request.has_transaction(): + txid = delete_request.transaction().handle() + txdata = self.__transactions[txid] + assert (txdata.thread_id == thread.get_ident(), + "Transactions are single-threaded.") + for key in delete_request.key_list(): + txdata.entities[key.Encode()] = (key, None) + else: + super(RemoteDatastoreStub, self).MakeSyncCall( + 'datastore_v3', 'Delete', delete_request, response) + + def _Dynamic_BeginTransaction(self, request, transaction): + self.__local_tx_lock.acquire() + try: + txid = self.__next_local_tx + self.__transactions[txid] = TransactionData(thread.get_ident()) + self.__next_local_tx += 1 + finally: + self.__local_tx_lock.release() + transaction.set_handle(txid) + + def _Dynamic_Commit(self, transaction, transaction_response): + txid = transaction.handle() + if txid not in self.__transactions: + raise apiproxy_errors.ApplicationError( + datastore_pb.Error.BAD_REQUEST, + 'Transaction %d not found.' % (txid,)) + + txdata = self.__transactions[txid] + assert (txdata.thread_id == thread.get_ident(), + "Transactions are single-threaded.") + del self.__transactions[txid] + + tx = remote_api_pb.TransactionRequest() + for key, hash in txdata.preconditions.values(): + precond = tx.add_precondition() + precond.mutable_key().CopyFrom(key) + if hash: + precond.set_hash(hash) + + puts = tx.mutable_puts() + deletes = tx.mutable_deletes() + for key, entity in txdata.entities.values(): + if entity: + puts.add_entity().CopyFrom(entity) + else: + deletes.add_key().CopyFrom(key) + + super(RemoteDatastoreStub, self).MakeSyncCall( + 'remote_datastore', 'Transaction', + tx, datastore_pb.PutResponse()) + + def _Dynamic_Rollback(self, transaction, transaction_response): + txid = transaction.handle() + self.__local_tx_lock.acquire() + try: + if txid not in self.__transactions: + raise apiproxy_errors.ApplicationError( + datastore_pb.Error.BAD_REQUEST, + 'Transaction %d not found.' % (txid,)) + + assert (txdata[txid].thread_id == thread.get_ident(), + "Transactions are single-threaded.") + del self.__transactions[txid] + finally: + self.__local_tx_lock.release() + + def _Dynamic_CreateIndex(self, index, id_response): + raise apiproxy_errors.CapabilityDisabledError( + 'The remote datastore does not support index manipulation.') + + def _Dynamic_UpdateIndex(self, index, void): + raise apiproxy_errors.CapabilityDisabledError( + 'The remote datastore does not support index manipulation.') + + def _Dynamic_DeleteIndex(self, index, void): + raise apiproxy_errors.CapabilityDisabledError( + 'The remote datastore does not support index manipulation.') + + +def ConfigureRemoteDatastore(app_id, + path, + auth_func, + servername=None, + rpc_server_factory=appengine_rpc.HttpRpcServer): + """Does necessary setup to allow easy remote access to an AppEngine datastore. + + Args: + app_id: The app_id of your app, as declared in app.yaml. + path: The path to the remote_api handler for your app + (for example, '/remote_api'). + auth_func: A function that takes no arguments and returns a + (username, password) tuple. This will be called if your application + requires authentication to access the remote_api handler (it should!) + and you do not already have a valid auth cookie. + servername: The hostname your app is deployed on. Defaults to + .appspot.com. + rpc_server_factory: A factory to construct the rpc server for the datastore. + """ + if not servername: + servername = '%s.appspot.com' % (app_id,) + os.environ['APPLICATION_ID'] = app_id + apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() + server = rpc_server_factory(servername, auth_func, GetUserAgent(), + GetSourceName()) + stub = RemoteDatastoreStub(server, path) + apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', stub) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/ext/search/__init__.py --- a/thirdparty/google_appengine/google/appengine/ext/search/__init__.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/ext/search/__init__.py Thu Feb 12 12:30:36 2009 +0000 @@ -260,11 +260,30 @@ filter.set_op(datastore_pb.Query_Filter.EQUAL) prop = filter.add_property() prop.set_name(SearchableEntity._FULL_TEXT_INDEX_PROPERTY) + prop.set_multiple(len(keywords) > 1) prop.mutable_value().set_stringvalue(unicode(keyword).encode('utf-8')) return pb +class SearchableMultiQuery(datastore.MultiQuery): + """A multiquery that supports Search() by searching subqueries.""" + + def Search(self, *args, **kwargs): + """Add a search query, by trying to add it to all subqueries. + + Args: + args: Passed to Search on each subquery. + kwargs: Passed to Search on each subquery. + + Returns: + self for consistency with SearchableQuery. + """ + for q in self: + q.Search(*args, **kwargs) + return self + + class SearchableModel(db.Model): """A subclass of db.Model that supports full text search and indexing. @@ -290,7 +309,9 @@ def _get_query(self): """Wraps db.Query._get_query() and injects SearchableQuery.""" - query = db.Query._get_query(self, _query_class=SearchableQuery) + query = db.Query._get_query(self, + _query_class=SearchableQuery, + _multi_query_class=SearchableMultiQuery) if self._search_query: query.Search(self._search_query) return query diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/ext/webapp/util.py --- a/thirdparty/google_appengine/google/appengine/ext/webapp/util.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/ext/webapp/util.py Thu Feb 12 12:30:36 2009 +0000 @@ -38,7 +38,7 @@ @login_required def get(self): - user = users.GetCurrentUser(self) + user = users.get_current_user(self) self.response.out.write('Hello, ' + user.nickname()) We will redirect to a login page if the user is not logged in. We always @@ -49,9 +49,9 @@ if self.request.method != 'GET': raise webapp.Error('The check_login decorator can only be used for GET ' 'requests') - user = users.GetCurrentUser() + user = users.get_current_user() if not user: - self.redirect(users.CreateLoginURL(self.request.uri)) + self.redirect(users.create_login_url(self.request.uri)) return else: handler_method(self, *args) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/runtime/apiproxy.py --- a/thirdparty/google_appengine/google/appengine/runtime/apiproxy.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/runtime/apiproxy.py Thu Feb 12 12:30:36 2009 +0000 @@ -127,7 +127,7 @@ _apphosting_runtime___python__apiproxy.MakeCall( self.package, self.call, e.buffer(), self.__result_dict, - self.__MakeCallDone, self) + self.__MakeCallDone, self, deadline=(self.deadline or -1)) def __MakeCallDone(self): self.__state = RPC.FINISHING diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/runtime/apiproxy_errors.py --- a/thirdparty/google_appengine/google/appengine/runtime/apiproxy_errors.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/runtime/apiproxy_errors.py Thu Feb 12 12:30:36 2009 +0000 @@ -49,6 +49,7 @@ def __init__(self, application_error, error_detail=''): self.application_error = application_error self.error_detail = error_detail + Error.__init__(self, application_error) def __str__(self): return 'ApplicationError: %d %s' % (self.application_error, diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/tools/appcfg.py --- a/thirdparty/google_appengine/google/appengine/tools/appcfg.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/tools/appcfg.py Thu Feb 12 12:30:36 2009 +0000 @@ -71,6 +71,10 @@ appinfo.AppInfoExternal.ATTRIBUTES[appinfo.RUNTIME] = "python" +_api_versions = os.environ.get('GOOGLE_TEST_API_VERSIONS', '1') +_options = validation.Options(*_api_versions.split(',')) +appinfo.AppInfoExternal.ATTRIBUTES[appinfo.API_VERSION] = _options +del _api_versions, _options def StatusUpdate(msg): @@ -189,6 +193,29 @@ return version +def RetryWithBackoff(initial_delay, backoff_factor, max_tries, callable): + """Calls a function multiple times, backing off more and more each time. + + Args: + initial_delay: Initial delay after first try, in seconds. + backoff_factor: Delay will be multiplied by this factor after each try. + max_tries: Maximum number of tries. + callable: The method to call, will pass no arguments. + + Returns: + True if the function succeded in one of its tries. + + Raises: + Whatever the function raises--an exception will immediately stop retries. + """ + delay = initial_delay + while not callable() and max_tries > 0: + StatusUpdate("Will check again in %s seconds." % delay) + time.sleep(delay) + delay *= backoff_factor + max_tries -= 1 + return max_tries > 0 + class UpdateCheck(object): """Determines if the local SDK is the latest version. @@ -789,7 +816,7 @@ header uses UTC), and the client's local time is irrelevant. Args: - A posix timestamp giving current UTC time. + now: A posix timestamp giving current UTC time. Returns: A pseudo-posix timestamp giving current Pacific time. Passing @@ -913,6 +940,7 @@ hash of the file contents. in_transaction: True iff a transaction with the server has started. An AppVersionUpload can do only one transaction at a time. + deployed: True iff the Deploy method has been called. """ def __init__(self, server, config): @@ -930,6 +958,7 @@ self.version = self.config.version self.files = {} self.in_transaction = False + self.deployed = False def _Hash(self, content): """Compute the hash of the content. @@ -1059,6 +1088,8 @@ All the files returned by Begin() must have been uploaded with UploadFile() before Commit() can be called. + This tries the new 'deploy' method; if that fails it uses the old 'commit'. + Raises: Exception: Some required files were not uploaded. """ @@ -1066,10 +1097,65 @@ if self.files: raise Exception("Not all required files have been uploaded.") - StatusUpdate("Closing update.") - self.server.Send("/api/appversion/commit", app_id=self.app_id, + try: + self.Deploy() + if not RetryWithBackoff(1, 2, 8, self.IsReady): + logging.warning("Version still not ready to serve, aborting.") + raise Exception("Version not ready.") + self.StartServing() + except urllib2.HTTPError, e: + if e.code != 404: + raise + StatusUpdate("Closing update.") + self.server.Send("/api/appversion/commit", app_id=self.app_id, + version=self.version) + self.in_transaction = False + + def Deploy(self): + """Deploys the new app version but does not make it default. + + All the files returned by Begin() must have been uploaded with UploadFile() + before Deploy() can be called. + + Raises: + Exception: Some required files were not uploaded. + """ + assert self.in_transaction, "Begin() must be called before Deploy()." + if self.files: + raise Exception("Not all required files have been uploaded.") + + StatusUpdate("Deploying new version.") + self.server.Send("/api/appversion/deploy", app_id=self.app_id, version=self.version) - self.in_transaction = False + self.deployed = True + + def IsReady(self): + """Check if the new app version is ready to serve traffic. + + Raises: + Exception: Deploy has not yet been called. + + Returns: + True if the server returned the app is ready to serve. + """ + assert self.deployed, "Deploy() must be called before IsReady()." + + StatusUpdate("Checking if new version is ready to serve.") + result = self.server.Send("/api/appversion/isready", app_id=self.app_id, + version=self.version) + return result == "1" + + def StartServing(self): + """Start serving with the newly created version. + + Raises: + Exception: Deploy has not yet been called. + """ + assert self.deployed, "Deploy() must be called before IsReady()." + + StatusUpdate("Closing update: new version is ready to start serving.") + self.server.Send("/api/appversion/startserving", + app_id=self.app_id, version=self.version) def Rollback(self): """Rolls back the transaction if one is in progress.""" @@ -1140,12 +1226,13 @@ StatusUpdate("Uploaded %d files." % num_files) self.Commit() + except KeyboardInterrupt: logging.info("User interrupted. Aborting.") self.Rollback() raise except: - logging.error("An unexpected error occurred. Aborting.") + logging.exception("An unexpected error occurred. Aborting.") self.Rollback() raise @@ -1271,7 +1358,8 @@ rpc_server_class=appengine_rpc.HttpRpcServer, raw_input_fn=raw_input, password_input_fn=getpass.getpass, - error_fh=sys.stderr): + error_fh=sys.stderr, + update_check_class=UpdateCheck): """Initializer. Parses the cmdline and selects the Action to use. Initializes all of the attributes described in the class docstring. @@ -1284,6 +1372,7 @@ raw_input_fn: Function used for getting user email. password_input_fn: Function used for getting user password. error_fh: Unexpected HTTPErrors are printed to this file handle. + update_check_class: UpdateCheck class (can be replaced for testing). """ self.parser_class = parser_class self.argv = argv @@ -1291,6 +1380,7 @@ self.raw_input_fn = raw_input_fn self.password_input_fn = password_input_fn self.error_fh = error_fh + self.update_check_class = update_check_class self.parser = self._GetOptionParser() for action in self.actions.itervalues(): @@ -1571,7 +1661,7 @@ appyaml = self._ParseAppYaml(basepath) rpc_server = self._GetRpcServer() - updatecheck = UpdateCheck(rpc_server, appyaml) + updatecheck = self.update_check_class(rpc_server, appyaml) updatecheck.CheckForUpdates() appversion = AppVersionUpload(rpc_server, appyaml) @@ -1603,7 +1693,7 @@ parser: An instance of OptionsParser. """ parser.add_option("-S", "--max_size", type="int", dest="max_size", - default=1048576, metavar="SIZE", + default=10485760, metavar="SIZE", help="Maximum size of a file to upload.") def VacuumIndexes(self): diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/tools/bulkloader.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/thirdparty/google_appengine/google/appengine/tools/bulkloader.py Thu Feb 12 12:30:36 2009 +0000 @@ -0,0 +1,2588 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Imports CSV data over HTTP. + +Usage: + %(arg0)s [flags] + + --debug Show debugging information. (Optional) + --app_id= Application ID of endpoint (Optional for + *.appspot.com) + --auth_domain= The auth domain to use for logging in and for + UserProperties. (Default: gmail.com) + --bandwidth_limit= The maximum number of bytes per second for the + aggregate transfer of data to the server. Bursts + --batch_size= Number of Entity objects to include in each post to + the URL endpoint. The more data per row/Entity, the + smaller the batch size should be. (Default 10) + --config_file= File containing Model and Loader definitions. + (Required) + --db_filename= Specific progress database to write to, or to + resume from. If not supplied, then a new database + will be started, named: + bulkloader-progress-TIMESTAMP. + The special filename "skip" may be used to simply + skip reading/writing any progress information. + --filename= Path to the CSV file to import. (Required) + --http_limit= The maximum numer of HTTP requests per second to + send to the server. (Default: 8) + --kind= Name of the Entity object kind to put in the + datastore. (Required) + --num_threads= Number of threads to use for uploading entities + (Default 10) + may exceed this, but overall transfer rate is + restricted to this rate. (Default 250000) + --rps_limit= The maximum number of records per second to + transfer to the server. (Default: 20) + --url= URL endpoint to post to for importing data. + (Required) + +The exit status will be 0 on success, non-zero on import failure. + +Works with the remote_api mix-in library for google.appengine.ext.remote_api. +Please look there for documentation about how to setup the server side. + +Example: + +%(arg0)s --url=http://app.appspot.com/remote_api --kind=Model \ + --filename=data.csv --config_file=loader_config.py + +""" + + + +import csv +import getopt +import getpass +import logging +import new +import os +import Queue +import signal +import sys +import threading +import time +import traceback +import urllib2 +import urlparse + +from google.appengine.ext import db +from google.appengine.ext.remote_api import remote_api_stub +from google.appengine.tools import appengine_rpc + +try: + import sqlite3 +except ImportError: + pass + +UPLOADER_VERSION = '1' + +DEFAULT_THREAD_COUNT = 10 + +DEFAULT_BATCH_SIZE = 10 + +DEFAULT_QUEUE_SIZE = DEFAULT_THREAD_COUNT * 10 + +_THREAD_SHOULD_EXIT = '_THREAD_SHOULD_EXIT' + +STATE_READ = 0 +STATE_SENDING = 1 +STATE_SENT = 2 +STATE_NOT_SENT = 3 + +MINIMUM_THROTTLE_SLEEP_DURATION = 0.001 + +DATA_CONSUMED_TO_HERE = 'DATA_CONSUMED_TO_HERE' + +INITIAL_BACKOFF = 1.0 + +BACKOFF_FACTOR = 2.0 + + +DEFAULT_BANDWIDTH_LIMIT = 250000 + +DEFAULT_RPS_LIMIT = 20 + +DEFAULT_REQUEST_LIMIT = 8 + +BANDWIDTH_UP = 'http-bandwidth-up' +BANDWIDTH_DOWN = 'http-bandwidth-down' +REQUESTS = 'http-requests' +HTTPS_BANDWIDTH_UP = 'https-bandwidth-up' +HTTPS_BANDWIDTH_DOWN = 'https-bandwidth-down' +HTTPS_REQUESTS = 'https-requests' +RECORDS = 'records' + + +def StateMessage(state): + """Converts a numeric state identifier to a status message.""" + return ({ + STATE_READ: 'Batch read from file.', + STATE_SENDING: 'Sending batch to server.', + STATE_SENT: 'Batch successfully sent.', + STATE_NOT_SENT: 'Error while sending batch.' + }[state]) + + +class Error(Exception): + """Base-class for exceptions in this module.""" + + +class FatalServerError(Error): + """An unrecoverable error occurred while trying to post data to the server.""" + + +class ResumeError(Error): + """Error while trying to resume a partial upload.""" + + +class ConfigurationError(Error): + """Error in configuration options.""" + + +class AuthenticationError(Error): + """Error while trying to authenticate with the server.""" + + +def GetCSVGeneratorFactory(csv_filename, batch_size, + openfile=open, create_csv_reader=csv.reader): + """Return a factory that creates a CSV-based WorkItem generator. + + Args: + csv_filename: File on disk containing CSV data. + batch_size: Maximum number of CSV rows to stash into a WorkItem. + openfile: Used for dependency injection. + create_csv_reader: Used for dependency injection. + + Returns: A callable (accepting the Progress Queue and Progress + Generators as input) which creates the WorkItem generator. + """ + + def CreateGenerator(progress_queue, progress_generator): + """Initialize a CSV generator linked to a progress generator and queue. + + Args: + progress_queue: A ProgressQueue instance to send progress information. + progress_generator: A generator of progress information or None. + + Returns: + A CSVGenerator instance. + """ + return CSVGenerator(progress_queue, + progress_generator, + csv_filename, + batch_size, + openfile, + create_csv_reader) + return CreateGenerator + + +class CSVGenerator(object): + """Reads a CSV file and generates WorkItems containing batches of records.""" + + def __init__(self, + progress_queue, + progress_generator, + csv_filename, + batch_size, + openfile, + create_csv_reader): + """Initializes a CSV generator. + + Args: + progress_queue: A queue used for tracking progress information. + progress_generator: A generator of prior progress information, or None + if there is no prior status. + csv_filename: File on disk containing CSV data. + batch_size: Maximum number of CSV rows to stash into a WorkItem. + openfile: Used for dependency injection of 'open'. + create_csv_reader: Used for dependency injection of 'csv.reader'. + """ + self.progress_queue = progress_queue + self.progress_generator = progress_generator + self.csv_filename = csv_filename + self.batch_size = batch_size + self.openfile = openfile + self.create_csv_reader = create_csv_reader + self.line_number = 1 + self.column_count = None + self.read_rows = [] + self.reader = None + self.row_count = 0 + self.sent_count = 0 + + def _AdvanceTo(self, line): + """Advance the reader to the given line. + + Args: + line: A line number to advance to. + """ + while self.line_number < line: + self.reader.next() + self.line_number += 1 + self.row_count += 1 + self.sent_count += 1 + + def _ReadRows(self, key_start, key_end): + """Attempts to read and encode rows [key_start, key_end]. + + The encoded rows are stored in self.read_rows. + + Args: + key_start: The starting line number. + key_end: The ending line number. + + Raises: + StopIteration: if the reader runs out of rows + ResumeError: if there are an inconsistent number of columns. + """ + assert self.line_number == key_start + self.read_rows = [] + while self.line_number <= key_end: + row = self.reader.next() + self.row_count += 1 + if self.column_count is None: + self.column_count = len(row) + else: + if self.column_count != len(row): + raise ResumeError('Column count mismatch, %d: %s' % + (self.column_count, str(row))) + self.read_rows.append((self.line_number, row)) + self.line_number += 1 + + def _MakeItem(self, key_start, key_end, rows, progress_key=None): + """Makes a WorkItem containing the given rows, with the given keys. + + Args: + key_start: The start key for the WorkItem. + key_end: The end key for the WorkItem. + rows: A list of the rows for the WorkItem. + progress_key: The progress key for the WorkItem + + Returns: + A WorkItem instance for the given batch. + """ + assert rows + + item = WorkItem(self.progress_queue, rows, + key_start, key_end, + progress_key=progress_key) + + return item + + def Batches(self): + """Reads the CSV data file and generates WorkItems. + + Yields: + Instances of class WorkItem + + Raises: + ResumeError: If the progress database and data file indicate a different + number of rows. + """ + csv_file = self.openfile(self.csv_filename, 'r') + csv_content = csv_file.read() + if csv_content: + has_headers = csv.Sniffer().has_header(csv_content) + else: + has_headers = False + csv_file.seek(0) + self.reader = self.create_csv_reader(csv_file, skipinitialspace=True) + if has_headers: + logging.info('The CSV file appears to have a header line, skipping.') + self.reader.next() + + exhausted = False + + self.line_number = 1 + self.column_count = None + + logging.info('Starting import; maximum %d entities per post', + self.batch_size) + + state = None + if self.progress_generator is not None: + for progress_key, state, key_start, key_end in self.progress_generator: + if key_start: + try: + self._AdvanceTo(key_start) + self._ReadRows(key_start, key_end) + yield self._MakeItem(key_start, + key_end, + self.read_rows, + progress_key=progress_key) + except StopIteration: + logging.error('Mismatch between data file and progress database') + raise ResumeError( + 'Mismatch between data file and progress database') + elif state == DATA_CONSUMED_TO_HERE: + try: + self._AdvanceTo(key_end + 1) + except StopIteration: + state = None + + if self.progress_generator is None or state == DATA_CONSUMED_TO_HERE: + while not exhausted: + key_start = self.line_number + key_end = self.line_number + self.batch_size - 1 + try: + self._ReadRows(key_start, key_end) + except StopIteration: + exhausted = True + key_end = self.line_number - 1 + if key_start <= key_end: + yield self._MakeItem(key_start, key_end, self.read_rows) + + +class ReQueue(object): + """A special thread-safe queue. + + A ReQueue allows unfinished work items to be returned with a call to + reput(). When an item is reput, task_done() should *not* be called + in addition, getting an item that has been reput does not increase + the number of outstanding tasks. + + This class shares an interface with Queue.Queue and provides the + additional Reput method. + """ + + def __init__(self, + queue_capacity, + requeue_capacity=None, + queue_factory=Queue.Queue, + get_time=time.time): + """Initialize a ReQueue instance. + + Args: + queue_capacity: The number of items that can be put in the ReQueue. + requeue_capacity: The numer of items that can be reput in the ReQueue. + queue_factory: Used for dependency injection. + get_time: Used for dependency injection. + """ + if requeue_capacity is None: + requeue_capacity = queue_capacity + + self.get_time = get_time + self.queue = queue_factory(queue_capacity) + self.requeue = queue_factory(requeue_capacity) + self.lock = threading.Lock() + self.put_cond = threading.Condition(self.lock) + self.get_cond = threading.Condition(self.lock) + + def _DoWithTimeout(self, + action, + exc, + wait_cond, + done_cond, + lock, + timeout=None, + block=True): + """Performs the given action with a timeout. + + The action must be non-blocking, and raise an instance of exc on a + recoverable failure. If the action fails with an instance of exc, + we wait on wait_cond before trying again. Failure after the + timeout is reached is propagated as an exception. Success is + signalled by notifying on done_cond and returning the result of + the action. If action raises any exception besides an instance of + exc, it is immediately propagated. + + Args: + action: A callable that performs a non-blocking action. + exc: An exception type that is thrown by the action to indicate + a recoverable error. + wait_cond: A condition variable which should be waited on when + action throws exc. + done_cond: A condition variable to signal if the action returns. + lock: The lock used by wait_cond and done_cond. + timeout: A non-negative float indicating the maximum time to wait. + block: Whether to block if the action cannot complete immediately. + + Returns: + The result of the action, if it is successful. + + Raises: + ValueError: If the timeout argument is negative. + """ + if timeout is not None and timeout < 0.0: + raise ValueError('\'timeout\' must not be a negative number') + if not block: + timeout = 0.0 + result = None + success = False + start_time = self.get_time() + lock.acquire() + try: + while not success: + try: + result = action() + success = True + except Exception, e: + if not isinstance(e, exc): + raise e + if timeout is not None: + elapsed_time = self.get_time() - start_time + timeout -= elapsed_time + if timeout <= 0.0: + raise e + wait_cond.wait(timeout) + finally: + if success: + done_cond.notify() + lock.release() + return result + + def put(self, item, block=True, timeout=None): + """Put an item into the requeue. + + Args: + item: An item to add to the requeue. + block: Whether to block if the requeue is full. + timeout: Maximum on how long to wait until the queue is non-full. + + Raises: + Queue.Full if the queue is full and the timeout expires. + """ + def PutAction(): + self.queue.put(item, block=False) + self._DoWithTimeout(PutAction, + Queue.Full, + self.get_cond, + self.put_cond, + self.lock, + timeout=timeout, + block=block) + + def reput(self, item, block=True, timeout=None): + """Re-put an item back into the requeue. + + Re-putting an item does not increase the number of outstanding + tasks, so the reput item should be uniquely associated with an + item that was previously removed from the requeue and for which + task_done has not been called. + + Args: + item: An item to add to the requeue. + block: Whether to block if the requeue is full. + timeout: Maximum on how long to wait until the queue is non-full. + + Raises: + Queue.Full is the queue is full and the timeout expires. + """ + def ReputAction(): + self.requeue.put(item, block=False) + self._DoWithTimeout(ReputAction, + Queue.Full, + self.get_cond, + self.put_cond, + self.lock, + timeout=timeout, + block=block) + + def get(self, block=True, timeout=None): + """Get an item from the requeue. + + Args: + block: Whether to block if the requeue is empty. + timeout: Maximum on how long to wait until the requeue is non-empty. + + Returns: + An item from the requeue. + + Raises: + Queue.Empty if the queue is empty and the timeout expires. + """ + def GetAction(): + try: + result = self.requeue.get(block=False) + self.requeue.task_done() + except Queue.Empty: + result = self.queue.get(block=False) + return result + return self._DoWithTimeout(GetAction, + Queue.Empty, + self.put_cond, + self.get_cond, + self.lock, + timeout=timeout, + block=block) + + def join(self): + """Blocks until all of the items in the requeue have been processed.""" + self.queue.join() + + def task_done(self): + """Indicate that a previously enqueued item has been fully processed.""" + self.queue.task_done() + + def empty(self): + """Returns true if the requeue is empty.""" + return self.queue.empty() and self.requeue.empty() + + def get_nowait(self): + """Try to get an item from the queue without blocking.""" + return self.get(block=False) + + +class ThrottleHandler(urllib2.BaseHandler): + """A urllib2 handler for http and https requests that adds to a throttle.""" + + def __init__(self, throttle): + """Initialize a ThrottleHandler. + + Args: + throttle: A Throttle instance to call for bandwidth and http/https request + throttling. + """ + self.throttle = throttle + + def AddRequest(self, throttle_name, req): + """Add to bandwidth throttle for given request. + + Args: + throttle_name: The name of the bandwidth throttle to add to. + req: The request whose size will be added to the throttle. + """ + size = 0 + for key, value in req.headers.iteritems(): + size += len('%s: %s\n' % (key, value)) + for key, value in req.unredirected_hdrs.iteritems(): + size += len('%s: %s\n' % (key, value)) + (unused_scheme, + unused_host_port, url_path, + unused_query, unused_fragment) = urlparse.urlsplit(req.get_full_url()) + size += len('%s %s HTTP/1.1\n' % (req.get_method(), url_path)) + data = req.get_data() + if data: + size += len(data) + self.throttle.AddTransfer(throttle_name, size) + + def AddResponse(self, throttle_name, res): + """Add to bandwidth throttle for given response. + + Args: + throttle_name: The name of the bandwidth throttle to add to. + res: The response whose size will be added to the throttle. + """ + content = res.read() + def ReturnContent(): + return content + res.read = ReturnContent + size = len(content) + headers = res.info() + for key, value in headers.items(): + size += len('%s: %s\n' % (key, value)) + self.throttle.AddTransfer(throttle_name, size) + + def http_request(self, req): + """Process an HTTP request. + + If the throttle is over quota, sleep first. Then add request size to + throttle before returning it to be sent. + + Args: + req: A urllib2.Request object. + + Returns: + The request passed in. + """ + self.throttle.Sleep() + self.AddRequest(BANDWIDTH_UP, req) + return req + + def https_request(self, req): + """Process an HTTPS request. + + If the throttle is over quota, sleep first. Then add request size to + throttle before returning it to be sent. + + Args: + req: A urllib2.Request object. + + Returns: + The request passed in. + """ + self.throttle.Sleep() + self.AddRequest(HTTPS_BANDWIDTH_UP, req) + return req + + def http_response(self, unused_req, res): + """Process an HTTP response. + + The size of the response is added to the bandwidth throttle and the request + throttle is incremented by one. + + Args: + unused_req: The urllib2 request for this response. + res: A urllib2 response object. + + Returns: + The response passed in. + """ + self.AddResponse(BANDWIDTH_DOWN, res) + self.throttle.AddTransfer(REQUESTS, 1) + return res + + def https_response(self, unused_req, res): + """Process an HTTPS response. + + The size of the response is added to the bandwidth throttle and the request + throttle is incremented by one. + + Args: + unused_req: The urllib2 request for this response. + res: A urllib2 response object. + + Returns: + The response passed in. + """ + self.AddResponse(HTTPS_BANDWIDTH_DOWN, res) + self.throttle.AddTransfer(HTTPS_REQUESTS, 1) + return res + + +class ThrottledHttpRpcServer(appengine_rpc.HttpRpcServer): + """Provides a simplified RPC-style interface for HTTP requests. + + This RPC server uses a Throttle to prevent exceeding quotas. + """ + + def __init__(self, throttle, request_manager, *args, **kwargs): + """Initialize a ThrottledHttpRpcServer. + + Also sets request_manager.rpc_server to the ThrottledHttpRpcServer instance. + + Args: + throttle: A Throttles instance. + request_manager: A RequestManager instance. + args: Positional arguments to pass through to + appengine_rpc.HttpRpcServer.__init__ + kwargs: Keyword arguments to pass through to + appengine_rpc.HttpRpcServer.__init__ + """ + self.throttle = throttle + appengine_rpc.HttpRpcServer.__init__(self, *args, **kwargs) + request_manager.rpc_server = self + + def _GetOpener(self): + """Returns an OpenerDirector that supports cookies and ignores redirects. + + Returns: + A urllib2.OpenerDirector object. + """ + opener = appengine_rpc.HttpRpcServer._GetOpener(self) + opener.add_handler(ThrottleHandler(self.throttle)) + + return opener + + +def ThrottledHttpRpcServerFactory(throttle, request_manager): + """Create a factory to produce ThrottledHttpRpcServer for a given throttle. + + Args: + throttle: A Throttle instance to use for the ThrottledHttpRpcServer. + request_manager: A RequestManager instance. + + Returns: + A factory to produce a ThrottledHttpRpcServer. + """ + def MakeRpcServer(*args, **kwargs): + kwargs['account_type'] = 'HOSTED_OR_GOOGLE' + kwargs['save_cookies'] = True + return ThrottledHttpRpcServer(throttle, request_manager, *args, **kwargs) + return MakeRpcServer + + +class RequestManager(object): + """A class which wraps a connection to the server.""" + + source = 'google-bulkloader-%s' % UPLOADER_VERSION + user_agent = source + + def __init__(self, + app_id, + host_port, + url_path, + kind, + throttle): + """Initialize a RequestManager object. + + Args: + app_id: String containing the application id for requests. + host_port: String containing the "host:port" pair; the port is optional. + url_path: partial URL (path) to post entity data to. + kind: Kind of the Entity records being posted. + throttle: A Throttle instance. + """ + self.app_id = app_id + self.host_port = host_port + self.host = host_port.split(':')[0] + if url_path and url_path[0] != '/': + url_path = '/' + url_path + self.url_path = url_path + self.kind = kind + self.throttle = throttle + self.credentials = None + throttled_rpc_server_factory = ThrottledHttpRpcServerFactory( + self.throttle, self) + logging.debug('Configuring remote_api. app_id = %s, url_path = %s, ' + 'servername = %s' % (app_id, url_path, host_port)) + remote_api_stub.ConfigureRemoteDatastore( + app_id, + url_path, + self.AuthFunction, + servername=host_port, + rpc_server_factory=throttled_rpc_server_factory) + self.authenticated = False + + def Authenticate(self): + """Invoke authentication if necessary.""" + self.rpc_server.Send(self.url_path, payload=None) + self.authenticated = True + + def AuthFunction(self, + raw_input_fn=raw_input, + password_input_fn=getpass.getpass): + """Prompts the user for a username and password. + + Caches the results the first time it is called and returns the + same result every subsequent time. + + Args: + raw_input_fn: Used for dependency injection. + password_input_fn: Used for dependency injection. + + Returns: + A pair of the username and password. + """ + if self.credentials is not None: + return self.credentials + print 'Please enter login credentials for %s (%s)' % ( + self.host, self.app_id) + email = raw_input_fn('Email: ') + if email: + password_prompt = 'Password for %s: ' % email + password = password_input_fn(password_prompt) + else: + password = None + self.credentials = (email, password) + return self.credentials + + def _GetHeaders(self): + """Constructs a dictionary of extra headers to send with a request.""" + headers = { + 'GAE-Uploader-Version': UPLOADER_VERSION, + 'GAE-Uploader-Kind': self.kind + } + return headers + + def EncodeContent(self, rows): + """Encodes row data to the wire format. + + Args: + rows: A list of pairs of a line number and a list of column values. + + Returns: + A list of db.Model instances. + """ + try: + loader = Loader.RegisteredLoaders()[self.kind] + except KeyError: + logging.error('No Loader defined for kind %s.' % self.kind) + raise ConfigurationError('No Loader defined for kind %s.' % self.kind) + entities = [] + for line_number, values in rows: + key = loader.GenerateKey(line_number, values) + entity = loader.CreateEntity(values, key_name=key) + entities.extend(entity) + + return entities + + def PostEntities(self, item): + """Posts Entity records to a remote endpoint over HTTP. + + Args: + item: A workitem containing the entities to post. + + Returns: + A pair of the estimated size of the request in bytes and the response + from the server as a str. + """ + entities = item.content + db.put(entities) + + +class WorkItem(object): + """Holds a unit of uploading work. + + A WorkItem represents a number of entities that need to be uploaded to + Google App Engine. These entities are encoded in the "content" field of + the WorkItem, and will be POST'd as-is to the server. + + The entities are identified by a range of numeric keys, inclusively. In + the case of a resumption of an upload, or a replay to correct errors, + these keys must be able to identify the same set of entities. + + Note that keys specify a range. The entities do not have to sequentially + fill the entire range, they must simply bound a range of valid keys. + """ + + def __init__(self, progress_queue, rows, key_start, key_end, + progress_key=None): + """Initialize the WorkItem instance. + + Args: + progress_queue: A queue used for tracking progress information. + rows: A list of pairs of a line number and a list of column values + key_start: The (numeric) starting key, inclusive. + key_end: The (numeric) ending key, inclusive. + progress_key: If this WorkItem represents state from a prior run, + then this will be the key within the progress database. + """ + self.state = STATE_READ + + self.progress_queue = progress_queue + + assert isinstance(key_start, (int, long)) + assert isinstance(key_end, (int, long)) + assert key_start <= key_end + + self.key_start = key_start + self.key_end = key_end + self.progress_key = progress_key + + self.progress_event = threading.Event() + + self.rows = rows + self.content = None + self.count = len(rows) + + def MarkAsRead(self): + """Mark this WorkItem as read/consumed from the data source.""" + + assert self.state == STATE_READ + + self._StateTransition(STATE_READ, blocking=True) + + assert self.progress_key is not None + + def MarkAsSending(self): + """Mark this WorkItem as in-process on being uploaded to the server.""" + + assert self.state == STATE_READ or self.state == STATE_NOT_SENT + assert self.progress_key is not None + + self._StateTransition(STATE_SENDING, blocking=True) + + def MarkAsSent(self): + """Mark this WorkItem as sucessfully-sent to the server.""" + + assert self.state == STATE_SENDING + assert self.progress_key is not None + + self._StateTransition(STATE_SENT, blocking=False) + + def MarkAsError(self): + """Mark this WorkItem as required manual error recovery.""" + + assert self.state == STATE_SENDING + assert self.progress_key is not None + + self._StateTransition(STATE_NOT_SENT, blocking=True) + + def _StateTransition(self, new_state, blocking=False): + """Transition the work item to a new state, storing progress information. + + Args: + new_state: The state to transition to. + blocking: Whether to block for the progress thread to acknowledge the + transition. + """ + logging.debug('[%s-%s] %s' % + (self.key_start, self.key_end, StateMessage(self.state))) + assert not self.progress_event.isSet() + + self.state = new_state + + self.progress_queue.put(self) + + if blocking: + self.progress_event.wait() + + self.progress_event.clear() + + + +def InterruptibleSleep(sleep_time): + """Puts thread to sleep, checking this threads exit_flag twice a second. + + Args: + sleep_time: Time to sleep. + """ + slept = 0.0 + epsilon = .0001 + thread = threading.currentThread() + while slept < sleep_time - epsilon: + remaining = sleep_time - slept + this_sleep_time = min(remaining, 0.5) + time.sleep(this_sleep_time) + slept += this_sleep_time + if thread.exit_flag: + return + + +class ThreadGate(object): + """Manage the number of active worker threads. + + The ThreadGate limits the number of threads that are simultaneously + uploading batches of records in order to implement adaptive rate + control. The number of simultaneous upload threads that it takes to + start causing timeout varies widely over the course of the day, so + adaptive rate control allows the uploader to do many uploads while + reducing the error rate and thus increasing the throughput. + + Initially the ThreadGate allows only one uploader thread to be active. + For each successful upload, another thread is activated and for each + failed upload, the number of active threads is reduced by one. + """ + + def __init__(self, enabled, sleep=InterruptibleSleep): + self.enabled = enabled + self.enabled_count = 1 + self.lock = threading.Lock() + self.thread_semaphore = threading.Semaphore(self.enabled_count) + self._threads = [] + self.backoff_time = 0 + self.sleep = sleep + + def Register(self, thread): + """Register a thread with the thread gate.""" + self._threads.append(thread) + + def Threads(self): + """Yields the registered threads.""" + for thread in self._threads: + yield thread + + def EnableThread(self): + """Enable one more worker thread.""" + self.lock.acquire() + try: + self.enabled_count += 1 + finally: + self.lock.release() + self.thread_semaphore.release() + + def EnableAllThreads(self): + """Enable all worker threads.""" + for unused_idx in range(len(self._threads) - self.enabled_count): + self.EnableThread() + + def StartWork(self): + """Starts a critical section in which the number of workers is limited. + + If thread throttling is enabled then this method starts a critical + section which allows self.enabled_count simultaneously operating + threads. The critical section is ended by calling self.FinishWork(). + """ + if self.enabled: + self.thread_semaphore.acquire() + if self.backoff_time > 0.0: + if not threading.currentThread().exit_flag: + logging.info('Backing off: %.1f seconds', + self.backoff_time) + self.sleep(self.backoff_time) + + def FinishWork(self): + """Ends a critical section started with self.StartWork().""" + if self.enabled: + self.thread_semaphore.release() + + def IncreaseWorkers(self): + """Informs the throttler that an item was successfully sent. + + If thread throttling is enabled, this method will cause an + additional thread to run in the critical section. + """ + if self.enabled: + if self.backoff_time > 0.0: + logging.info('Resetting backoff to 0.0') + self.backoff_time = 0.0 + do_enable = False + self.lock.acquire() + try: + if self.enabled and len(self._threads) > self.enabled_count: + do_enable = True + self.enabled_count += 1 + finally: + self.lock.release() + if do_enable: + self.thread_semaphore.release() + + def DecreaseWorkers(self): + """Informs the thread_gate that an item failed to send. + + If thread throttling is enabled, this method will cause the + throttler to allow one fewer thread in the critical section. If + there is only one thread remaining, failures will result in + exponential backoff until there is a success. + """ + if self.enabled: + do_disable = False + self.lock.acquire() + try: + if self.enabled: + if self.enabled_count > 1: + do_disable = True + self.enabled_count -= 1 + else: + if self.backoff_time == 0.0: + self.backoff_time = INITIAL_BACKOFF + else: + self.backoff_time *= BACKOFF_FACTOR + finally: + self.lock.release() + if do_disable: + self.thread_semaphore.acquire() + + +class Throttle(object): + """A base class for upload rate throttling. + + Transferring large number of records, too quickly, to an application + could trigger quota limits and cause the transfer process to halt. + In order to stay within the application's quota, we throttle the + data transfer to a specified limit (across all transfer threads). + This limit defaults to about half of the Google App Engine default + for an application, but can be manually adjusted faster/slower as + appropriate. + + This class tracks a moving average of some aspect of the transfer + rate (bandwidth, records per second, http connections per + second). It keeps two windows of counts of bytes transferred, on a + per-thread basis. One block is the "current" block, and the other is + the "prior" block. It will rotate the counts from current to prior + when ROTATE_PERIOD has passed. Thus, the current block will + represent from 0 seconds to ROTATE_PERIOD seconds of activity + (determined by: time.time() - self.last_rotate). The prior block + will always represent a full ROTATE_PERIOD. + + Sleeping is performed just before a transfer of another block, and is + based on the counts transferred *before* the next transfer. It really + does not matter how much will be transferred, but only that for all the + data transferred SO FAR that we have interspersed enough pauses to + ensure the aggregate transfer rate is within the specified limit. + + These counts are maintained on a per-thread basis, so we do not require + any interlocks around incrementing the counts. There IS an interlock on + the rotation of the counts because we do not want multiple threads to + multiply-rotate the counts. + + There are various race conditions in the computation and collection + of these counts. We do not require precise values, but simply to + keep the overall transfer within the bandwidth limits. If a given + pause is a little short, or a little long, then the aggregate delays + will be correct. + """ + + ROTATE_PERIOD = 600 + + def __init__(self, + get_time=time.time, + thread_sleep=InterruptibleSleep, + layout=None): + self.get_time = get_time + self.thread_sleep = thread_sleep + + self.start_time = get_time() + self.transferred = {} + self.prior_block = {} + self.totals = {} + self.throttles = {} + + self.last_rotate = {} + self.rotate_mutex = {} + if layout: + self.AddThrottles(layout) + + def AddThrottle(self, name, limit): + self.throttles[name] = limit + self.transferred[name] = {} + self.prior_block[name] = {} + self.totals[name] = {} + self.last_rotate[name] = self.get_time() + self.rotate_mutex[name] = threading.Lock() + + def AddThrottles(self, layout): + for key, value in layout.iteritems(): + self.AddThrottle(key, value) + + def Register(self, thread): + """Register this thread with the throttler.""" + thread_name = thread.getName() + for throttle_name in self.throttles.iterkeys(): + self.transferred[throttle_name][thread_name] = 0 + self.prior_block[throttle_name][thread_name] = 0 + self.totals[throttle_name][thread_name] = 0 + + def VerifyName(self, throttle_name): + if throttle_name not in self.throttles: + raise AssertionError('%s is not a registered throttle' % throttle_name) + + def AddTransfer(self, throttle_name, token_count): + """Add a count to the amount this thread has transferred. + + Each time a thread transfers some data, it should call this method to + note the amount sent. The counts may be rotated if sufficient time + has passed since the last rotation. + + Note: this method should only be called by the BulkLoaderThread + instances. The token count is allocated towards the + "current thread". + + Args: + throttle_name: The name of the throttle to add to. + token_count: The number to add to the throttle counter. + """ + self.VerifyName(throttle_name) + transferred = self.transferred[throttle_name] + transferred[threading.currentThread().getName()] += token_count + + if self.last_rotate[throttle_name] + self.ROTATE_PERIOD < self.get_time(): + self._RotateCounts(throttle_name) + + def Sleep(self, throttle_name=None): + """Possibly sleep in order to limit the transfer rate. + + Note that we sleep based on *prior* transfers rather than what we + may be about to transfer. The next transfer could put us under/over + and that will be rectified *after* that transfer. Net result is that + the average transfer rate will remain within bounds. Spiky behavior + or uneven rates among the threads could possibly bring the transfer + rate above the requested limit for short durations. + + Args: + throttle_name: The name of the throttle to sleep on. If None or + omitted, then sleep on all throttles. + """ + if throttle_name is None: + for throttle_name in self.throttles: + self.Sleep(throttle_name=throttle_name) + return + + self.VerifyName(throttle_name) + + thread = threading.currentThread() + + while True: + duration = self.get_time() - self.last_rotate[throttle_name] + + total = 0 + for count in self.prior_block[throttle_name].values(): + total += count + + if total: + duration += self.ROTATE_PERIOD + + for count in self.transferred[throttle_name].values(): + total += count + + sleep_time = (float(total) / self.throttles[throttle_name]) - duration + + if sleep_time < MINIMUM_THROTTLE_SLEEP_DURATION: + break + + logging.debug('[%s] Throttling on %s. Sleeping for %.1f ms ' + '(duration=%.1f ms, total=%d)', + thread.getName(), throttle_name, + sleep_time * 1000, duration * 1000, total) + self.thread_sleep(sleep_time) + if thread.exit_flag: + break + self._RotateCounts(throttle_name) + + def _RotateCounts(self, throttle_name): + """Rotate the transfer counters. + + If sufficient time has passed, then rotate the counters from active to + the prior-block of counts. + + This rotation is interlocked to ensure that multiple threads do not + over-rotate the counts. + + Args: + throttle_name: The name of the throttle to rotate. + """ + self.VerifyName(throttle_name) + self.rotate_mutex[throttle_name].acquire() + try: + next_rotate_time = self.last_rotate[throttle_name] + self.ROTATE_PERIOD + if next_rotate_time >= self.get_time(): + return + + for name, count in self.transferred[throttle_name].items(): + + + self.prior_block[throttle_name][name] = count + self.transferred[throttle_name][name] = 0 + + self.totals[throttle_name][name] += count + + self.last_rotate[throttle_name] = self.get_time() + + finally: + self.rotate_mutex[throttle_name].release() + + def TotalTransferred(self, throttle_name): + """Return the total transferred, and over what period. + + Args: + throttle_name: The name of the throttle to total. + + Returns: + A tuple of the total count and running time for the given throttle name. + """ + total = 0 + for count in self.totals[throttle_name].values(): + total += count + for count in self.transferred[throttle_name].values(): + total += count + return total, self.get_time() - self.start_time + + +class _ThreadBase(threading.Thread): + """Provide some basic features for the threads used in the uploader. + + This abstract base class is used to provide some common features: + + * Flag to ask thread to exit as soon as possible. + * Record exit/error status for the primary thread to pick up. + * Capture exceptions and record them for pickup. + * Some basic logging of thread start/stop. + * All threads are "daemon" threads. + * Friendly names for presenting to users. + + Concrete sub-classes must implement PerformWork(). + + Either self.NAME should be set or GetFriendlyName() be overridden to + return a human-friendly name for this thread. + + The run() method starts the thread and prints start/exit messages. + + self.exit_flag is intended to signal that this thread should exit + when it gets the chance. PerformWork() should check self.exit_flag + whenever it has the opportunity to exit gracefully. + """ + + def __init__(self): + threading.Thread.__init__(self) + + self.setDaemon(True) + + self.exit_flag = False + self.error = None + + def run(self): + """Perform the work of the thread.""" + logging.info('[%s] %s: started', self.getName(), self.__class__.__name__) + + try: + self.PerformWork() + except: + self.error = sys.exc_info()[1] + logging.exception('[%s] %s:', self.getName(), self.__class__.__name__) + + logging.info('[%s] %s: exiting', self.getName(), self.__class__.__name__) + + def PerformWork(self): + """Perform the thread-specific work.""" + raise NotImplementedError() + + def CheckError(self): + """If an error is present, then log it.""" + if self.error: + logging.error('Error in %s: %s', self.GetFriendlyName(), self.error) + + def GetFriendlyName(self): + """Returns a human-friendly description of the thread.""" + if hasattr(self, 'NAME'): + return self.NAME + return 'unknown thread' + + +class BulkLoaderThread(_ThreadBase): + """A thread which transmits entities to the server application. + + This thread will read WorkItem instances from the work_queue and upload + the entities to the server application. Progress information will be + pushed into the progress_queue as the work is being performed. + + If a BulkLoaderThread encounters a transient error, the entities will be + resent, if a fatal error is encoutered the BulkLoaderThread exits. + """ + + def __init__(self, + work_queue, + throttle, + thread_gate, + request_manager): + """Initialize the BulkLoaderThread instance. + + Args: + work_queue: A queue containing WorkItems for processing. + throttle: A Throttles to control upload bandwidth. + thread_gate: A ThreadGate to control number of simultaneous uploads. + request_manager: A RequestManager instance. + """ + _ThreadBase.__init__(self) + + self.work_queue = work_queue + self.throttle = throttle + self.thread_gate = thread_gate + + self.request_manager = request_manager + + def PerformWork(self): + """Perform the work of a BulkLoaderThread.""" + while not self.exit_flag: + success = False + self.thread_gate.StartWork() + try: + try: + item = self.work_queue.get(block=True, timeout=1.0) + except Queue.Empty: + continue + if item == _THREAD_SHOULD_EXIT: + break + + logging.debug('[%s] Got work item [%d-%d]', + self.getName(), item.key_start, item.key_end) + + try: + + item.MarkAsSending() + try: + if item.content is None: + item.content = self.request_manager.EncodeContent(item.rows) + try: + self.request_manager.PostEntities(item) + success = True + logging.debug( + '[%d-%d] Sent %d entities', + item.key_start, item.key_end, item.count) + self.throttle.AddTransfer(RECORDS, item.count) + except (db.InternalError, db.NotSavedError, db.Timeout), e: + logging.debug('Caught non-fatal error: %s', e) + except urllib2.HTTPError, e: + if e.code == 403 or (e.code >= 500 and e.code < 600): + logging.debug('Caught HTTP error %d', e.code) + logging.debug('%s', e.read()) + else: + raise e + + except: + self.error = sys.exc_info()[1] + logging.exception('[%s] %s: caught exception %s', self.getName(), + self.__class__.__name__, str(sys.exc_info())) + raise + + finally: + if success: + item.MarkAsSent() + self.thread_gate.IncreaseWorkers() + self.work_queue.task_done() + else: + item.MarkAsError() + self.thread_gate.DecreaseWorkers() + try: + self.work_queue.reput(item, block=False) + except Queue.Full: + logging.error('[%s] Failed to reput work item.', self.getName()) + raise Error('Failed to reput work item') + logging.info('[%d-%d] %s', + item.key_start, item.key_end, StateMessage(item.state)) + + finally: + self.thread_gate.FinishWork() + + + def GetFriendlyName(self): + """Returns a human-friendly name for this thread.""" + return 'worker [%s]' % self.getName() + + +class DataSourceThread(_ThreadBase): + """A thread which reads WorkItems and pushes them into queue. + + This thread will read/consume WorkItems from a generator (produced by + the generator factory). These WorkItems will then be pushed into the + work_queue. Note that reading will block if/when the work_queue becomes + full. Information on content consumed from the generator will be pushed + into the progress_queue. + """ + + NAME = 'data source thread' + + def __init__(self, + work_queue, + progress_queue, + workitem_generator_factory, + progress_generator_factory): + """Initialize the DataSourceThread instance. + + Args: + work_queue: A queue containing WorkItems for processing. + progress_queue: A queue used for tracking progress information. + workitem_generator_factory: A factory that creates a WorkItem generator + progress_generator_factory: A factory that creates a generator which + produces prior progress status, or None if there is no prior status + to use. + """ + _ThreadBase.__init__(self) + + self.work_queue = work_queue + self.progress_queue = progress_queue + self.workitem_generator_factory = workitem_generator_factory + self.progress_generator_factory = progress_generator_factory + self.entity_count = 0 + + def PerformWork(self): + """Performs the work of a DataSourceThread.""" + if self.progress_generator_factory: + progress_gen = self.progress_generator_factory() + else: + progress_gen = None + + content_gen = self.workitem_generator_factory(self.progress_queue, + progress_gen) + + self.sent_count = 0 + self.read_count = 0 + self.read_all = False + + for item in content_gen.Batches(): + item.MarkAsRead() + + while not self.exit_flag: + try: + self.work_queue.put(item, block=True, timeout=1.0) + self.entity_count += item.count + break + except Queue.Full: + pass + + if self.exit_flag: + break + + if not self.exit_flag: + self.read_all = True + self.read_count = content_gen.row_count + self.sent_count = content_gen.sent_count + + + +def _RunningInThread(thread): + """Return True if we are running within the specified thread.""" + return threading.currentThread().getName() == thread.getName() + + +class ProgressDatabase(object): + """Persistently record all progress information during an upload. + + This class wraps a very simple SQLite database which records each of + the relevant details from the WorkItem instances. If the uploader is + resumed, then data is replayed out of the database. + """ + + def __init__(self, db_filename, commit_periodicity=100): + """Initialize the ProgressDatabase instance. + + Args: + db_filename: The name of the SQLite database to use. + commit_periodicity: How many operations to perform between commits. + """ + self.db_filename = db_filename + + logging.info('Using progress database: %s', db_filename) + self.primary_conn = sqlite3.connect(db_filename, isolation_level=None) + self.primary_thread = threading.currentThread() + + self.progress_conn = None + self.progress_thread = None + + self.operation_count = 0 + self.commit_periodicity = commit_periodicity + + self.prior_key_end = None + + try: + self.primary_conn.execute( + """create table progress ( + id integer primary key autoincrement, + state integer not null, + key_start integer not null, + key_end integer not null + ) + """) + except sqlite3.OperationalError, e: + if 'already exists' not in e.message: + raise + + try: + self.primary_conn.execute('create index i_state on progress (state)') + except sqlite3.OperationalError, e: + if 'already exists' not in e.message: + raise + + def ThreadComplete(self): + """Finalize any operations the progress thread has performed. + + The database aggregates lots of operations into a single commit, and + this method is used to commit any pending operations as the thread + is about to shut down. + """ + if self.progress_conn: + self._MaybeCommit(force_commit=True) + + def _MaybeCommit(self, force_commit=False): + """Periodically commit changes into the SQLite database. + + Committing every operation is quite expensive, and slows down the + operation of the script. Thus, we only commit after every N operations, + as determined by the self.commit_periodicity value. Optionally, the + caller can force a commit. + + Args: + force_commit: Pass True in order for a commit to occur regardless + of the current operation count. + """ + self.operation_count += 1 + if force_commit or (self.operation_count % self.commit_periodicity) == 0: + self.progress_conn.commit() + + def _OpenProgressConnection(self): + """Possibly open a database connection for the progress tracker thread. + + If the connection is not open (for the calling thread, which is assumed + to be the progress tracker thread), then open it. We also open a couple + cursors for later use (and reuse). + """ + if self.progress_conn: + return + + assert not _RunningInThread(self.primary_thread) + + self.progress_thread = threading.currentThread() + + self.progress_conn = sqlite3.connect(self.db_filename) + + self.insert_cursor = self.progress_conn.cursor() + self.update_cursor = self.progress_conn.cursor() + + def HasUnfinishedWork(self): + """Returns True if the database has progress information. + + Note there are two basic cases for progress information: + 1) All saved records indicate a successful upload. In this case, we + need to skip everything transmitted so far and then send the rest. + 2) Some records for incomplete transfer are present. These need to be + sent again, and then we resume sending after all the successful + data. + + Returns: + True if the database has progress information, False otherwise. + + Raises: + ResumeError: If there is an error reading the progress database. + """ + assert _RunningInThread(self.primary_thread) + + cursor = self.primary_conn.cursor() + cursor.execute('select count(*) from progress') + row = cursor.fetchone() + if row is None: + raise ResumeError('Error reading progress information.') + + return row[0] != 0 + + def StoreKeys(self, key_start, key_end): + """Record a new progress record, returning a key for later updates. + + The specified progress information will be persisted into the database. + A unique key will be returned that identifies this progress state. The + key is later used to (quickly) update this record. + + For the progress resumption to proceed properly, calls to StoreKeys + MUST specify monotonically increasing key ranges. This will result in + a database whereby the ID, KEY_START, and KEY_END rows are all + increasing (rather than having ranges out of order). + + NOTE: the above precondition is NOT tested by this method (since it + would imply an additional table read or two on each invocation). + + Args: + key_start: The starting key of the WorkItem (inclusive) + key_end: The end key of the WorkItem (inclusive) + + Returns: + A string to later be used as a unique key to update this state. + """ + self._OpenProgressConnection() + + assert _RunningInThread(self.progress_thread) + assert isinstance(key_start, int) + assert isinstance(key_end, int) + assert key_start <= key_end + + if self.prior_key_end is not None: + assert key_start > self.prior_key_end + self.prior_key_end = key_end + + self.insert_cursor.execute( + 'insert into progress (state, key_start, key_end) values (?, ?, ?)', + (STATE_READ, key_start, key_end)) + + progress_key = self.insert_cursor.lastrowid + + self._MaybeCommit() + + return progress_key + + def UpdateState(self, key, new_state): + """Update a specified progress record with new information. + + Args: + key: The key for this progress record, returned from StoreKeys + new_state: The new state to associate with this progress record. + """ + self._OpenProgressConnection() + + assert _RunningInThread(self.progress_thread) + assert isinstance(new_state, int) + + self.update_cursor.execute('update progress set state=? where id=?', + (new_state, key)) + + self._MaybeCommit() + + def GetProgressStatusGenerator(self): + """Get a generator which returns progress information. + + The returned generator will yield a series of 4-tuples that specify + progress information about a prior run of the uploader. The 4-tuples + have the following values: + + progress_key: The unique key to later update this record with new + progress information. + state: The last state saved for this progress record. + key_start: The starting key of the items for uploading (inclusive). + key_end: The ending key of the items for uploading (inclusive). + + After all incompletely-transferred records are provided, then one + more 4-tuple will be generated: + + None + DATA_CONSUMED_TO_HERE: A unique string value indicating this record + is being provided. + None + key_end: An integer value specifying the last data source key that + was handled by the previous run of the uploader. + + The caller should begin uploading records which occur after key_end. + + Yields: + Progress information as tuples (progress_key, state, key_start, key_end). + """ + conn = sqlite3.connect(self.db_filename, isolation_level=None) + cursor = conn.cursor() + + cursor.execute('select max(id) from progress') + batch_id = cursor.fetchone()[0] + + cursor.execute('select key_end from progress where id = ?', (batch_id,)) + key_end = cursor.fetchone()[0] + + self.prior_key_end = key_end + + cursor.execute( + 'select id, state, key_start, key_end from progress' + ' where state != ?' + ' order by id', + (STATE_SENT,)) + + rows = cursor.fetchall() + + for row in rows: + if row is None: + break + + yield row + + yield None, DATA_CONSUMED_TO_HERE, None, key_end + + +class StubProgressDatabase(object): + """A stub implementation of ProgressDatabase which does nothing.""" + + def HasUnfinishedWork(self): + """Whether the stub database has progress information (it doesn't).""" + return False + + def StoreKeys(self, unused_key_start, unused_key_end): + """Pretend to store a key in the stub database.""" + return 'fake-key' + + def UpdateState(self, unused_key, unused_new_state): + """Pretend to update the state of a progress item.""" + pass + + def ThreadComplete(self): + """Finalize operations on the stub database (i.e. do nothing).""" + pass + + +class ProgressTrackerThread(_ThreadBase): + """A thread which records progress information for the upload process. + + The progress information is stored into the provided progress database. + This class is not responsible for replaying a prior run's progress + information out of the database. Separate mechanisms must be used to + resume a prior upload attempt. + """ + + NAME = 'progress tracking thread' + + def __init__(self, progress_queue, progress_db): + """Initialize the ProgressTrackerThread instance. + + Args: + progress_queue: A Queue used for tracking progress information. + progress_db: The database for tracking progress information; should + be an instance of ProgressDatabase. + """ + _ThreadBase.__init__(self) + + self.progress_queue = progress_queue + self.db = progress_db + self.entities_sent = 0 + + def PerformWork(self): + """Performs the work of a ProgressTrackerThread.""" + while not self.exit_flag: + try: + item = self.progress_queue.get(block=True, timeout=1.0) + except Queue.Empty: + continue + if item == _THREAD_SHOULD_EXIT: + break + + if item.state == STATE_READ and item.progress_key is None: + item.progress_key = self.db.StoreKeys(item.key_start, item.key_end) + else: + assert item.progress_key is not None + + self.db.UpdateState(item.progress_key, item.state) + if item.state == STATE_SENT: + self.entities_sent += item.count + + item.progress_event.set() + + self.progress_queue.task_done() + + self.db.ThreadComplete() + + + +def Validate(value, typ): + """Checks that value is non-empty and of the right type. + + Args: + value: any value + typ: a type or tuple of types + + Raises: + ValueError if value is None or empty. + TypeError if it's not the given type. + + """ + if not value: + raise ValueError('Value should not be empty; received %s.' % value) + elif not isinstance(value, typ): + raise TypeError('Expected a %s, but received %s (a %s).' % + (typ, value, value.__class__)) + + +class Loader(object): + """A base class for creating datastore entities from input data. + + To add a handler for bulk loading a new entity kind into your datastore, + write a subclass of this class that calls Loader.__init__ from your + class's __init__. + + If you need to run extra code to convert entities from the input + data, create new properties, or otherwise modify the entities before + they're inserted, override HandleEntity. + + See the CreateEntity method for the creation of entities from the + (parsed) input data. + """ + + __loaders = {} + __kind = None + __properties = None + + def __init__(self, kind, properties): + """Constructor. + + Populates this Loader's kind and properties map. Also registers it with + the bulk loader, so that all you need to do is instantiate your Loader, + and the bulkload handler will automatically use it. + + Args: + kind: a string containing the entity kind that this loader handles + + properties: list of (name, converter) tuples. + + This is used to automatically convert the CSV columns into + properties. The converter should be a function that takes one + argument, a string value from the CSV file, and returns a + correctly typed property value that should be inserted. The + tuples in this list should match the columns in your CSV file, + in order. + + For example: + [('name', str), + ('id_number', int), + ('email', datastore_types.Email), + ('user', users.User), + ('birthdate', lambda x: datetime.datetime.fromtimestamp(float(x))), + ('description', datastore_types.Text), + ] + """ + Validate(kind, basestring) + self.__kind = kind + + db.class_for_kind(kind) + + Validate(properties, list) + for name, fn in properties: + Validate(name, basestring) + assert callable(fn), ( + 'Conversion function %s for property %s is not callable.' % (fn, name)) + + self.__properties = properties + + @staticmethod + def RegisterLoader(loader): + + Loader.__loaders[loader.__kind] = loader + + def kind(self): + """ Return the entity kind that this Loader handes. + """ + return self.__kind + + def CreateEntity(self, values, key_name=None): + """Creates a entity from a list of property values. + + Args: + values: list/tuple of str + key_name: if provided, the name for the (single) resulting entity + + Returns: + list of db.Model + + The returned entities are populated with the property values from the + argument, converted to native types using the properties map given in + the constructor, and passed through HandleEntity. They're ready to be + inserted. + + Raises: + AssertionError if the number of values doesn't match the number + of properties in the properties map. + ValueError if any element of values is None or empty. + TypeError if values is not a list or tuple. + """ + Validate(values, (list, tuple)) + assert len(values) == len(self.__properties), ( + 'Expected %d CSV columns, found %d.' % + (len(self.__properties), len(values))) + + model_class = db.class_for_kind(self.__kind) + + properties = {'key_name': key_name} + for (name, converter), val in zip(self.__properties, values): + if converter is bool and val.lower() in ('0', 'false', 'no'): + val = False + properties[name] = converter(val) + + entity = model_class(**properties) + entities = self.HandleEntity(entity) + + if entities: + if not isinstance(entities, (list, tuple)): + entities = [entities] + + for entity in entities: + if not isinstance(entity, db.Model): + raise TypeError('Expected a db.Model, received %s (a %s).' % + (entity, entity.__class__)) + + return entities + + def GenerateKey(self, i, values): + """Generates a key_name to be used in creating the underlying object. + + The default implementation returns None. + + This method can be overridden to control the key generation for + uploaded entities. The value returned should be None (to use a + server generated numeric key), or a string which neither starts + with a digit nor has the form __*__. (See + http://code.google.com/appengine/docs/python/datastore/keysandentitygroups.html) + + If you generate your own string keys, keep in mind: + + 1. The key name for each entity must be unique. + 2. If an entity of the same kind and key already exists in the + datastore, it will be overwritten. + + Args: + i: Number corresponding to this object (assume it's run in a loop, + this is your current count. + values: list/tuple of str. + + Returns: + A string to be used as the key_name for an entity. + """ + return None + + def HandleEntity(self, entity): + """Subclasses can override this to add custom entity conversion code. + + This is called for each entity, after its properties are populated from + CSV but before it is stored. Subclasses can override this to add custom + entity handling code. + + The entity to be inserted should be returned. If multiple entities should + be inserted, return a list of entities. If no entities should be inserted, + return None or []. + + Args: + entity: db.Model + + Returns: + db.Model or list of db.Model + """ + return entity + + + @staticmethod + def RegisteredLoaders(): + """Returns a list of the Loader instances that have been created. + """ + return dict(Loader.__loaders) + + +class QueueJoinThread(threading.Thread): + """A thread that joins a queue and exits. + + Queue joins do not have a timeout. To simulate a queue join with + timeout, run this thread and join it with a timeout. + """ + + def __init__(self, queue): + """Initialize a QueueJoinThread. + + Args: + queue: The queue for this thread to join. + """ + threading.Thread.__init__(self) + assert isinstance(queue, (Queue.Queue, ReQueue)) + self.queue = queue + + def run(self): + """Perform the queue join in this thread.""" + self.queue.join() + + +def InterruptibleQueueJoin(queue, + thread_local, + thread_gate, + queue_join_thread_factory=QueueJoinThread): + """Repeatedly joins the given ReQueue or Queue.Queue with short timeout. + + Between each timeout on the join, worker threads are checked. + + Args: + queue: A Queue.Queue or ReQueue instance. + thread_local: A threading.local instance which indicates interrupts. + thread_gate: A ThreadGate instance. + queue_join_thread_factory: Used for dependency injection. + + Returns: + True unless the queue join is interrupted by SIGINT or worker death. + """ + thread = queue_join_thread_factory(queue) + thread.start() + while True: + thread.join(timeout=.5) + if not thread.isAlive(): + return True + if thread_local.shut_down: + logging.debug('Queue join interrupted') + return False + for worker_thread in thread_gate.Threads(): + if not worker_thread.isAlive(): + return False + + +def ShutdownThreads(data_source_thread, work_queue, thread_gate): + """Shuts down the worker and data source threads. + + Args: + data_source_thread: A running DataSourceThread instance. + work_queue: The work queue. + thread_gate: A ThreadGate instance with workers registered. + """ + logging.info('An error occurred. Shutting down...') + + data_source_thread.exit_flag = True + + for thread in thread_gate.Threads(): + thread.exit_flag = True + + for unused_thread in thread_gate.Threads(): + thread_gate.EnableThread() + + data_source_thread.join(timeout=3.0) + if data_source_thread.isAlive(): + logging.warn('%s hung while trying to exit', + data_source_thread.GetFriendlyName()) + + while not work_queue.empty(): + try: + unused_item = work_queue.get_nowait() + work_queue.task_done() + except Queue.Empty: + pass + + +def PerformBulkUpload(app_id, + post_url, + kind, + workitem_generator_factory, + num_threads, + throttle, + progress_db, + max_queue_size=DEFAULT_QUEUE_SIZE, + request_manager_factory=RequestManager, + bulkloaderthread_factory=BulkLoaderThread, + progresstrackerthread_factory=ProgressTrackerThread, + datasourcethread_factory=DataSourceThread, + work_queue_factory=ReQueue, + progress_queue_factory=Queue.Queue): + """Uploads data into an application using a series of HTTP POSTs. + + This function will spin up a number of threads to read entities from + the data source, pass those to a number of worker ("uploader") threads + for sending to the application, and track all of the progress in a + small database in case an error or pause/termination requires a + restart/resumption of the upload process. + + Args: + app_id: String containing application id. + post_url: URL to post the Entity data to. + kind: Kind of the Entity records being posted. + workitem_generator_factory: A factory that creates a WorkItem generator. + num_threads: How many uploader threads should be created. + throttle: A Throttle instance. + progress_db: The database to use for replaying/recording progress. + max_queue_size: Maximum size of the queues before they should block. + request_manager_factory: Used for dependency injection. + bulkloaderthread_factory: Used for dependency injection. + progresstrackerthread_factory: Used for dependency injection. + datasourcethread_factory: Used for dependency injection. + work_queue_factory: Used for dependency injection. + progress_queue_factory: Used for dependency injection. + + Raises: + AuthenticationError: If authentication is required and fails. + """ + thread_gate = ThreadGate(True) + + (unused_scheme, + host_port, url_path, + unused_query, unused_fragment) = urlparse.urlsplit(post_url) + + work_queue = work_queue_factory(max_queue_size) + progress_queue = progress_queue_factory(max_queue_size) + request_manager = request_manager_factory(app_id, + host_port, + url_path, + kind, + throttle) + + throttle.Register(threading.currentThread()) + try: + request_manager.Authenticate() + except Exception, e: + logging.exception(e) + raise AuthenticationError('Authentication failed') + if (request_manager.credentials is not None and + not request_manager.authenticated): + raise AuthenticationError('Authentication failed') + + for unused_idx in range(num_threads): + thread = bulkloaderthread_factory(work_queue, + throttle, + thread_gate, + request_manager) + throttle.Register(thread) + thread_gate.Register(thread) + + progress_thread = progresstrackerthread_factory(progress_queue, progress_db) + + if progress_db.HasUnfinishedWork(): + logging.debug('Restarting upload using progress database') + progress_generator_factory = progress_db.GetProgressStatusGenerator + else: + progress_generator_factory = None + + data_source_thread = datasourcethread_factory(work_queue, + progress_queue, + workitem_generator_factory, + progress_generator_factory) + + thread_local = threading.local() + thread_local.shut_down = False + + def Interrupt(unused_signum, unused_frame): + """Shutdown gracefully in response to a signal.""" + thread_local.shut_down = True + + signal.signal(signal.SIGINT, Interrupt) + + progress_thread.start() + data_source_thread.start() + for thread in thread_gate.Threads(): + thread.start() + + + while not thread_local.shut_down: + data_source_thread.join(timeout=0.25) + + if data_source_thread.isAlive(): + for thread in list(thread_gate.Threads()) + [progress_thread]: + if not thread.isAlive(): + logging.info('Unexpected thread death: %s', thread.getName()) + thread_local.shut_down = True + break + else: + break + + if thread_local.shut_down: + ShutdownThreads(data_source_thread, work_queue, thread_gate) + + def _Join(ob, msg): + logging.debug('Waiting for %s...', msg) + if isinstance(ob, threading.Thread): + ob.join(timeout=3.0) + if ob.isAlive(): + logging.debug('Joining %s failed', ob.GetFriendlyName()) + else: + logging.debug('... done.') + elif isinstance(ob, (Queue.Queue, ReQueue)): + if not InterruptibleQueueJoin(ob, thread_local, thread_gate): + ShutdownThreads(data_source_thread, work_queue, thread_gate) + else: + ob.join() + logging.debug('... done.') + + _Join(work_queue, 'work_queue to flush') + + for unused_thread in thread_gate.Threads(): + work_queue.put(_THREAD_SHOULD_EXIT) + + for unused_thread in thread_gate.Threads(): + thread_gate.EnableThread() + + for thread in thread_gate.Threads(): + _Join(thread, 'thread [%s] to terminate' % thread.getName()) + + thread.CheckError() + + if progress_thread.isAlive(): + _Join(progress_queue, 'progress_queue to finish') + else: + logging.warn('Progress thread exited prematurely') + + progress_queue.put(_THREAD_SHOULD_EXIT) + _Join(progress_thread, 'progress_thread to terminate') + progress_thread.CheckError() + + data_source_thread.CheckError() + + total_up, duration = throttle.TotalTransferred(BANDWIDTH_UP) + s_total_up, unused_duration = throttle.TotalTransferred(HTTPS_BANDWIDTH_UP) + total_up += s_total_up + logging.info('%d entites read, %d previously transferred', + data_source_thread.read_count, + data_source_thread.sent_count) + logging.info('%d entities (%d bytes) transferred in %.1f seconds', + progress_thread.entities_sent, total_up, duration) + if (data_source_thread.read_all and + progress_thread.entities_sent + data_source_thread.sent_count >= + data_source_thread.read_count): + logging.info('All entities successfully uploaded') + else: + logging.info('Some entities not successfully uploaded') + + +def PrintUsageExit(code): + """Prints usage information and exits with a status code. + + Args: + code: Status code to pass to sys.exit() after displaying usage information. + """ + print __doc__ % {'arg0': sys.argv[0]} + sys.stdout.flush() + sys.stderr.flush() + sys.exit(code) + + +def ParseArguments(argv): + """Parses command-line arguments. + + Prints out a help message if -h or --help is supplied. + + Args: + argv: List of command-line arguments. + + Returns: + Tuple (url, filename, cookie, batch_size, kind) containing the values from + each corresponding command-line flag. + """ + opts, unused_args = getopt.getopt( + argv[1:], + 'h', + ['debug', + 'help', + 'url=', + 'filename=', + 'batch_size=', + 'kind=', + 'num_threads=', + 'bandwidth_limit=', + 'rps_limit=', + 'http_limit=', + 'db_filename=', + 'app_id=', + 'config_file=', + 'auth_domain=', + ]) + + url = None + filename = None + batch_size = DEFAULT_BATCH_SIZE + kind = None + num_threads = DEFAULT_THREAD_COUNT + bandwidth_limit = DEFAULT_BANDWIDTH_LIMIT + rps_limit = DEFAULT_RPS_LIMIT + http_limit = DEFAULT_REQUEST_LIMIT + db_filename = None + app_id = None + config_file = None + auth_domain = 'gmail.com' + + for option, value in opts: + if option == '--debug': + logging.getLogger().setLevel(logging.DEBUG) + elif option in ('-h', '--help'): + PrintUsageExit(0) + elif option == '--url': + url = value + elif option == '--filename': + filename = value + elif option == '--batch_size': + batch_size = int(value) + elif option == '--kind': + kind = value + elif option == '--num_threads': + num_threads = int(value) + elif option == '--bandwidth_limit': + bandwidth_limit = int(value) + elif option == '--rps_limit': + rps_limit = int(value) + elif option == '--http_limit': + http_limit = int(value) + elif option == '--db_filename': + db_filename = value + elif option == '--app_id': + app_id = value + elif option == '--config_file': + config_file = value + elif option == '--auth_domain': + auth_domain = value + + return ProcessArguments(app_id=app_id, + url=url, + filename=filename, + batch_size=batch_size, + kind=kind, + num_threads=num_threads, + bandwidth_limit=bandwidth_limit, + rps_limit=rps_limit, + http_limit=http_limit, + db_filename=db_filename, + config_file=config_file, + auth_domain=auth_domain, + die_fn=lambda: PrintUsageExit(1)) + + +def ThrottleLayout(bandwidth_limit, http_limit, rps_limit): + return { + BANDWIDTH_UP: bandwidth_limit, + BANDWIDTH_DOWN: bandwidth_limit, + REQUESTS: http_limit, + HTTPS_BANDWIDTH_UP: bandwidth_limit / 5, + HTTPS_BANDWIDTH_DOWN: bandwidth_limit / 5, + HTTPS_REQUESTS: http_limit / 5, + RECORDS: rps_limit, + } + + +def LoadConfig(config_file): + """Loads a config file and registers any Loader classes present.""" + if config_file: + global_dict = dict(globals()) + execfile(config_file, global_dict) + for cls in Loader.__subclasses__(): + Loader.RegisterLoader(cls()) + + +def _MissingArgument(arg_name, die_fn): + """Print error message about missing argument and die.""" + print >>sys.stderr, '%s argument required' % arg_name + die_fn() + + +def ProcessArguments(app_id=None, + url=None, + filename=None, + batch_size=DEFAULT_BATCH_SIZE, + kind=None, + num_threads=DEFAULT_THREAD_COUNT, + bandwidth_limit=DEFAULT_BANDWIDTH_LIMIT, + rps_limit=DEFAULT_RPS_LIMIT, + http_limit=DEFAULT_REQUEST_LIMIT, + db_filename=None, + config_file=None, + auth_domain='gmail.com', + die_fn=lambda: sys.exit(1)): + """Processes non command-line input arguments.""" + if db_filename is None: + db_filename = time.strftime('bulkloader-progress-%Y%m%d.%H%M%S.sql3') + + if batch_size <= 0: + print >>sys.stderr, 'batch_size must be 1 or larger' + die_fn() + + if url is None: + _MissingArgument('url', die_fn) + + if filename is None: + _MissingArgument('filename', die_fn) + + if kind is None: + _MissingArgument('kind', die_fn) + + if config_file is None: + _MissingArgument('config_file', die_fn) + + if app_id is None: + (unused_scheme, host_port, unused_url_path, + unused_query, unused_fragment) = urlparse.urlsplit(url) + suffix_idx = host_port.find('.appspot.com') + if suffix_idx > -1: + app_id = host_port[:suffix_idx] + elif host_port.split(':')[0].endswith('google.com'): + app_id = host_port.split('.')[0] + else: + print >>sys.stderr, 'app_id required for non appspot.com domains' + die_fn() + + return (app_id, url, filename, batch_size, kind, num_threads, + bandwidth_limit, rps_limit, http_limit, db_filename, config_file, + auth_domain) + + +def _PerformBulkload(app_id=None, + url=None, + filename=None, + batch_size=DEFAULT_BATCH_SIZE, + kind=None, + num_threads=DEFAULT_THREAD_COUNT, + bandwidth_limit=DEFAULT_BANDWIDTH_LIMIT, + rps_limit=DEFAULT_RPS_LIMIT, + http_limit=DEFAULT_REQUEST_LIMIT, + db_filename=None, + config_file=None, + auth_domain='gmail.com'): + """Runs the bulkloader, given the options as keyword arguments. + + Args: + app_id: The application id. + url: The url of the remote_api endpoint. + filename: The name of the file containing the CSV data. + batch_size: The number of records to send per request. + kind: The kind of entity to transfer. + num_threads: The number of threads to use to transfer data. + bandwidth_limit: Maximum bytes/second to transfers. + rps_limit: Maximum records/second to transfer. + http_limit: Maximum requests/second for transfers. + db_filename: The name of the SQLite3 progress database file. + config_file: The name of the configuration file. + auth_domain: The auth domain to use for logins and UserProperty. + + Returns: + An exit code. + """ + os.environ['AUTH_DOMAIN'] = auth_domain + LoadConfig(config_file) + + throttle_layout = ThrottleLayout(bandwidth_limit, http_limit, rps_limit) + + throttle = Throttle(layout=throttle_layout) + + + workitem_generator_factory = GetCSVGeneratorFactory(filename, batch_size) + + if db_filename == 'skip': + progress_db = StubProgressDatabase() + else: + progress_db = ProgressDatabase(db_filename) + + + max_queue_size = max(DEFAULT_QUEUE_SIZE, 2 * num_threads + 5) + + PerformBulkUpload(app_id, + url, + kind, + workitem_generator_factory, + num_threads, + throttle, + progress_db, + max_queue_size=max_queue_size) + + return 0 + + +def Run(app_id=None, + url=None, + filename=None, + batch_size=DEFAULT_BATCH_SIZE, + kind=None, + num_threads=DEFAULT_THREAD_COUNT, + bandwidth_limit=DEFAULT_BANDWIDTH_LIMIT, + rps_limit=DEFAULT_RPS_LIMIT, + http_limit=DEFAULT_REQUEST_LIMIT, + db_filename=None, + auth_domain='gmail.com', + config_file=None): + """Sets up and runs the bulkloader, given the options as keyword arguments. + + Args: + app_id: The application id. + url: The url of the remote_api endpoint. + filename: The name of the file containing the CSV data. + batch_size: The number of records to send per request. + kind: The kind of entity to transfer. + num_threads: The number of threads to use to transfer data. + bandwidth_limit: Maximum bytes/second to transfers. + rps_limit: Maximum records/second to transfer. + http_limit: Maximum requests/second for transfers. + db_filename: The name of the SQLite3 progress database file. + config_file: The name of the configuration file. + auth_domain: The auth domain to use for logins and UserProperty. + + Returns: + An exit code. + """ + logging.basicConfig( + format='%(levelname)-8s %(asctime)s %(filename)s] %(message)s') + args = ProcessArguments(app_id=app_id, + url=url, + filename=filename, + batch_size=batch_size, + kind=kind, + num_threads=num_threads, + bandwidth_limit=bandwidth_limit, + rps_limit=rps_limit, + http_limit=http_limit, + db_filename=db_filename, + config_file=config_file) + + (app_id, url, filename, batch_size, kind, num_threads, bandwidth_limit, + rps_limit, http_limit, db_filename, config_file, auth_domain) = args + + return _PerformBulkload(app_id=app_id, + url=url, + filename=filename, + batch_size=batch_size, + kind=kind, + num_threads=num_threads, + bandwidth_limit=bandwidth_limit, + rps_limit=rps_limit, + http_limit=http_limit, + db_filename=db_filename, + config_file=config_file, + auth_domain=auth_domain) + + +def main(argv): + """Runs the importer from the command line.""" + logging.basicConfig( + level=logging.INFO, + format='%(levelname)-8s %(asctime)s %(filename)s] %(message)s') + + args = ParseArguments(argv) + if None in args: + print >>sys.stderr, 'Invalid arguments' + PrintUsageExit(1) + + (app_id, url, filename, batch_size, kind, num_threads, + bandwidth_limit, rps_limit, http_limit, db_filename, config_file, + auth_domain) = args + + return _PerformBulkload(app_id=app_id, + url=url, + filename=filename, + batch_size=batch_size, + kind=kind, + num_threads=num_threads, + bandwidth_limit=bandwidth_limit, + rps_limit=rps_limit, + http_limit=http_limit, + db_filename=db_filename, + config_file=config_file, + auth_domain=auth_domain) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/tools/dev_appserver.py --- a/thirdparty/google_appengine/google/appengine/tools/dev_appserver.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/tools/dev_appserver.py Thu Feb 12 12:30:36 2009 +0000 @@ -83,6 +83,8 @@ from google.appengine.api.capabilities import capability_stub from google.appengine.api.memcache import memcache_stub +from google.appengine import dist + from google.appengine.tools import dev_appserver_index from google.appengine.tools import dev_appserver_login @@ -113,10 +115,12 @@ ('.wbmp', 'image/vnd.wap.wbmp')): mimetypes.add_type(mime_type, ext) -MAX_RUNTIME_RESPONSE_SIZE = 1 << 20 +MAX_RUNTIME_RESPONSE_SIZE = 10 << 20 MAX_REQUEST_SIZE = 10 * 1024 * 1024 +API_VERSION = '1' + class Error(Exception): """Base-class for exceptions in this module.""" @@ -193,9 +197,31 @@ outfile: File-like object where output data should be written. base_env_dict: Dictionary of CGI environment parameters if available. Defaults to None. + + Returns: + None if request handling is complete. + Tuple (path, headers, input_file) for an internal redirect: + path: Path of URL to redirect to. + headers: Headers to send to other dispatcher. + input_file: New input to send to new dispatcher. """ raise NotImplementedError + def EndRedirect(self, dispatched_output, original_output): + """Process the end of an internal redirect. + + This method is called after all subsequent dispatch requests have finished. + By default the output from the dispatched process is copied to the original. + + This will not be called on dispatchers that do not return an internal + redirect. + + Args: + dispatched_output: StringIO buffer containing the results from the + dispatched + """ + original_output.write(dispatched_output.read()) + class URLMatcher(object): """Matches an arbitrary URL using a list of URL patterns from an application. @@ -346,12 +372,25 @@ 'authorized to view this page.' % (httplib.FORBIDDEN, email)) else: - dispatcher.Dispatch(relative_url, - matched_path, - headers, - infile, - outfile, - base_env_dict=base_env_dict) + forward = dispatcher.Dispatch(relative_url, + matched_path, + headers, + infile, + outfile, + base_env_dict=base_env_dict) + + if forward: + new_path, new_headers, new_input = forward + logging.info('Internal redirection to %s' % new_path) + new_outfile = cStringIO.StringIO() + self.Dispatch(new_path, + None, + new_headers, + new_input, + new_outfile, + dict(base_env_dict)) + new_outfile.seek(0) + dispatcher.EndRedirect(new_outfile, outfile) return @@ -514,11 +553,6 @@ return env -def FakeTemporaryFile(*args, **kwargs): - """Fake for tempfile.TemporaryFile that just uses StringIO.""" - return cStringIO.StringIO() - - def NotImplementedFake(*args, **kwargs): """Fake for methods/functions that are not implemented in the production environment. @@ -577,6 +611,27 @@ return ('Linux', '', '', '', '') +def FakeUnlink(path): + """Fake version of os.unlink.""" + if os.path.isdir(path): + raise OSError(2, "Is a directory", path) + else: + raise OSError(1, "Operation not permitted", path) + + +def FakeReadlink(path): + """Fake version of os.readlink.""" + raise OSError(22, "Invalid argument", path) + + +def FakeAccess(path, mode): + """Fake version of os.access where only reads are supported.""" + if not os.path.exists(path) or mode != os.R_OK: + return False + else: + return True + + def FakeSetLocale(category, value=None, original_setlocale=locale.setlocale): """Fake version of locale.setlocale that only supports the default.""" if value not in (None, '', 'C', 'POSIX'): @@ -715,30 +770,74 @@ ]) + _original_file = file + + _root_path = None _application_paths = None - _original_file = file + _skip_files = None + _static_file_config_matcher = None + + _availability_cache = {} @staticmethod - def SetAllowedPaths(application_paths): - """Sets the root path of the application that is currently running. + def SetAllowedPaths(root_path, application_paths): + """Configures which paths are allowed to be accessed. Must be called at least once before any file objects are created in the hardened environment. Args: - root_path: Path to the root of the application. + root_path: Absolute path to the root of the application. + application_paths: List of additional paths that the application may + access, this must include the App Engine runtime but + not the Python library directories. """ FakeFile._application_paths = (set(os.path.realpath(path) for path in application_paths) | set(os.path.abspath(path) for path in application_paths)) + FakeFile._application_paths.add(root_path) + + FakeFile._root_path = os.path.join(root_path, '') + + FakeFile._availability_cache = {} + + @staticmethod + def SetSkippedFiles(skip_files): + """Sets which files in the application directory are to be ignored. + + Must be called at least once before any file objects are created in the + hardened environment. + + Must be called whenever the configuration was updated. + + Args: + skip_files: Object with .match() method (e.g. compiled regexp). + """ + FakeFile._skip_files = skip_files + FakeFile._availability_cache = {} + + @staticmethod + def SetStaticFileConfigMatcher(static_file_config_matcher): + """Sets StaticFileConfigMatcher instance for checking if a file is static. + + Must be called at least once before any file objects are created in the + hardened environment. + + Must be called whenever the configuration was updated. + + Args: + static_file_config_matcher: StaticFileConfigMatcher instance. + """ + FakeFile._static_file_config_matcher = static_file_config_matcher + FakeFile._availability_cache = {} @staticmethod def IsFileAccessible(filename, normcase=os.path.normcase): """Determines if a file's path is accessible. - SetAllowedPaths() must be called before this method or else all file - accesses will raise an error. + SetAllowedPaths(), SetSkippedFiles() and SetStaticFileConfigMatcher() must + be called before this method or else all file accesses will raise an error. Args: filename: Path of the file to check (relative or absolute). May be a @@ -754,6 +853,40 @@ if os.path.isdir(logical_filename): logical_filename = os.path.join(logical_filename, 'foo') + result = FakeFile._availability_cache.get(logical_filename) + if result is None: + result = FakeFile._IsFileAccessibleNoCache(logical_filename, + normcase=normcase) + FakeFile._availability_cache[logical_filename] = result + return result + + @staticmethod + def _IsFileAccessibleNoCache(logical_filename, normcase=os.path.normcase): + """Determines if a file's path is accessible. + + This is an internal part of the IsFileAccessible implementation. + + Args: + logical_filename: Absolute path of the file to check. + normcase: Used for dependency injection. + + Returns: + True if the file is accessible, False otherwise. + """ + if IsPathInSubdirectories(logical_filename, [FakeFile._root_path], + normcase=normcase): + relative_filename = logical_filename[len(FakeFile._root_path):] + + if FakeFile._skip_files.match(relative_filename): + logging.warning('Blocking access to skipped file "%s"', + logical_filename) + return False + + if FakeFile._static_file_config_matcher.IsStaticFile(relative_filename): + logging.warning('Blocking access to static file "%s"', + logical_filename) + return False + if logical_filename in FakeFile.ALLOWED_FILES: return True @@ -887,8 +1020,6 @@ indent = self._indent_level * ' ' print >>sys.stderr, indent + (message % args) - EMPTY_MODULE_FILE = '' - _WHITE_LIST_C_MODULES = [ 'array', 'binascii', @@ -959,6 +1090,7 @@ 'os': [ + 'access', 'altsep', 'curdir', 'defpath', @@ -1007,6 +1139,8 @@ 'path', 'pathsep', 'R_OK', + 'readlink', + 'remove', 'SEEK_CUR', 'SEEK_END', 'SEEK_SET', @@ -1016,6 +1150,7 @@ 'stat_result', 'strerror', 'TMP_MAX', + 'unlink', 'urandom', 'walk', 'WCOREDUMP', @@ -1032,46 +1167,23 @@ ], } - _EMPTY_MODULES = [ - 'imp', - 'ftplib', - 'select', - 'socket', - 'tempfile', - ] - _MODULE_OVERRIDES = { 'locale': { 'setlocale': FakeSetLocale, }, 'os': { + 'access': FakeAccess, 'listdir': RestrictedPathFunction(os.listdir), 'lstat': RestrictedPathFunction(os.stat), + 'readlink': FakeReadlink, + 'remove': FakeUnlink, 'stat': RestrictedPathFunction(os.stat), 'uname': FakeUname, + 'unlink': FakeUnlink, 'urandom': FakeURandom, }, - - 'socket': { - 'AF_INET': None, - 'SOCK_STREAM': None, - 'SOCK_DGRAM': None, - '_GLOBAL_DEFAULT_TIMEOUT': getattr(socket, '_GLOBAL_DEFAULT_TIMEOUT', - None), - }, - - 'tempfile': { - 'TemporaryFile': FakeTemporaryFile, - 'gettempdir': NotImplementedFake, - 'gettempprefix': NotImplementedFake, - 'mkdtemp': NotImplementedFake, - 'mkstemp': NotImplementedFake, - 'mktemp': NotImplementedFake, - 'NamedTemporaryFile': NotImplementedFake, - 'tempdir': NotImplementedFake, - }, } _ENABLED_FILE_TYPES = ( @@ -1107,8 +1219,7 @@ @Trace def find_module(self, fullname, path=None): """See PEP 302.""" - if (fullname in ('cPickle', 'thread') or - fullname in HardenedModulesHook._EMPTY_MODULES): + if fullname in ('cPickle', 'thread'): return self search_path = path @@ -1116,7 +1227,8 @@ try: for index, current_module in enumerate(all_modules): current_module_fullname = '.'.join(all_modules[:index + 1]) - if current_module_fullname == fullname: + if (current_module_fullname == fullname and not + self.StubModuleExists(fullname)): self.FindModuleRestricted(current_module, current_module_fullname, search_path) @@ -1135,6 +1247,21 @@ return self + def StubModuleExists(self, name): + """Check if the named module has a stub replacement.""" + if name in sys.builtin_module_names: + name = 'py_%s' % name + if name in dist.__all__: + return True + return False + + def ImportStubModule(self, name): + """Import the stub module replacement for the specified module.""" + if name in sys.builtin_module_names: + name = 'py_%s' % name + module = __import__(dist.__name__, {}, {}, [name]) + return getattr(module, name) + @Trace def FixModule(self, module): """Prunes and overrides restricted module attributes. @@ -1334,9 +1461,7 @@ """ module = self._imp.new_module(submodule_fullname) - if submodule_fullname in self._EMPTY_MODULES: - module.__file__ = self.EMPTY_MODULE_FILE - elif submodule_fullname == 'thread': + if submodule_fullname == 'thread': module.__dict__.update(self._dummy_thread.__dict__) module.__name__ = 'thread' elif submodule_fullname == 'cPickle': @@ -1345,6 +1470,8 @@ elif submodule_fullname == 'os': module.__dict__.update(self._os.__dict__) self._module_dict['os.path'] = module.path + elif self.StubModuleExists(submodule_fullname): + module = self.ImportStubModule(submodule_fullname) else: source_file, pathname, description = self.FindModuleRestricted(submodule, submodule_fullname, search_path) module = self.LoadModuleRestricted(submodule_fullname, @@ -2004,7 +2131,7 @@ path = entry.static_dir if path[-1] == '/': path = path[:-1] - regex = re.escape(path) + r'/(.*)' + regex = re.escape(path + os.path.sep) + r'(.*)' try: path_re = re.compile(regex) @@ -2021,6 +2148,20 @@ self._patterns.append((path_re, entry.mime_type, expiration)) + def IsStaticFile(self, path): + """Tests if the given path points to a "static" file. + + Args: + path: String containing the file's path relative to the app. + + Returns: + Boolean, True if the file was configured to be static. + """ + for (path_re, _, _) in self._patterns: + if path_re.match(path): + return True + return False + def GetMimeType(self, path): """Returns the mime type that we should use when serving the specified file. @@ -2143,8 +2284,25 @@ ]) -def RewriteResponse(response_file): - """Interprets server-side headers and adjusts the HTTP response accordingly. +def IgnoreHeadersRewriter(status_code, status_message, headers, body): + """Ignore specific response headers. + + Certain response headers cannot be modified by an Application. For a + complete list of these headers please see: + + http://code.google.com/appengine/docs/webapp/responseclass.html#Disallowed_HTTP_Response_Headers + + This rewriter simply removes those headers. + """ + for h in _IGNORE_RESPONSE_HEADERS: + if h in headers: + del headers[h] + + return status_code, status_message, headers, body + + +def ParseStatusRewriter(status_code, status_message, headers, body): + """Parse status header, if it exists. Handles the server-side 'status' header, which instructs the server to change the HTTP response code accordingly. Handles the 'location' header, which @@ -2154,12 +2312,113 @@ If the 'status' header supplied by the client is invalid, this method will set the response to a 500 with an error message as content. + """ + location_value = headers.getheader('location') + status_value = headers.getheader('status') + if status_value: + response_status = status_value + del headers['status'] + elif location_value: + response_status = '%d Redirecting' % httplib.FOUND + else: + return status_code, status_message, headers, body + + status_parts = response_status.split(' ', 1) + status_code, status_message = (status_parts + [''])[:2] + try: + status_code = int(status_code) + except ValueError: + status_code = 500 + body = cStringIO.StringIO('Error: Invalid "status" header value returned.') + + return status_code, status_message, headers, body + + +def CacheRewriter(status_code, status_message, headers, body): + """Update the cache header.""" + if not 'Cache-Control' in headers: + headers['Cache-Control'] = 'no-cache' + return status_code, status_message, headers, body + + +def ContentLengthRewriter(status_code, status_message, headers, body): + """Rewrite the Content-Length header. + + Even though Content-Length is not a user modifiable header, App Engine + sends a correct Content-Length to the user based on the actual response. + """ + current_position = body.tell() + body.seek(0, 2) + + headers['Content-Length'] = str(body.tell() - current_position) + body.seek(current_position) + return status_code, status_message, headers, body + + +def CreateResponseRewritersChain(): + """Create the default response rewriter chain. + + A response rewriter is the a function that gets a final chance to change part + of the dev_appservers response. A rewriter is not like a dispatcher in that + it is called after every request has been handled by the dispatchers + regardless of which dispatcher was used. + + The order in which rewriters are registered will be the order in which they + are used to rewrite the response. Modifications from earlier rewriters + are used as input to later rewriters. + + A response rewriter is a function that can rewrite the request in any way. + Thefunction can returned modified values or the original values it was + passed. + + A rewriter function has the following parameters and return values: + + Args: + status_code: Status code of response from dev_appserver or previous + rewriter. + status_message: Text corresponding to status code. + headers: mimetools.Message instance with parsed headers. NOTE: These + headers can contain its own 'status' field, but the default + dev_appserver implementation will remove this. Future rewriters + should avoid re-introducing the status field and return new codes + instead. + body: File object containing the body of the response. This position of + this file may not be at the start of the file. Any content before the + files position is considered not to be part of the final body. + + Returns: + status_code: Rewritten status code or original. + status_message: Rewritter message or original. + headers: Rewritten/modified headers or original. + body: Rewritten/modified body or original. + + Returns: + List of response rewriters. + """ + return [IgnoreHeadersRewriter, + ParseStatusRewriter, + CacheRewriter, + ContentLengthRewriter, + ] + + +def RewriteResponse(response_file, response_rewriters=None): + """Allows final rewrite of dev_appserver response. + + This function receives the unparsed HTTP response from the application + or internal handler, parses out the basic structure and feeds that structure + in to a chain of response rewriters. + + It also makes sure the final HTTP headers are properly terminated. + + For more about response rewriters, please see documentation for + CreateResponeRewritersChain. Args: response_file: File-like object containing the full HTTP response including the response code, all headers, and the request body. - gmtime: Function which returns current time in a format matching standard - time.gmtime(). + response_rewriters: A list of response rewriters. If none is provided it + will create a new chain using CreateResponseRewritersChain. Returns: Tuple (status_code, status_message, header, body) where: @@ -2170,36 +2429,19 @@ a trailing new-line (CRLF). body: String containing the body of the response. """ + if response_rewriters is None: + response_rewriters = CreateResponseRewritersChain() + + status_code = 200 + status_message = 'Good to go' headers = mimetools.Message(response_file) - for h in _IGNORE_RESPONSE_HEADERS: - if h in headers: - del headers[h] - - response_status = '%d Good to go' % httplib.OK - - location_value = headers.getheader('location') - status_value = headers.getheader('status') - if status_value: - response_status = status_value - del headers['status'] - elif location_value: - response_status = '%d Redirecting' % httplib.FOUND - - if not 'Cache-Control' in headers: - headers['Cache-Control'] = 'no-cache' - - status_parts = response_status.split(' ', 1) - status_code, status_message = (status_parts + [''])[:2] - try: - status_code = int(status_code) - except ValueError: - status_code = 500 - body = 'Error: Invalid "status" header value returned.' - else: - body = response_file.read() - - headers['Content-Length'] = str(len(body)) + for response_rewriter in response_rewriters: + status_code, status_message, headers, response_file = response_rewriter( + status_code, + status_message, + headers, + response_file) header_list = [] for header in headers.headers: @@ -2208,7 +2450,7 @@ header_list.append(header) header_data = '\r\n'.join(header_list) + '\r\n' - return status_code, status_message, header_data, body + return status_code, status_message, header_data, response_file.read() class ModuleManager(object): @@ -2245,7 +2487,7 @@ __file__ attribute, None will be returned. """ module_file = getattr(module, '__file__', None) - if not module_file or module_file == HardenedModulesHook.EMPTY_MODULE_FILE: + if module_file is None: return None source_file = module_file[:module_file.rfind('py') + 2] @@ -2309,7 +2551,9 @@ template_module.template_cache.clear() -def CreateRequestHandler(root_path, login_url, require_indexes=False, +def CreateRequestHandler(root_path, + login_url, + require_indexes=False, static_caching=True): """Creates a new BaseHTTPRequestHandler sub-class for use with the Python BaseHTTPServer module's HTTP server. @@ -2359,6 +2603,8 @@ config_cache = application_config_cache + rewriter_chain = CreateResponseRewritersChain() + def __init__(self, *args, **kwargs): """Initializer. @@ -2432,6 +2678,10 @@ config, explicit_matcher = LoadAppConfig(root_path, self.module_dict, cache=self.config_cache, static_caching=static_caching) + if config.api_version != API_VERSION: + logging.error("API versions cannot be switched dynamically: %r != %r" + % (config.api_version, API_VERSION)) + sys.exit(1) env_dict['CURRENT_VERSION_ID'] = config.version + ".1" env_dict['APPLICATION_ID'] = config.application dispatcher = MatcherDispatcher(login_url, @@ -2465,7 +2715,7 @@ outfile.flush() outfile.seek(0) - status_code, status_message, header_data, body = RewriteResponse(outfile) + status_code, status_message, header_data, body = RewriteResponse(outfile, self.rewriter_chain) runtime_response_size = len(outfile.getvalue()) if runtime_response_size > MAX_RUNTIME_RESPONSE_SIZE: @@ -2582,8 +2832,13 @@ url_matcher = create_url_matcher() path_adjuster = create_path_adjuster(root_path) cgi_dispatcher = create_cgi_dispatcher(module_dict, root_path, path_adjuster) + static_file_config_matcher = StaticFileConfigMatcher(url_map_list, + path_adjuster, + default_expiration) file_dispatcher = create_file_dispatcher(path_adjuster, - StaticFileConfigMatcher(url_map_list, path_adjuster, default_expiration)) + static_file_config_matcher) + + FakeFile.SetStaticFileConfigMatcher(static_file_config_matcher) for url_map in url_map_list: admin_only = url_map.login == appinfo.LOGIN_ADMIN @@ -2687,6 +2942,8 @@ module_dict, default_expiration) + FakeFile.SetSkippedFiles(config.skip_files) + if cache is not None: cache.path = appinfo_path cache.config = config @@ -2868,9 +3125,15 @@ serve_address='', require_indexes=False, static_caching=True, - python_path_list=sys.path): + python_path_list=sys.path, + sdk_dir=os.path.dirname(os.path.dirname(google.__file__))): """Creates an new HTTPServer for an application. + The sdk_dir argument must be specified for the directory storing all code for + the SDK so as to allow for the sandboxing of module access to work for any + and all SDK code. While typically this is where the 'google' package lives, + it can be in another location because of API version support. + Args: root_path: String containing the path to the root directory of the application where the app.yaml file is. @@ -2882,6 +3145,7 @@ require_indexes: True if index.yaml is read-only gospel; default False. static_caching: True if browser caching of static files should be allowed. python_path_list: Used for dependency injection. + sdk_dir: Directory where the SDK is stored. Returns: Instance of BaseHTTPServer.HTTPServer that's ready to start accepting. @@ -2889,12 +3153,14 @@ absolute_root_path = os.path.realpath(root_path) SetupTemplates(template_dir) - FakeFile.SetAllowedPaths([absolute_root_path, - os.path.dirname(os.path.dirname(google.__file__)), + FakeFile.SetAllowedPaths(absolute_root_path, + [sdk_dir, template_dir]) - handler_class = CreateRequestHandler(absolute_root_path, login_url, - require_indexes, static_caching) + handler_class = CreateRequestHandler(absolute_root_path, + login_url, + require_indexes, + static_caching) if absolute_root_path not in python_path_list: python_path_list.insert(0, absolute_root_path) diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/google/appengine/tools/dev_appserver_main.py --- a/thirdparty/google_appengine/google/appengine/tools/dev_appserver_main.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/google/appengine/tools/dev_appserver_main.py Thu Feb 12 12:30:36 2009 +0000 @@ -65,14 +65,24 @@ import getopt import logging import os +import re import sys import traceback import tempfile -from google.appengine.api import yaml_errors -from google.appengine.tools import appcfg -from google.appengine.tools import appengine_rpc -from google.appengine.tools import dev_appserver + +def SetGlobals(): + """Set various global variables involving the 'google' package. + + This function should not be called until sys.path has been properly set. + """ + global yaml_errors, appcfg, appengine_rpc, dev_appserver, os_compat + from google.appengine.api import yaml_errors + from google.appengine.tools import appcfg + from google.appengine.tools import appengine_rpc + from google.appengine.tools import dev_appserver + from google.appengine.tools import os_compat + DEFAULT_ADMIN_CONSOLE_SERVER = 'appengine.google.com' @@ -98,9 +108,13 @@ ARG_STATIC_CACHING = 'static_caching' ARG_TEMPLATE_DIR = 'template_dir' - -BASE_PATH = os.path.abspath( - os.path.join(os.path.dirname(dev_appserver.__file__), '../../../')) +SDK_PATH = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname(os_compat.__file__) + ) + ) + ) DEFAULT_ARGS = { ARG_PORT: 8080, @@ -112,7 +126,7 @@ ARG_LOGIN_URL: '/_ah/login', ARG_CLEAR_DATASTORE: False, ARG_REQUIRE_INDEXES: False, - ARG_TEMPLATE_DIR: os.path.join(BASE_PATH, 'templates'), + ARG_TEMPLATE_DIR: os.path.join(SDK_PATH, 'templates'), ARG_SMTP_HOST: '', ARG_SMTP_PORT: 25, ARG_SMTP_USER: '', @@ -126,6 +140,74 @@ ARG_STATIC_CACHING: True, } +API_PATHS = {'1': + {'google': (), + 'antlr3': ('lib', 'antlr3'), + 'django': ('lib', 'django'), + 'webob': ('lib', 'webob'), + 'yaml': ('lib', 'yaml', 'lib'), + } + } + +DEFAULT_API_VERSION = '1' + +API_PATHS['test'] = API_PATHS[DEFAULT_API_VERSION].copy() +API_PATHS['test']['_test'] = ('nonexistent', 'test', 'path') + + +def SetPaths(app_config_path): + """Set the interpreter to use the specified API version. + + The app.yaml file is scanned for the api_version field and the value is + extracted. With that information, the paths in API_PATHS are added to the + front of sys.paths to make sure that they take precedent over any other paths + to older versions of a package. All modules for each package set are cleared + out of sys.modules to make sure only the newest version is used. + + Args: + - app_config_path: Path to the app.yaml file. + """ + api_version_re = re.compile(r'api_version:\s*(?P[\w.]{1,32})') + api_version = None + app_config_file = open(app_config_path, 'r') + try: + for line in app_config_file: + re_match = api_version_re.match(line) + if re_match: + api_version = re_match.group('api_version') + break + finally: + app_config_file.close() + + if api_version is None: + logging.error("Application configuration file missing an 'api_version' " + "value:\n%s" % app_config_path) + sys.exit(1) + if api_version not in API_PATHS: + logging.error("Value of %r for 'api_version' from the application " + "configuration file is not valid:\n%s" % + (api_version, app_config_path)) + sys.exit(1) + + if api_version == DEFAULT_API_VERSION: + return DEFAULT_API_VERSION + + sdk_path = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname(os_compat.__file__) + ) + ) + ) + for pkg_name, path_parts in API_PATHS[api_version].iteritems(): + for name in sys.modules.keys(): + if name == pkg_name or name.startswith('%s.' % pkg_name): + del sys.modules[name] + pkg_path = os.path.join(sdk_path, *path_parts) + sys.path.insert(0, pkg_path) + + return api_version + def PrintUsageExit(code): """Prints usage information and exits with a status code. @@ -205,10 +287,10 @@ option_dict[ARG_ADDRESS] = value if option == '--datastore_path': - option_dict[ARG_DATASTORE_PATH] = value + option_dict[ARG_DATASTORE_PATH] = os.path.abspath(value) if option == '--history_path': - option_dict[ARG_HISTORY_PATH] = value + option_dict[ARG_HISTORY_PATH] = os.path.abspath(value) if option in ('-c', '--clear_datastore'): option_dict[ARG_CLEAR_DATASTORE] = True @@ -241,10 +323,10 @@ option_dict[ARG_SHOW_MAIL_BODY] = True if option == '--auth_domain': - dev_appserver.DEFAULT_ENV['AUTH_DOMAIN'] = value + option_dict['_DEFAULT_ENV_AUTH_DOMAIN'] = value if option == '--debug_imports': - dev_appserver.HardenedModulesHook.ENABLE_LOGGING = True + option_dict['_ENABLE_LOGGING'] = True if option == '--template_dir': option_dict[ARG_TEMPLATE_DIR] = value @@ -291,6 +373,25 @@ PrintUsageExit(1) root_path = args[0] + for suffix in ('yaml', 'yml'): + path = os.path.join(root_path, 'app.%s' % suffix) + if os.path.exists(path): + api_version = SetPaths(path) + break + else: + logging.error("Application configuration file not found in %s" % root_path) + return 1 + + SetGlobals() + dev_appserver.API_VERSION = api_version + + if '_DEFAULT_ENV_AUTH_DOMAIN' in option_dict: + auth_domain = option_dict['_DEFAULT_ENV_AUTH_DOMAIN'] + dev_appserver.DEFAULT_ENV['AUTH_DOMAIN'] = auth_domain + if '_ENABLE_LOGGING' in option_dict: + enable_logging = option_dict['_ENABLE_LOGGING'] + dev_appserver.HardenedModulesHook.ENABLE_LOGGING = enable_logging + log_level = option_dict[ARG_LOG_LEVEL] port = option_dict[ARG_PORT] datastore_path = option_dict[ARG_DATASTORE_PATH] @@ -335,6 +436,7 @@ login_url, port, template_dir, + sdk_dir=SDK_PATH, serve_address=serve_address, require_indexes=require_indexes, static_caching=static_caching) @@ -359,3 +461,5 @@ if __name__ == '__main__': sys.exit(main(sys.argv)) +else: + SetGlobals() diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/lib/antlr3/antlr3/__init__.py --- a/thirdparty/google_appengine/lib/antlr3/antlr3/__init__.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/lib/antlr3/antlr3/__init__.py Thu Feb 12 12:30:36 2009 +0000 @@ -138,7 +138,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = '3.1' +__version__ = '3.1.1' def version_str_to_tuple(version_str): import re diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/lib/antlr3/antlr_python_runtime.egg-info/PKG-INFO --- a/thirdparty/google_appengine/lib/antlr3/antlr_python_runtime.egg-info/PKG-INFO Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/lib/antlr3/antlr_python_runtime.egg-info/PKG-INFO Thu Feb 12 12:30:36 2009 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: antlr-python-runtime -Version: 3.1 +Version: 3.1.1 Summary: Runtime package for ANTLR3 Home-page: http://www.antlr.org/ Author: Benjamin Niemann diff -r 5c931bd3dc1e -r a7766286a7be thirdparty/google_appengine/lib/antlr3/setup.py --- a/thirdparty/google_appengine/lib/antlr3/setup.py Thu Feb 12 10:24:37 2009 +0000 +++ b/thirdparty/google_appengine/lib/antlr3/setup.py Thu Feb 12 12:30:36 2009 +0000 @@ -267,7 +267,7 @@ setup(name='antlr_python_runtime', - version='3.1', + version='3.1.1', packages=['antlr3'], author="Benjamin Niemann",