py_tasks_melange: comparison thirdparty/google_appengine/google/appengine/ext/bulkload/__init_

equal deleted inserted replaced

-:37505d64e57b
+:f2e327a7c5de
 bulkload.main(person)
 See the Loader class for more information. Then, add a handler for it in your
 app.yaml, e.g.:
-urlmap:
+handlers:
-- regex: /load
+- url: /load
-handler:
+script: bulkload.py
-type: 1
+login: admin
-path: bulkload.py
-requires_login: true
+Finally, deploy your app and run bulkloader.py. For example, to load the
-admin_only: true
-Finally, deploy your app and run bulkload_client.py. For example, to load the
 file people.csv into a dev_appserver running on your local machine:
-./bulkload_client.py --filename people.csv --kind Person --cookie ... \
+./bulkloader.py --filename people.csv --kind Person --cookie ... \
 --url http://localhost:8080/load
 The kind parameter is used to look up the Loader instance that will be used.
 The bulkload handler should usually be admin_only, so that non-admins can't use
 the shell to modify your app's data. The bulkload client uses the cookie
 import httplib
 import os
 import sys
 import traceback
 import types
+import struct
 import google
 import wsgiref.handlers
 raise TypeError('Expected a %s, but received %s (a %s).' %
 (type, value, value.__class__))
 class Loader(object):
-""" A base class for creating datastore entities from CSV input data.
+"""A base class for creating datastore entities from input data.
 To add a handler for bulk loading a new entity kind into your datastore,
 write a subclass of this class that calls Loader.__init__ from your
 class's __init__.
-If you need to run extra code to convert entities from CSV, create new
+If you need to run extra code to convert entities from the input
-properties, or otherwise modify the entities before they're inserted,
+data, create new properties, or otherwise modify the entities before
-override HandleEntity.
+they're inserted, override HandleEntity.
+See the CreateEntity method for the creation of entities from the
+(parsed) input data.
 """
 __loaders = {}
 __kind = None
 __properties = None
 def kind(self):
 """ Return the entity kind that this Loader handes.
 """
 return self.__kind
+def CreateEntity(self, values, key_name=None):
-def CreateEntity(self, values):
 """ Creates an entity from a list of property values.
 Args:
-values: list of str
+values: list/tuple of str
+key_name: if provided, the name for the (single) resulting Entity
 Returns:
 list of datastore.Entity
 The returned entities are populated with the property values from the
 argument, converted to native types using the properties map given in
 the constructor, and passed through HandleEntity. They're ready to be
 inserted.
-Raises an AssertionError if the number of values doesn't match the number
+Raises:
-of properties in the properties map.
+AssertionError if the number of values doesn't match the number
-"""
+of properties in the properties map.
-Validate(values, list)
+"""
+Validate(values, (list, tuple))
 assert len(values) == len(self.__properties), (
 'Expected %d CSV columns, found %d.' %
 (len(self.__properties), len(values)))
-entity = datastore.Entity(self.__kind)
+entity = datastore.Entity(self.__kind, name=key_name)
 for (name, converter), val in zip(self.__properties, values):
 entity[name] = converter(val)
 entities = self.HandleEntity(entity)
 if entities is not None:
-if not isinstance(entities, list):
+if not isinstance(entities, (list, tuple)):
 entities = [entities]
 for entity in entities:
 if not isinstance(entity, datastore.Entity):
 raise TypeError('Expected a datastore.Entity, received %s (a %s).' %
 """
 return dict(Loader.__loaders)
 class BulkLoad(webapp.RequestHandler):
-""" A handler for bulk load requests.
+"""A handler for bulk load requests.
+This class contains handlers for the bulkloading process. One for
+GET to provide cookie information for the upload script, and one
+handler for a POST request to upload the entities.
+In the POST request, the body contains the data representing the
+entities' property values. The original format was a sequences of
+lines of comma-separated values (and is handled by the Load
+method). The current (version 1) format is a binary format described
+in the Tools and Libraries section of the documentation, and is
+handled by the LoadV1 method).
 """
 def get(self):
 """ Handle a GET. Just show an info page.
 """
 def post(self):
 """ Handle a POST. Reads CSV data, converts to entities, and stores them.
 """
 self.response.headers['Content-Type'] = 'text/plain'
-response, output = self.Load(self.request.get(constants.KIND_PARAM),
+version = self.request.headers.get('GAE-Uploader-Version', '0')
-self.request.get(constants.CSV_PARAM))
+if version == '1':
+kind = self.request.headers.get('GAE-Uploader-Kind')
+response, output = self.LoadV1(kind, self.request.body)
+else:
+response, output = self.Load(self.request.get(constants.KIND_PARAM),
+self.request.get(constants.CSV_PARAM))
 self.response.set_status(response)
 self.response.out.write(output)
 def InfoPage(self, uri):
 new_entities = loader.CreateEntity(columns)
 if new_entities:
 entities.extend(new_entities)
 output.append('done.')
 except:
-exc_info = sys.exc_info()
+stacktrace = traceback.format_exc()
-stacktrace = traceback.format_exception(*exc_info)
 output.append('error:\n%s' % stacktrace)
 return (httplib.BAD_REQUEST, ''.join(output))
 line_num += 1
 for entity in entities:
 datastore.Put(entity)
 return (httplib.OK, ''.join(output))
+def LoadV1(self, kind, data):
+"""Parses version-1 format data, converts to entities, and stores them.
+On error, fails fast. Returns a "bad request" HTTP response code and
+includes the traceback in the output.
+Args:
+kind: a string containing the entity kind that this loader handles
+data: a string containing the (v1 format) data to load
+Returns:
+tuple (response code, output) where:
+response code: integer HTTP response code to return
+output: string containing the HTTP response body
+"""
+Validate(kind, basestring)
+Validate(data, basestring)
+output = []
+try:
+loader = Loader.RegisteredLoaders()[kind]
+except KeyError:
+output.append('Error: no Loader defined for kind %s.' % kind)
+return httplib.BAD_REQUEST, ''.join(output)
+entities = []
+column_count, = struct.unpack_from('!i', data)
+offset = 4
+lengths_format = '!%di' % (column_count,)
+while offset < len(data):
+id_num = struct.unpack_from('!i', data, offset=offset)
+offset += 4
+key_name = 'i%010d' % id_num
+value_lengths = struct.unpack_from(lengths_format, data, offset=offset)
+offset += 4 * column_count
+columns = struct.unpack_from(''.join('%ds' % length
+for length in value_lengths), data,
+offset=offset)
+offset += sum(value_lengths)
+try:
+output.append('Loading key_name=%s... ' % key_name)
+new_entities = loader.CreateEntity(columns, key_name=key_name)
+if new_entities:
+entities.extend(new_entities)
+output.append('done.\n')
+except:
+stacktrace = traceback.format_exc()
+output.append('error:\n%s' % stacktrace)
+return httplib.BAD_REQUEST, ''.join(output)
+for entity in entities:
+datastore.Put(entity)
+return httplib.OK, ''.join(output)
 def main(*loaders):
 """Starts bulk upload.

changeset 149	f2e327a7c5de
parent 109	620f9b141567
child 297	35211afcd563