--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/thirdparty/google_appengine/google/appengine/tools/bulkload_client.py Tue Aug 26 21:49:54 2008 +0000
@@ -0,0 +1,297 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Imports CSV data over HTTP.
+
+Usage:
+ %s [flags]
+
+ --debug Show debugging information. (Optional)
+ --cookie=<string> Whole Cookie header to supply to the server, including
+ the parameter name (e.g., "ACSID=..."). (Optional)
+ --url=<string> URL endpoint to post to for importing data. (Required)
+ --batch_size=<int> Number of Entity objects to include in each post to
+ the URL endpoint. The more data per row/Entity, the
+ smaller the batch size should be. (Default 10)
+ --filename=<path> Path to the CSV file to import. (Required)
+ --kind=<string> Name of the Entity object kind to put in the datastore.
+ (Required)
+
+The exit status will be 0 on success, non-zero on import failure.
+
+Works with the bulkload mix-in library for google.appengine.ext.bulkload.
+Please look there for documentation about how to setup the server side.
+"""
+
+
+import StringIO
+import httplib
+import logging
+import csv
+import getopt
+import socket
+import sys
+import urllib
+import urlparse
+
+from google.appengine.ext.bulkload import constants
+
+
+
+class Error(Exception):
+ """Base-class for exceptions in this module."""
+
+
+class PostError(Error):
+ """An error has occured while trying to post data to the server."""
+
+
+class BadServerStatusError(PostError):
+ """The server has returned an error while importing data."""
+
+
+def ContentGenerator(csv_file,
+ batch_size,
+ create_csv_reader=csv.reader,
+ create_csv_writer=csv.writer):
+ """Retrieves CSV data up to a batch size at a time.
+
+ Args:
+ csv_file: A file-like object for reading CSV data.
+ batch_size: Maximum number of CSV rows to yield on each iteration.
+ create_csv_reader, create_csv_writer: Used for dependency injection.
+
+ Yields:
+ Tuple (entity_count, csv_content) where:
+ entity_count: Number of entities contained in the csv_content. Will be
+ less than or equal to the batch_size and greater than 0.
+ csv_content: String containing the CSV content containing the next
+ entity_count entities.
+ """
+ try:
+ csv.field_size_limit(800000)
+ except AttributeError:
+ pass
+
+ reader = create_csv_reader(csv_file, skipinitialspace=True)
+ exhausted = False
+
+ while not exhausted:
+ rows_written = 0
+ content = StringIO.StringIO()
+ writer = create_csv_writer(content)
+ try:
+ for i in xrange(batch_size):
+ row = reader.next()
+ writer.writerow(row)
+ rows_written += 1
+ except StopIteration:
+ exhausted = True
+
+ if rows_written > 0:
+ yield rows_written, content.getvalue()
+
+
+def PostEntities(host_port, uri, cookie, kind, content):
+ """Posts Entity records to a remote endpoint over HTTP.
+
+ Args:
+ host_port: String containing the "host:port" pair; the port is optional.
+ uri: Relative URI to access on the remote host (e.g., '/bulkload').
+ cookie: String containing the Cookie header to use, if any.
+ kind: Kind of the Entity records being posted.
+ content: String containing the CSV data for the entities.
+
+ Raises:
+ BadServerStatusError if the server was contactable but returns an error.
+ PostError If an error occurred while connecting to the server or reading
+ or writing data.
+ """
+ logging.debug('Connecting to %s', host_port)
+ try:
+ body = urllib.urlencode({
+ constants.KIND_PARAM: kind,
+ constants.CSV_PARAM: content,
+ })
+ headers = {
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ 'Content-Length': len(body),
+ 'Cookie': cookie,
+ }
+
+ logging.debug('Posting %d bytes to http://%s%s', len(body), host_port, uri)
+ connection = httplib.HTTPConnection(host_port)
+ try:
+ connection.request('POST', uri, body, headers)
+ response = connection.getresponse()
+
+ status = response.status
+ reason = response.reason
+ content = response.read()
+ logging.debug('Received response code %d: %s', status, reason)
+ if status != httplib.OK:
+ raise BadServerStatusError('Received code %d: %s\n%s' % (
+ status, reason, content))
+ finally:
+ connection.close()
+ except (IOError, httplib.HTTPException, socket.error), e:
+ logging.debug('Encountered exception accessing HTTP server: %s', e)
+ raise PostError(e)
+
+
+def SplitURL(url):
+ """Splits an HTTP URL into pieces.
+
+ Args:
+ url: String containing a full URL string (e.g.,
+ 'http://blah.com:8080/stuff?param=1#foo')
+
+ Returns:
+ Tuple (netloc, uri) where:
+ netloc: String containing the host/port combination from the URL. The
+ port is optional. (e.g., 'blah.com:8080').
+ uri: String containing the relative URI of the URL. (e.g., '/stuff').
+ """
+ scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
+ return netloc, path
+
+
+def ImportCSV(filename,
+ post_url,
+ cookie,
+ batch_size,
+ kind,
+ split_url=SplitURL,
+ openfile=file,
+ create_content_generator=ContentGenerator,
+ post_entities=PostEntities):
+ """Imports CSV data using a series of HTTP posts.
+
+ Args:
+ filename: File on disk containing CSV data.
+ post_url: URL to post the Entity data to.
+ cookie: Full cookie header to use while connecting.
+ batch_size: Maximum number of Entity objects to post with each request.
+ kind: Entity kind of the objects being posted.
+ split_url, openfile, create_content_generator, post_entities: Used for
+ dependency injection.
+
+ Returns:
+ True if all entities were imported successfully; False otherwise.
+ """
+ host_port, uri = split_url(post_url)
+ csv_file = openfile(filename, 'r')
+ try:
+ content_gen = create_content_generator(csv_file, batch_size)
+ logging.info('Starting import; maximum %d entities per post', batch_size)
+ for num_entities, content in content_gen:
+ logging.info('Importing %d entities in %d bytes',
+ num_entities, len(content))
+ try:
+ content = post_entities(host_port, uri, cookie, kind, content)
+ except PostError, e:
+ logging.error('An error occurred while importing: %s', e)
+ return False
+ finally:
+ csv_file.close()
+ return True
+
+
+def PrintUsageExit(code):
+ """Prints usage information and exits with a status code.
+
+ Args:
+ code: Status code to pass to sys.exit() after displaying usage information.
+ """
+ print sys.modules['__main__'].__doc__ % sys.argv[0]
+ sys.stdout.flush()
+ sys.stderr.flush()
+ sys.exit(code)
+
+
+def ParseArguments(argv):
+ """Parses command-line arguments.
+
+ Prints out a help message if -h or --help is supplied.
+
+ Args:
+ argv: List of command-line arguments.
+
+ Returns:
+ Tuple (url, filename, cookie, batch_size, kind) containing the values from
+ each corresponding command-line flag.
+ """
+ opts, args = getopt.getopt(
+ argv[1:],
+ 'h',
+ ['debug',
+ 'help',
+ 'url=',
+ 'filename=',
+ 'cookie=',
+ 'batch_size=',
+ 'kind='])
+
+ url = None
+ filename = None
+ cookie = ''
+ batch_size = 10
+ kind = None
+ encoding = None
+
+ for option, value in opts:
+ if option == '--debug':
+ logging.getLogger().setLevel(logging.DEBUG)
+ if option in ('-h', '--help'):
+ PrintUsageExit(0)
+ if option == '--url':
+ url = value
+ if option == '--filename':
+ filename = value
+ if option == '--cookie':
+ cookie = value
+ if option == '--batch_size':
+ batch_size = int(value)
+ if batch_size <= 0:
+ print >>sys.stderr, 'batch_size must be 1 or larger'
+ PrintUsageExit(1)
+ if option == '--kind':
+ kind = value
+
+ return (url, filename, cookie, batch_size, kind)
+
+
+def main(argv):
+ """Runs the importer."""
+ logging.basicConfig(
+ level=logging.INFO,
+ format='%(levelname)-8s %(asctime)s %(filename)s] %(message)s')
+
+ args = ParseArguments(argv)
+ if [arg for arg in args if arg is None]:
+ print >>sys.stderr, 'Invalid arguments'
+ PrintUsageExit(1)
+
+ url, filename, cookie, batch_size, kind = args
+ if ImportCSV(filename, url, cookie, batch_size, kind):
+ logging.info('Import succcessful')
+ return 0
+ logging.error('Import failed')
+ return 1
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))