56 from google.appengine.tools import bulkloader

  1160     self.in_transaction = False

  1475     parser.add_option("--secure", action="store_true", dest="secure",

  1476                       default=False,

  1477                       help="Use SSL when communicating with the server.")

  1855 

  1856   def _CheckRequiredUploadOptions(self):

  1857     """Checks that upload options are present."""

  1858     for option in ["filename", "kind", "config_file"]:

  1859       if getattr(self.options, option) is None:

  1860         self.parser.error("Option '%s' is required." % option)

  1861     if not self.options.url:

  1862       self.parser.error("You must have google.appengine.ext.remote_api.handler "

  1863                         "assigned to an endpoint in app.yaml, or provide "

  1864                         "the url of the handler via the 'url' option.")

  1865 

  1866   def InferUploadUrl(self, appyaml):

  1867     """Uses app.yaml to determine the remote_api endpoint.

  1868 

  1869     Args:

  1870       appyaml: A parsed app.yaml file.

  1871 

  1872     Returns:

  1873       The url of the remote_api endpoint as a string, or None

  1874     """

  1875     handlers = appyaml.handlers

  1876     handler_suffix = "remote_api/handler.py"

  1877     app_id = appyaml.application

  1878     for handler in handlers:

  1879       if hasattr(handler, "script") and handler.script:

  1880         if handler.script.endswith(handler_suffix):

  1881           server = self.options.server

  1882           if server == "appengine.google.com":

  1883             return "http://%s.appspot.com%s" % (app_id, handler.url)

  1884           else:

  1885             return "http://%s%s" % (server, handler.url)

  1886     return None

  1887 

  1888   def RunBulkloader(self, **kwargs):

  1889     """Invokes the bulkloader with the given keyword arguments.

  1890 

  1891     Args:

  1892       kwargs: Keyword arguments to pass to bulkloader.Run().

  1893     """

  1894     try:

  1895       import sqlite3

  1896     except ImportError:

  1897       logging.error("upload_data action requires SQLite3 and the python "

  1898                     "sqlite3 module (included in python since 2.5).")

  1899       sys.exit(1)

  1900 

  1901     sys.exit(bulkloader.Run(kwargs))

  1902 

  1903   def PerformUpload(self, run_fn=None):

  1904     """Performs a datastore upload via the bulkloader.

  1905 

  1906     Args:

  1907       run_fn: Function to invoke the bulkloader, used for testing.

  1908     """

  1909     if run_fn is None:

  1910       run_fn = self.RunBulkloader

  1911 

  1912     if len(self.args) != 1:

  1913       self.parser.error("Expected <directory> argument.")

  1914 

  1915     basepath = self.args[0]

  1916     appyaml = self._ParseAppYaml(basepath)

  1917 

  1918     self.options.app_id = appyaml.application

  1919 

  1920     if not self.options.url:

  1921       url = self.InferUploadUrl(appyaml)

  1922       if url is not None:

  1923         self.options.url = url

  1924 

  1925     self._CheckRequiredUploadOptions()

  1926 

  1927     if self.options.batch_size < 1:

  1928       self.parser.error("batch_size must be 1 or larger.")

  1929 

  1930     if verbosity == 1:

  1931       logging.getLogger().setLevel(logging.INFO)

  1932       self.options.debug = False

  1933     else:

  1934       logging.getLogger().setLevel(logging.DEBUG)

  1935       self.options.debug = True

  1936 

  1937     StatusUpdate("Uploading data records.")

  1938 

  1939     run_fn(app_id=self.options.app_id,

  1940            url=self.options.url,

  1941            filename=self.options.filename,

  1942            batch_size=self.options.batch_size,

  1943            kind=self.options.kind,

  1944            num_threads=self.options.num_threads,

  1945            bandwidth_limit=self.options.bandwidth_limit,

  1946            rps_limit=self.options.rps_limit,

  1947            http_limit=self.options.http_limit,

  1948            db_filename=self.options.db_filename,

  1949            config_file=self.options.config_file,

  1950            auth_domain=self.options.auth_domain,

  1951            has_header=self.options.has_header,

  1952            loader_opts=self.options.loader_opts,

  1953            log_file=self.options.log_file,

  1954            passin=self.options.passin,

  1955            email=self.options.email,

  1956            debug=self.options.debug,

  1957 

  1958            exporter_opts=None,

  1959            download=False,

  1960            result_db_filename=None,

  1961            )

  1962 

  1963   def _PerformUploadOptions(self, parser):

  1964     """Adds 'upload_data' specific options to the 'parser' passed in.

  1965 

  1966     Args:

  1967       parser: An instance of OptionsParser.

  1968     """

  1969     parser.add_option("--filename", type="string", dest="filename",

  1970                       action="store",

  1971                       help="The name of the file containing the input data."

  1972                       " (Required)")

  1973     parser.add_option("--config_file", type="string", dest="config_file",

  1974                       action="store",

  1975                       help="Name of the configuration file. (Required)")

  1976     parser.add_option("--kind", type="string", dest="kind",

  1977                       action="store",

  1978                       help="The kind of the entities to store. (Required)")

  1979     parser.add_option("--url", type="string", dest="url",

  1980                       action="store",

  1981                       help="The location of the remote_api endpoint.")

  1982     parser.add_option("--num_threads", type="int", dest="num_threads",

  1983                       action="store", default=10,

  1984                       help="Number of threads to upload records with.")

  1985     parser.add_option("--batch_size", type="int", dest="batch_size",

  1986                       action="store", default=10,

  1987                       help="Number of records to post in each request.")

  1988     parser.add_option("--bandwidth_limit", type="int", dest="bandwidth_limit",

  1989                       action="store", default=250000,

  1990                       help="The maximum bytes/second bandwidth for transfers.")

  1991     parser.add_option("--rps_limit", type="int", dest="rps_limit",

  1992                       action="store", default=20,

  1993                       help="The maximum records/second for transfers.")

  1994     parser.add_option("--http_limit", type="int", dest="http_limit",

  1995                       action="store", default=8,

  1996                       help="The maximum requests/second for transfers.")

  1997     parser.add_option("--db_filename", type="string", dest="db_filename",

  1998                       action="store",

  1999                       help="Name of the progress database file.")

  2000     parser.add_option("--auth_domain", type="string", dest="auth_domain",

  2001                       action="store", default="gmail.com",

  2002                       help="The name of the authorization domain to use.")

  2003     parser.add_option("--has_header", dest="has_header",

  2004                       action="store_true", default=False,

  2005                       help="Whether the first line of the input file should be"

  2006                       " skipped")

  2007     parser.add_option("--loader_opts", type="string", dest="loader_opts",

  2008                       help="A string to pass to the Loader.Initialize method.")

  2009     parser.add_option("--log_file", type="string", dest="log_file",

  2010                       help="File to write bulkloader logs.  If not supplied "

  2011                            "then a new log file will be created, named: "

  2012                            "bulkloader-log-TIMESTAMP.")

  2120       "upload_data": Action(

  2121           function="PerformUpload",

  2122           usage="%prog [options] upload_data <directory>",

  2123           options=_PerformUploadOptions,

  2124           short_desc="Upload CSV records to datastore",

  2125           long_desc="""

  2126 The 'upload_data' command translates CSV records into datastore entities and

  2127 uploads them into your application's datastore."""),

  2128