1468 parser.add_option("--noisy", action="store_const", const=3, |
1470 parser.add_option("--noisy", action="store_const", const=3, |
1469 dest="verbose", help="Print all logs.") |
1471 dest="verbose", help="Print all logs.") |
1470 parser.add_option("-s", "--server", action="store", dest="server", |
1472 parser.add_option("-s", "--server", action="store", dest="server", |
1471 default="appengine.google.com", |
1473 default="appengine.google.com", |
1472 metavar="SERVER", help="The server to connect to.") |
1474 metavar="SERVER", help="The server to connect to.") |
|
1475 parser.add_option("--secure", action="store_true", dest="secure", |
|
1476 default=False, |
|
1477 help="Use SSL when communicating with the server.") |
1473 parser.add_option("-e", "--email", action="store", dest="email", |
1478 parser.add_option("-e", "--email", action="store", dest="email", |
1474 metavar="EMAIL", default=None, |
1479 metavar="EMAIL", default=None, |
1475 help="The username to use. Will prompt if omitted.") |
1480 help="The username to use. Will prompt if omitted.") |
1476 parser.add_option("-H", "--host", action="store", dest="host", |
1481 parser.add_option("-H", "--host", action="store", dest="host", |
1477 metavar="HOST", default=None, |
1482 metavar="HOST", default=None, |
1844 """ |
1850 """ |
1845 parser.add_option("-n", "--num_runs", type="int", dest="num_runs", |
1851 parser.add_option("-n", "--num_runs", type="int", dest="num_runs", |
1846 action="store", default=5, |
1852 action="store", default=5, |
1847 help="Number of runs of each cron job to display" |
1853 help="Number of runs of each cron job to display" |
1848 "Default is 5") |
1854 "Default is 5") |
|
1855 |
|
1856 def _CheckRequiredUploadOptions(self): |
|
1857 """Checks that upload options are present.""" |
|
1858 for option in ["filename", "kind", "config_file"]: |
|
1859 if getattr(self.options, option) is None: |
|
1860 self.parser.error("Option '%s' is required." % option) |
|
1861 if not self.options.url: |
|
1862 self.parser.error("You must have google.appengine.ext.remote_api.handler " |
|
1863 "assigned to an endpoint in app.yaml, or provide " |
|
1864 "the url of the handler via the 'url' option.") |
|
1865 |
|
1866 def InferUploadUrl(self, appyaml): |
|
1867 """Uses app.yaml to determine the remote_api endpoint. |
|
1868 |
|
1869 Args: |
|
1870 appyaml: A parsed app.yaml file. |
|
1871 |
|
1872 Returns: |
|
1873 The url of the remote_api endpoint as a string, or None |
|
1874 """ |
|
1875 handlers = appyaml.handlers |
|
1876 handler_suffix = "remote_api/handler.py" |
|
1877 app_id = appyaml.application |
|
1878 for handler in handlers: |
|
1879 if hasattr(handler, "script") and handler.script: |
|
1880 if handler.script.endswith(handler_suffix): |
|
1881 server = self.options.server |
|
1882 if server == "appengine.google.com": |
|
1883 return "http://%s.appspot.com%s" % (app_id, handler.url) |
|
1884 else: |
|
1885 return "http://%s%s" % (server, handler.url) |
|
1886 return None |
|
1887 |
|
1888 def RunBulkloader(self, **kwargs): |
|
1889 """Invokes the bulkloader with the given keyword arguments. |
|
1890 |
|
1891 Args: |
|
1892 kwargs: Keyword arguments to pass to bulkloader.Run(). |
|
1893 """ |
|
1894 try: |
|
1895 import sqlite3 |
|
1896 except ImportError: |
|
1897 logging.error("upload_data action requires SQLite3 and the python " |
|
1898 "sqlite3 module (included in python since 2.5).") |
|
1899 sys.exit(1) |
|
1900 |
|
1901 sys.exit(bulkloader.Run(kwargs)) |
|
1902 |
|
1903 def PerformUpload(self, run_fn=None): |
|
1904 """Performs a datastore upload via the bulkloader. |
|
1905 |
|
1906 Args: |
|
1907 run_fn: Function to invoke the bulkloader, used for testing. |
|
1908 """ |
|
1909 if run_fn is None: |
|
1910 run_fn = self.RunBulkloader |
|
1911 |
|
1912 if len(self.args) != 1: |
|
1913 self.parser.error("Expected <directory> argument.") |
|
1914 |
|
1915 basepath = self.args[0] |
|
1916 appyaml = self._ParseAppYaml(basepath) |
|
1917 |
|
1918 self.options.app_id = appyaml.application |
|
1919 |
|
1920 if not self.options.url: |
|
1921 url = self.InferUploadUrl(appyaml) |
|
1922 if url is not None: |
|
1923 self.options.url = url |
|
1924 |
|
1925 self._CheckRequiredUploadOptions() |
|
1926 |
|
1927 if self.options.batch_size < 1: |
|
1928 self.parser.error("batch_size must be 1 or larger.") |
|
1929 |
|
1930 if verbosity == 1: |
|
1931 logging.getLogger().setLevel(logging.INFO) |
|
1932 self.options.debug = False |
|
1933 else: |
|
1934 logging.getLogger().setLevel(logging.DEBUG) |
|
1935 self.options.debug = True |
|
1936 |
|
1937 StatusUpdate("Uploading data records.") |
|
1938 |
|
1939 run_fn(app_id=self.options.app_id, |
|
1940 url=self.options.url, |
|
1941 filename=self.options.filename, |
|
1942 batch_size=self.options.batch_size, |
|
1943 kind=self.options.kind, |
|
1944 num_threads=self.options.num_threads, |
|
1945 bandwidth_limit=self.options.bandwidth_limit, |
|
1946 rps_limit=self.options.rps_limit, |
|
1947 http_limit=self.options.http_limit, |
|
1948 db_filename=self.options.db_filename, |
|
1949 config_file=self.options.config_file, |
|
1950 auth_domain=self.options.auth_domain, |
|
1951 has_header=self.options.has_header, |
|
1952 loader_opts=self.options.loader_opts, |
|
1953 log_file=self.options.log_file, |
|
1954 passin=self.options.passin, |
|
1955 email=self.options.email, |
|
1956 debug=self.options.debug, |
|
1957 |
|
1958 exporter_opts=None, |
|
1959 download=False, |
|
1960 result_db_filename=None, |
|
1961 ) |
|
1962 |
|
1963 def _PerformUploadOptions(self, parser): |
|
1964 """Adds 'upload_data' specific options to the 'parser' passed in. |
|
1965 |
|
1966 Args: |
|
1967 parser: An instance of OptionsParser. |
|
1968 """ |
|
1969 parser.add_option("--filename", type="string", dest="filename", |
|
1970 action="store", |
|
1971 help="The name of the file containing the input data." |
|
1972 " (Required)") |
|
1973 parser.add_option("--config_file", type="string", dest="config_file", |
|
1974 action="store", |
|
1975 help="Name of the configuration file. (Required)") |
|
1976 parser.add_option("--kind", type="string", dest="kind", |
|
1977 action="store", |
|
1978 help="The kind of the entities to store. (Required)") |
|
1979 parser.add_option("--url", type="string", dest="url", |
|
1980 action="store", |
|
1981 help="The location of the remote_api endpoint.") |
|
1982 parser.add_option("--num_threads", type="int", dest="num_threads", |
|
1983 action="store", default=10, |
|
1984 help="Number of threads to upload records with.") |
|
1985 parser.add_option("--batch_size", type="int", dest="batch_size", |
|
1986 action="store", default=10, |
|
1987 help="Number of records to post in each request.") |
|
1988 parser.add_option("--bandwidth_limit", type="int", dest="bandwidth_limit", |
|
1989 action="store", default=250000, |
|
1990 help="The maximum bytes/second bandwidth for transfers.") |
|
1991 parser.add_option("--rps_limit", type="int", dest="rps_limit", |
|
1992 action="store", default=20, |
|
1993 help="The maximum records/second for transfers.") |
|
1994 parser.add_option("--http_limit", type="int", dest="http_limit", |
|
1995 action="store", default=8, |
|
1996 help="The maximum requests/second for transfers.") |
|
1997 parser.add_option("--db_filename", type="string", dest="db_filename", |
|
1998 action="store", |
|
1999 help="Name of the progress database file.") |
|
2000 parser.add_option("--auth_domain", type="string", dest="auth_domain", |
|
2001 action="store", default="gmail.com", |
|
2002 help="The name of the authorization domain to use.") |
|
2003 parser.add_option("--has_header", dest="has_header", |
|
2004 action="store_true", default=False, |
|
2005 help="Whether the first line of the input file should be" |
|
2006 " skipped") |
|
2007 parser.add_option("--loader_opts", type="string", dest="loader_opts", |
|
2008 help="A string to pass to the Loader.Initialize method.") |
|
2009 parser.add_option("--log_file", type="string", dest="log_file", |
|
2010 help="File to write bulkloader logs. If not supplied " |
|
2011 "then a new log file will be created, named: " |
|
2012 "bulkloader-log-TIMESTAMP.") |
1849 |
2013 |
1850 class Action(object): |
2014 class Action(object): |
1851 """Contains information about a command line action. |
2015 """Contains information about a command line action. |
1852 |
2016 |
1853 Attributes: |
2017 Attributes: |