1 #!/usr/bin/env python

     2 #

     3 # Copyright 2007 Google Inc.

     4 #

     5 # Licensed under the Apache License, Version 2.0 (the "License");

     6 # you may not use this file except in compliance with the License.

     7 # You may obtain a copy of the License at

     8 #

     9 #     http://www.apache.org/licenses/LICENSE-2.0

    10 #

    11 # Unless required by applicable law or agreed to in writing, software

    12 # distributed under the License is distributed on an "AS IS" BASIS,

    13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

    14 # See the License for the specific language governing permissions and

    15 # limitations under the License.

    16 

    17 """Tool for uploading diffs from a version control system to the codereview app.

    18 

    19 Usage summary: upload.py [options] [-- diff_options]

    20 

    21 Diff options are passed to the diff command of the underlying system.

    22 

    23 Supported version control systems:

    24   Git

    25   Subversion

    26 

    27 (It is important for Git users to specify a tree-ish to diff against.)

    28 """

    29 # This code is derived from appcfg.py in the App Engine SDK (open source),

    30 # and from ASPN recipe #146306.

    31 

    32 import cookielib

    33 import getpass

    34 import logging

    35 import md5

    36 import mimetypes

    37 import optparse

    38 import os

    39 import re

    40 import socket

    41 import subprocess

    42 import sys

    43 import urllib

    44 import urllib2

    45 import urlparse

    46 

    47 try:

    48   import readline

    49 except ImportError:

    50   pass

    51 

    52 # The logging verbosity:

    53 #  0: Errors only.

    54 #  1: Status messages.

    55 #  2: Info logs.

    56 #  3: Debug logs.

    57 verbosity = 1

    58 

    59 # Max size of patch or base file.

    60 MAX_UPLOAD_SIZE = 900 * 1024

    61 

    62 

    63 def StatusUpdate(msg):

    64   """Print a status message to stdout.

    65 

    66   If 'verbosity' is greater than 0, print the message.

    67 

    68   Args:

    69     msg: The string to print.

    70   """

    71   if verbosity > 0:

    72     print msg

    73 

    74 

    75 def ErrorExit(msg):

    76   """Print an error message to stderr and exit."""

    77   print >>sys.stderr, msg

    78   sys.exit(1)

    79 

    80 

    81 class ClientLoginError(urllib2.HTTPError):

    82   """Raised to indicate there was an error authenticating with ClientLogin."""

    83 

    84   def __init__(self, url, code, msg, headers, args):

    85     urllib2.HTTPError.__init__(self, url, code, msg, headers, None)

    86     self.args = args

    87     self.reason = args["Error"]

    88 

    89 

    90 class AbstractRpcServer(object):

    91   """Provides a common interface for a simple RPC server."""

    92 

    93   def __init__(self, host, auth_function, host_override=None, extra_headers={},

    94                save_cookies=False):

    95     """Creates a new HttpRpcServer.

    96 

    97     Args:

    98       host: The host to send requests to.

    99       auth_function: A function that takes no arguments and returns an

   100         (email, password) tuple when called. Will be called if authentication

   101         is required.

   102       host_override: The host header to send to the server (defaults to host).

   103       extra_headers: A dict of extra headers to append to every request.

   104       save_cookies: If True, save the authentication cookies to local disk.

   105         If False, use an in-memory cookiejar instead.  Subclasses must

   106         implement this functionality.  Defaults to False.

   107     """

   108     self.host = host

   109     self.host_override = host_override

   110     self.auth_function = auth_function

   111     self.authenticated = False

   112     self.extra_headers = extra_headers

   113     self.save_cookies = save_cookies

   114     self.opener = self._GetOpener()

   115     if self.host_override:

   116       logging.info("Server: %s; Host: %s", self.host, self.host_override)

   117     else:

   118       logging.info("Server: %s", self.host)

   119 

   120   def _GetOpener(self):

   121     """Returns an OpenerDirector for making HTTP requests.

   122 

   123     Returns:

   124       A urllib2.OpenerDirector object.

   125     """

   126     raise NotImplementedError()

   127 

   128   def _CreateRequest(self, url, data=None):

   129     """Creates a new urllib request."""

   130     logging.debug("Creating request for: '%s' with payload:\n%s", url, data)

   131     req = urllib2.Request(url, data=data)

   132     if self.host_override:

   133       req.add_header("Host", self.host_override)

   134     for key, value in self.extra_headers.iteritems():

   135       req.add_header(key, value)

   136     return req

   137 

   138   def _GetAuthToken(self, email, password):

   139     """Uses ClientLogin to authenticate the user, returning an auth token.

   140 

   141     Args:

   142       email:    The user's email address

   143       password: The user's password

   144 

   145     Raises:

   146       ClientLoginError: If there was an error authenticating with ClientLogin.

   147       HTTPError: If there was some other form of HTTP error.

   148 

   149     Returns:

   150       The authentication token returned by ClientLogin.

   151     """

   152     req = self._CreateRequest(

   153         url="https://www.google.com/accounts/ClientLogin",

   154         data=urllib.urlencode({

   155             "Email": email,

   156             "Passwd": password,

   157             "service": "ah",

   158             "source": "rietveld-codereview-upload",

   159             "accountType": "GOOGLE",

   160         })

   161     )

   162     try:

   163       response = self.opener.open(req)

   164       response_body = response.read()

   165       response_dict = dict(x.split("=")

   166                            for x in response_body.split("\n") if x)

   167       return response_dict["Auth"]

   168     except urllib2.HTTPError, e:

   169       if e.code == 403:

   170         body = e.read()

   171         response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)

   172         raise ClientLoginError(req.get_full_url(), e.code, e.msg,

   173                                e.headers, response_dict)

   174       else:

   175         raise

   176 

   177   def _GetAuthCookie(self, auth_token):

   178     """Fetches authentication cookies for an authentication token.

   179 

   180     Args:

   181       auth_token: The authentication token returned by ClientLogin.

   182 

   183     Raises:

   184       HTTPError: If there was an error fetching the authentication cookies.

   185     """

   186     # This is a dummy value to allow us to identify when we're successful.

   187     continue_location = "http://localhost/"

   188     args = {"continue": continue_location, "auth": auth_token}

   189     req = self._CreateRequest("http://%s/_ah/login?%s" %

   190                               (self.host, urllib.urlencode(args)))

   191     try:

   192       response = self.opener.open(req)

   193     except urllib2.HTTPError, e:

   194       response = e

   195     if (response.code != 302 or

   196         response.info()["location"] != continue_location):

   197       raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,

   198                               response.headers, response.fp)

   199     self.authenticated = True

   200 

   201   def _Authenticate(self):

   202     """Authenticates the user.

   203 

   204     The authentication process works as follows:

   205      1) We get a username and password from the user

   206      2) We use ClientLogin to obtain an AUTH token for the user

   207         (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).

   208      3) We pass the auth token to /_ah/login on the server to obtain an

   209         authentication cookie. If login was successful, it tries to redirect

   210         us to the URL we provided.

   211 

   212     If we attempt to access the upload API without first obtaining an

   213     authentication cookie, it returns a 401 response and directs us to

   214     authenticate ourselves with ClientLogin.

   215     """

   216     for i in range(3):

   217       credentials = self.auth_function()

   218       try:

   219         auth_token = self._GetAuthToken(credentials[0], credentials[1])

   220       except ClientLoginError, e:

   221         if e.reason == "BadAuthentication":

   222           print >>sys.stderr, "Invalid username or password."

   223           continue

   224         if e.reason == "CaptchaRequired":

   225           print >>sys.stderr, (

   226               "Please go to\n"

   227               "https://www.google.com/accounts/DisplayUnlockCaptcha\n"

   228               "and verify you are a human.  Then try again.")

   229           break

   230         if e.reason == "NotVerified":

   231           print >>sys.stderr, "Account not verified."

   232           break

   233         if e.reason == "TermsNotAgreed":

   234           print >>sys.stderr, "User has not agreed to TOS."

   235           break

   236         if e.reason == "AccountDeleted":

   237           print >>sys.stderr, "The user account has been deleted."

   238           break

   239         if e.reason == "AccountDisabled":

   240           print >>sys.stderr, "The user account has been disabled."

   241           break

   242         if e.reason == "ServiceDisabled":

   243           print >>sys.stderr, ("The user's access to the service has been "

   244                                "disabled.")

   245           break

   246         if e.reason == "ServiceUnavailable":

   247           print >>sys.stderr, "The service is not available; try again later."

   248           break

   249         raise

   250       self._GetAuthCookie(auth_token)

   251       return

   252 

   253   def Send(self, request_path, payload=None,

   254            content_type="application/octet-stream",

   255            timeout=None,

   256            **kwargs):

   257     """Sends an RPC and returns the response.

   258 

   259     Args:

   260       request_path: The path to send the request to, eg /api/appversion/create.

   261       payload: The body of the request, or None to send an empty request.

   262       content_type: The Content-Type header to use.

   263       timeout: timeout in seconds; default None i.e. no timeout.

   264         (Note: for large requests on OS X, the timeout doesn't work right.)

   265       kwargs: Any keyword arguments are converted into query string parameters.

   266 

   267     Returns:

   268       The response body, as a string.

   269     """

   270     # TODO: Don't require authentication.  Let the server say

   271     # whether it is necessary.

   272     if not self.authenticated:

   273       self._Authenticate()

   274 

   275     old_timeout = socket.getdefaulttimeout()

   276     socket.setdefaulttimeout(timeout)

   277     try:

   278       tries = 0

   279       while True:

   280         tries += 1

   281         args = dict(kwargs)

   282         url = "http://%s%s" % (self.host, request_path)

   283         if args:

   284           url += "?" + urllib.urlencode(args)

   285         req = self._CreateRequest(url=url, data=payload)

   286         req.add_header("Content-Type", content_type)

   287         try:

   288           f = self.opener.open(req)

   289           response = f.read()

   290           f.close()

   291           return response

   292         except urllib2.HTTPError, e:

   293           if tries > 3:

   294             raise

   295           elif e.code == 401:

   296             self._Authenticate()

   297 ##           elif e.code >= 500 and e.code < 600:

   298 ##             # Server Error - try again.

   299 ##             continue

   300           else:

   301             raise

   302     finally:

   303       socket.setdefaulttimeout(old_timeout)

   304 

   305 

   306 class HttpRpcServer(AbstractRpcServer):

   307   """Provides a simplified RPC-style interface for HTTP requests."""

   308 

   309   def _Authenticate(self):

   310     """Save the cookie jar after authentication."""

   311     super(HttpRpcServer, self)._Authenticate()

   312     if self.save_cookies:

   313       StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)

   314       self.cookie_jar.save()

   315 

   316   def _GetOpener(self):

   317     """Returns an OpenerDirector that supports cookies and ignores redirects.

   318 

   319     Returns:

   320       A urllib2.OpenerDirector object.

   321     """

   322     opener = urllib2.OpenerDirector()

   323     opener.add_handler(urllib2.ProxyHandler())

   324     opener.add_handler(urllib2.UnknownHandler())

   325     opener.add_handler(urllib2.HTTPHandler())

   326     opener.add_handler(urllib2.HTTPDefaultErrorHandler())

   327     opener.add_handler(urllib2.HTTPSHandler())

   328     opener.add_handler(urllib2.HTTPErrorProcessor())

   329     if self.save_cookies:

   330       self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")

   331       self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)

   332       if os.path.exists(self.cookie_file):

   333         try:

   334           self.cookie_jar.load()

   335           self.authenticated = True

   336           StatusUpdate("Loaded authentication cookies from %s" %

   337                        self.cookie_file)

   338         except (cookielib.LoadError, IOError):

   339           # Failed to load cookies - just ignore them.

   340           pass

   341       else:

   342         # Create an empty cookie file with mode 600

   343         fd = os.open(self.cookie_file, os.O_CREAT, 0600)

   344         os.close(fd)

   345       # Always chmod the cookie file

   346       os.chmod(self.cookie_file, 0600)

   347     else:

   348       # Don't save cookies across runs of update.py.

   349       self.cookie_jar = cookielib.CookieJar()

   350     opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))

   351     return opener

   352 

   353 

   354 parser = optparse.OptionParser(usage="%prog [options] [-- diff_options]")

   355 parser.add_option("-y", "--assume_yes", action="store_true",

   356                   dest="assume_yes", default=False,

   357                   help="Assume that the answer to yes/no questions is 'yes'.")

   358 # Logging

   359 group = parser.add_option_group("Logging options")

   360 group.add_option("-q", "--quiet", action="store_const", const=0,

   361                  dest="verbose", help="Print errors only.")

   362 group.add_option("-v", "--verbose", action="store_const", const=2,

   363                  dest="verbose", default=1,

   364                  help="Print info level logs (default).")

   365 group.add_option("--noisy", action="store_const", const=3,

   366                  dest="verbose", help="Print all logs.")

   367 # Review server

   368 group = parser.add_option_group("Review server options")

   369 group.add_option("-s", "--server", action="store", dest="server",

   370                  default="codereview.appspot.com",

   371                  metavar="SERVER",

   372                  help=("The server to upload to. The format is host[:port]. "

   373                        "Defaults to 'codereview.appspot.com'."))

   374 group.add_option("-e", "--email", action="store", dest="email",

   375                  metavar="EMAIL", default=None,

   376                  help="The username to use. Will prompt if omitted.")

   377 group.add_option("-H", "--host", action="store", dest="host",

   378                  metavar="HOST", default=None,

   379                  help="Overrides the Host header sent with all RPCs.")

   380 group.add_option("--no_cookies", action="store_false",

   381                  dest="save_cookies", default=True,

   382                  help="Do not save authentication cookies to local disk.")

   383 # Issue

   384 group = parser.add_option_group("Issue options")

   385 group.add_option("-d", "--description", action="store", dest="description",

   386                  metavar="DESCRIPTION", default=None,

   387                  help="Optional description when creating an issue.")

   388 group.add_option("-f", "--description_file", action="store",

   389                  dest="description_file", metavar="DESCRIPTION_FILE",

   390                  default=None,

   391                  help="Optional path of a file that contains "

   392                       "the description when creating an issue.")

   393 group.add_option("-r", "--reviewers", action="store", dest="reviewers",

   394                  metavar="REVIEWERS", default=None,

   395                  help="Add reviewers (comma separated email addresses).")

   396 group.add_option("--cc", action="store", dest="cc",

   397                  metavar="CC", default=None,

   398                  help="Add CC (comma separated email addresses).")

   399 # Upload options

   400 group = parser.add_option_group("Patch options")

   401 group.add_option("-m", "--message", action="store", dest="message",

   402                  metavar="MESSAGE", default=None,

   403                  help="A message to identify the patch. "

   404                       "Will prompt if omitted.")

   405 group.add_option("-i", "--issue", type="int", action="store",

   406                  metavar="ISSUE", default=None,

   407                  help="Issue number to which to add. Defaults to new issue.")

   408 group.add_option("-l", "--local_base", action="store_true",

   409                  dest="local_base", default=False,

   410                  help="Base files will be uploaded.")

   411 group.add_option("--send_mail", action="store_true",

   412                  dest="send_mail", default=False,

   413                  help="Send notification email to reviewers.")

   414 

   415 

   416 def GetRpcServer(options):

   417   """Returns an instance of an AbstractRpcServer.

   418 

   419   Returns:

   420     A new AbstractRpcServer, on which RPC calls can be made.

   421   """

   422 

   423   rpc_server_class = HttpRpcServer

   424 

   425   def GetUserCredentials():

   426     """Prompts the user for a username and password."""

   427     email = options.email

   428     if email is None:

   429       email = raw_input("Email: ").strip()

   430     password = getpass.getpass("Password for %s: " % email)

   431     return (email, password)

   432 

   433   # If this is the dev_appserver, use fake authentication.

   434   host = (options.host or options.server).lower()

   435   if host == "localhost" or host.startswith("localhost:"):

   436     email = options.email

   437     if email is None:

   438       email = "test@example.com"

   439       logging.info("Using debug user %s.  Override with --email" % email)

   440     server = rpc_server_class(

   441         options.server,

   442         lambda: (email, "password"),

   443         host_override=options.host,

   444         extra_headers={"Cookie":

   445                        'dev_appserver_login="%s:False"' % email},

   446         save_cookies=options.save_cookies)

   447     # Don't try to talk to ClientLogin.

   448     server.authenticated = True

   449     return server

   450 

   451   return rpc_server_class(options.server, GetUserCredentials,

   452                           host_override=options.host,

   453                           save_cookies=options.save_cookies)

   454 

   455 

   456 def EncodeMultipartFormData(fields, files):

   457   """Encode form fields for multipart/form-data.

   458 

   459   Args:

   460     fields: A sequence of (name, value) elements for regular form fields.

   461     files: A sequence of (name, filename, value) elements for data to be

   462            uploaded as files.

   463   Returns:

   464     (content_type, body) ready for httplib.HTTP instance.

   465 

   466   Source:

   467     http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306

   468   """

   469   BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'

   470   CRLF = '\r\n'

   471   lines = []

   472   for (key, value) in fields:

   473     lines.append('--' + BOUNDARY)

   474     lines.append('Content-Disposition: form-data; name="%s"' % key)

   475     lines.append('')

   476     lines.append(value)

   477   for (key, filename, value) in files:

   478     lines.append('--' + BOUNDARY)

   479     lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %

   480              (key, filename))

   481     lines.append('Content-Type: %s' % GetContentType(filename))

   482     lines.append('')

   483     lines.append(value)

   484   lines.append('--' + BOUNDARY + '--')

   485   lines.append('')

   486   body = CRLF.join(lines)

   487   content_type = 'multipart/form-data; boundary=%s' % BOUNDARY

   488   return content_type, body

   489 

   490 

   491 def GetContentType(filename):

   492   """Helper to guess the content-type from the filename."""

   493   return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

   494 

   495 

   496 # Use a shell for subcommands on Windows to get a PATH search.

   497 use_shell = sys.platform.startswith("win")

   498 

   499 

   500 def RunShell(command, silent_ok=False, universal_newlines=False):

   501   logging.info("Running %s", command)

   502   p = subprocess.Popen(command, stdout=subprocess.PIPE,

   503                        stderr=subprocess.STDOUT, shell=use_shell,

   504                        universal_newlines=universal_newlines)

   505   data = p.stdout.read()

   506   p.wait()

   507   p.stdout.close()

   508   if p.returncode:

   509     ErrorExit("Got error status from %s" % command)

   510   if not silent_ok and not data:

   511     ErrorExit("No output from %s" % command)

   512   return data

   513 

   514 

   515 class VersionControlSystem(object):

   516   """Abstract base class providing an interface to the VCS."""

   517 

   518   def GenerateDiff(self, args):

   519     """Return the current diff as a string.

   520 

   521     Args:

   522       args: Extra arguments to pass to the diff command.

   523     """

   524     raise NotImplementedError(

   525         "abstract method -- subclass %s must override" % self.__class__)

   526 

   527   def GetUnknownFiles(self):

   528     """Return a list of files unknown to the VCS."""

   529     raise NotImplementedError(

   530         "abstract method -- subclass %s must override" % self.__class__)

   531 

   532   def CheckForUnknownFiles(self):

   533     """Show an "are you sure?" prompt if there are unknown files."""

   534     unknown_files = self.GetUnknownFiles()

   535     if unknown_files:

   536       print "The following files are not added to version control:"

   537       for line in unknown_files:

   538         print line

   539       prompt = "Are you sure to continue?(y/N) "

   540       answer = raw_input(prompt).strip()

   541       if answer != "y":

   542         ErrorExit("User aborted")

   543 

   544   def GetBaseFile(self, filename):

   545     """Get the content of the upstream version of a file.

   546 

   547     Returns:

   548       A tuple (content, status) representing the file content and the status of

   549       the file.

   550     """

   551 

   552     raise NotImplementedError(

   553         "abstract method -- subclass %s must override" % self.__class__)

   554 

   555   def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options):

   556     """Uploads the base files."""

   557     patches = dict()

   558     [patches.setdefault(v, k) for k, v in patch_list]

   559     for filename in patches.keys():

   560       content, status = self.GetBaseFile(filename)

   561       no_base_file = False

   562       if len(content) > MAX_UPLOAD_SIZE:

   563         print ("Not uploading the base file for " + filename +

   564                " because the file is too large.")

   565         no_base_file = True

   566         content = ""

   567       checksum = md5.new(content).hexdigest()

   568       if options.verbose > 0:

   569         print "Uploading %s" % filename

   570       url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset),

   571                                           int(patches.get(filename)))

   572       form_fields = [("filename", filename),

   573                      ("status", status),

   574                      ("checksum", checksum),]

   575       if no_base_file:

   576         form_fields.append(("no_base_file", "1"))

   577       if options.email:

   578         form_fields.append(("user", options.email))

   579       ctype, body = EncodeMultipartFormData(form_fields,

   580                                             [("data", filename, content)])

   581       response_body = rpc_server.Send(url, body, content_type=ctype)

   582       if not response_body.startswith("OK"):

   583         StatusUpdate("  --> %s" % response_body)

   584         sys.exit(False)

   585 

   586 

   587 class SubversionVCS(VersionControlSystem):

   588   """Implementation of the VersionControlSystem interface for Subversion."""

   589 

   590   def GuessBase(self, required):

   591     """Returns the SVN base URL.

   592 

   593     Args:

   594       required: If true, exits if the url can't be guessed, otherwise None is

   595         returned.

   596     """

   597     info = RunShell(["svn", "info"])

   598     for line in info.splitlines():

   599       words = line.split()

   600       if len(words) == 2 and words[0] == "URL:":

   601         url = words[1]

   602         scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)

   603         username, netloc = urllib.splituser(netloc)

   604         if username:

   605           logging.info("Removed username from base URL")

   606         if netloc.endswith("svn.python.org"):

   607           if netloc == "svn.python.org":

   608             if path.startswith("/projects/"):

   609               path = path[9:]

   610           elif netloc != "pythondev@svn.python.org":

   611             ErrorExit("Unrecognized Python URL: %s" % url)

   612           base = "http://svn.python.org/view/*checkout*%s/" % path

   613           logging.info("Guessed Python base = %s", base)

   614         elif netloc.endswith("svn.collab.net"):

   615           if path.startswith("/repos/"):

   616             path = path[6:]

   617           base = "http://svn.collab.net/viewvc/*checkout*%s/" % path

   618           logging.info("Guessed CollabNet base = %s", base)

   619         elif netloc.endswith(".googlecode.com"):

   620           path = path + "/"

   621           base = urlparse.urlunparse(("http", netloc, path, params,

   622                                       query, fragment))

   623           logging.info("Guessed Google Code base = %s", base)

   624         else:

   625           path = path + "/"

   626           base = urlparse.urlunparse((scheme, netloc, path, params,

   627                                       query, fragment))

   628           logging.info("Guessed base = %s", base)

   629         return base

   630     if required:

   631       ErrorExit("Can't find URL in output from svn info")

   632     return None

   633 

   634   def GenerateDiff(self, args):

   635     cmd = ["svn", "diff"]

   636     if not sys.platform.startswith("win"):

   637       cmd.append("--diff-cmd=diff")

   638     cmd.extend(args)

   639     data = RunShell(cmd)

   640     count = 0

   641     for line in data.splitlines():

   642       if line.startswith("Index:") or line.startswith("Property changes on:"):

   643         count += 1

   644         logging.info(line)

   645     if not count:

   646       ErrorExit("No valid patches found in output from svn diff")

   647     return data

   648 

   649   def _CollapseKeywords(self, content, keyword_str):

   650     """Collapses SVN keywords."""

   651     # svn cat translates keywords but svn diff doesn't. As a result of this

   652     # behavior patching.PatchChunks() fails with a chunk mismatch error.

   653     # This part was originally written by the Review Board development team

   654     # who had the same problem (http://reviews.review-board.org/r/276/).

   655     # Mapping of keywords to known aliases

   656     svn_keywords = {

   657       # Standard keywords

   658       'Date':                ['Date', 'LastChangedDate'],

   659       'Revision':            ['Revision', 'LastChangedRevision', 'Rev'],

   660       'Author':              ['Author', 'LastChangedBy'],

   661       'HeadURL':             ['HeadURL', 'URL'],

   662       'Id':                  ['Id'],

   663 

   664       # Aliases

   665       'LastChangedDate':     ['LastChangedDate', 'Date'],

   666       'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],

   667       'LastChangedBy':       ['LastChangedBy', 'Author'],

   668       'URL':                 ['URL', 'HeadURL'],

   669     }

   670     def repl(m):

   671        if m.group(2):

   672          return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))

   673        return "$%s$" % m.group(1)

   674     keywords = [keyword

   675                 for name in keyword_str.split(" ")

   676                 for keyword in svn_keywords.get(name, [])]

   677     return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)

   678 

   679   def GetUnknownFiles(self):

   680     status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)

   681     unknown_files = []

   682     for line in status.split("\n"):

   683       if line and line[0] == "?":

   684         unknown_files.append(line)

   685     return unknown_files

   686 

   687   def GetBaseFile(self, filename):

   688     status = RunShell(["svn", "status", "--ignore-externals", filename])

   689     if not status:

   690       StatusUpdate("svn status returned no output for %s" % filename)

   691       sys.exit(False)

   692     status_lines = status.splitlines()

   693     # If file is in a cl, the output will begin with

   694     # "\n--- Changelist 'cl_name':\n".  See

   695     # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt

   696     if (len(status_lines) == 3 and

   697         not status_lines[0] and

   698         status_lines[1].startswith("--- Changelist")):

   699       status = status_lines[2]

   700     else:

   701       status = status_lines[0]

   702     # If a file is copied its status will be "A  +", which signifies

   703     # "addition-with-history".  See "svn st" for more information.  We need to

   704     # upload the original file or else diff parsing will fail if the file was

   705     # edited.

   706     if ((status[0] == "A" and status[3] != "+") or

   707         (status[0] == " " and status[1] == "M")):  # property changed

   708       content = ""

   709     elif (status[0] in ("M", "D", "R") or

   710           (status[0] == "A" and status[3] == "+")):

   711       mimetype = RunShell(["svn", "-rBASE", "propget", "svn:mime-type",

   712                            filename],

   713                           silent_ok=True)

   714       if mimetype.startswith("application/octet-stream"):

   715         content = ""

   716       else:

   717         # On Windows svn cat gives \r\n, and calling subprocess.Popen turns

   718         # them into \r\r\n, so use universal newlines to avoid the extra \r.

   719         if sys.platform.startswith("win"):

   720           nl = True

   721         else:

   722           nl = False

   723         content = RunShell(["svn", "cat", filename], universal_newlines=nl)

   724       keywords = RunShell(["svn", "-rBASE", "propget", "svn:keywords",

   725                            filename],

   726                           silent_ok=True)

   727       if keywords:

   728         content = self._CollapseKeywords(content, keywords)

   729     else:

   730       StatusUpdate("svn status returned unexpected output: %s" % status)

   731       sys.exit(False)

   732     return content, status[0:5]

   733 

   734 

   735 class GitVCS(VersionControlSystem):

   736   """Implementation of the VersionControlSystem interface for Git."""

   737 

   738   def __init__(self):

   739     # Map of filename -> hash of base file.

   740     self.base_hashes = {}

   741 

   742   def GenerateDiff(self, extra_args):

   743     # This is more complicated than svn's GenerateDiff because we must convert

   744     # the diff output to include an svn-style "Index:" line as well as record

   745     # the hashes of the base files, so we can upload them along with our diff.

   746     gitdiff = RunShell(["git", "diff", "--full-index"] + extra_args)

   747     svndiff = []

   748     filecount = 0

   749     filename = None

   750     for line in gitdiff.splitlines():

   751       match = re.match(r"diff --git a/(.*) b/.*$", line)

   752       if match:

   753         filecount += 1

   754         filename = match.group(1)

   755         svndiff.append("Index: %s\n" % filename)

   756       else:

   757         # The "index" line in a git diff looks like this (long hashes elided):

   758         #   index 82c0d44..b2cee3f 100755

   759         # We want to save the left hash, as that identifies the base file.

   760         match = re.match(r"index (\w+)\.\.", line)

   761         if match:

   762           self.base_hashes[filename] = match.group(1)

   763       svndiff.append(line + "\n")

   764     if not filecount:

   765       ErrorExit("No valid patches found in output from git diff")

   766     return "".join(svndiff)

   767 

   768   def GetUnknownFiles(self):

   769     status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],

   770                       silent_ok=True)

   771     return status.splitlines()

   772 

   773   def GetBaseFile(self, filename):

   774     hash = self.base_hashes[filename]

   775     if hash == "0" * 40:  # All-zero hash indicates no base file.

   776       return ("", "A")

   777     else:

   778       return (RunShell(["git", "show", hash]), "M")

   779 

   780 

   781 # NOTE: this function is duplicated in engine.py, keep them in sync.

   782 def SplitPatch(data):

   783   """Splits a patch into separate pieces for each file.

   784 

   785   Args:

   786     data: A string containing the output of svn diff.

   787 

   788   Returns:

   789     A list of 2-tuple (filename, text) where text is the svn diff output

   790       pertaining to filename.

   791   """

   792   patches = []

   793   filename = None

   794   diff = []

   795   for line in data.splitlines(True):

   796     new_filename = None

   797     if line.startswith('Index:'):

   798       unused, new_filename = line.split(':', 1)

   799       new_filename = new_filename.strip()

   800     elif line.startswith('Property changes on:'):

   801       unused, temp_filename = line.split(':', 1)

   802       # When a file is modified, paths use '/' between directories, however

   803       # when a property is modified '\' is used on Windows.  Make them the same

   804       # otherwise the file shows up twice.

   805       temp_filename = temp_filename.strip().replace('\\', '/')

   806       if temp_filename != filename:

   807         # File has property changes but no modifications, create a new diff.

   808         new_filename = temp_filename

   809     if new_filename:

   810       if filename and diff:

   811         patches.append((filename, ''.join(diff)))

   812       filename = new_filename

   813       diff = [line]

   814       continue

   815     if diff is not None:

   816       diff.append(line)

   817   if filename and diff:

   818     patches.append((filename, ''.join(diff)))

   819   return patches

   820 

   821 

   822 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):

   823   """Uploads a separate patch for each file in the diff output.

   824 

   825   Returns a list of [patch_key, filename] for each file.

   826   """

   827   patches = SplitPatch(data)

   828   rv = []

   829   for patch in patches:

   830     if len(patch[1]) > MAX_UPLOAD_SIZE:

   831       print ("Not uploading the patch for " + patch[0] +

   832              " because the file is too large.")

   833       continue

   834     form_fields = [("filename", patch[0])]

   835     if options.local_base:

   836       form_fields.append(("content_upload", "1"))

   837     files = [("data", "data.diff", patch[1])]

   838     ctype, body = EncodeMultipartFormData(form_fields, files)

   839     url = "/%d/upload_patch/%d" % (int(issue), int(patchset))

   840     print "Uploading patch for " + patch[0]

   841     response_body = rpc_server.Send(url, body, content_type=ctype)

   842     lines = response_body.splitlines()

   843     if not lines or lines[0] != "OK":

   844       StatusUpdate("  --> %s" % response_body)

   845       sys.exit(False)

   846     rv.append([lines[1], patch[0]])

   847   return rv

   848 

   849 

   850 def GuessVCS():

   851   """Helper to guess the version control system.

   852 

   853   This examines the current directory, guesses which VersionControlSystem

   854   we're using, and returns an instance of the appropriate class.  Exit with an

   855   error if we can't figure it out.

   856 

   857   Returns:

   858     A VersionControlSystem instance. Exits if the VCS can't be guessed.

   859   """

   860   # Subversion has a .svn in all working directories.

   861   if os.path.isdir('.svn'):

   862     logging.info("Guessed VCS = Subversion")

   863     return SubversionVCS()

   864 

   865   # Git has a command to test if you're in a git tree.

   866   # Try running it, but don't die if we don't have git installed.

   867   try:

   868     subproc = subprocess.Popen(["git", "rev-parse", "--is-inside-work-tree"],

   869                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)

   870     if subproc.wait() == 0:

   871       return GitVCS()

   872   except OSError, (errno, message):

   873     if errno != 2:  # ENOENT -- they don't have git installed.

   874       raise

   875 

   876   ErrorExit(("Could not guess version control system. "

   877              "Are you in a working copy directory?"))

   878 

   879 

   880 def RealMain(argv, data=None):

   881   logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"

   882                               "%(lineno)s %(message)s "))

   883   os.environ['LC_ALL'] = 'C'

   884   options, args = parser.parse_args(argv[1:])

   885   global verbosity

   886   verbosity = options.verbose

   887   if verbosity >= 3:

   888     logging.getLogger().setLevel(logging.DEBUG)

   889   elif verbosity >= 2:

   890     logging.getLogger().setLevel(logging.INFO)

   891   vcs = GuessVCS()

   892   if isinstance(vcs, SubversionVCS):

   893     # base field is only allowed for Subversion.

   894     # Note: Fetching base files may become deprecated in future releases.

   895     base = vcs.GuessBase(not options.local_base)

   896   else:

   897     base = None

   898   if not base and not options.local_base:

   899     options.local_base = True

   900     logging.info("Enabled upload of base file")

   901   if not options.assume_yes:

   902     vcs.CheckForUnknownFiles()

   903   if data is None:

   904     data = vcs.GenerateDiff(args)

   905   if verbosity >= 1:

   906     print "Upload server:", options.server, "(change with -s/--server)"

   907   if options.issue:

   908     prompt = "Message describing this patch set: "

   909   else:

   910     prompt = "New issue subject: "

   911   message = options.message or raw_input(prompt).strip()

   912   if not message:

   913     ErrorExit("A non-empty message is required")

   914   rpc_server = GetRpcServer(options)

   915   form_fields = [("subject", message)]

   916   if base:

   917     form_fields.append(("base", base))

   918   if options.issue:

   919     form_fields.append(("issue", str(options.issue)))

   920   if options.email:

   921     form_fields.append(("user", options.email))

   922   if options.reviewers:

   923     for reviewer in options.reviewers.split(','):

   924       if reviewer.count("@") != 1 or "." not in reviewer.split("@")[1]:

   925         ErrorExit("Invalid email address: %s" % reviewer)

   926     form_fields.append(("reviewers", options.reviewers))

   927   if options.cc:

   928     for cc in options.cc.split(','):

   929       if cc.count("@") != 1 or "." not in cc.split("@")[1]:

   930         ErrorExit("Invalid email address: %s" % cc)

   931     form_fields.append(("cc", options.cc))

   932   description = options.description

   933   if options.description_file:

   934     if options.description:

   935       ErrorExit("Can't specify description and description_file")

   936     file = open(options.description_file, 'r')

   937     description = file.read()

   938     file.close()

   939   if description:

   940     form_fields.append(("description", description))

   941   # If we're uploading base files, don't send the email before the uploads, so

   942   # that it contains the file status.

   943   if options.send_mail and not options.local_base:

   944     form_fields.append(("send_mail", "1"))

   945   if options.local_base:

   946     form_fields.append(("content_upload", "1"))

   947   if len(data) > MAX_UPLOAD_SIZE:

   948     print "Patch is large, so uploading file patches separately."

   949     files = []

   950     form_fields.append(("separate_patches", "1"))

   951   else:

   952     files = [("data", "data.diff", data)]

   953   ctype, body = EncodeMultipartFormData(form_fields, files)

   954   response_body = rpc_server.Send("/upload", body, content_type=ctype)

   955   if options.local_base or not files:

   956     lines = response_body.splitlines()

   957     if len(lines) >= 2:

   958       msg = lines[0]

   959       patchset = lines[1].strip()

   960       patches = [x.split(" ", 1) for x in lines[2:]]

   961     else:

   962       msg = response_body

   963   else:

   964     msg = response_body

   965   StatusUpdate(msg)

   966   if not response_body.startswith("Issue created.") and \

   967   not response_body.startswith("Issue updated."):

   968     sys.exit(0)

   969   issue = msg[msg.rfind("/")+1:]

   970 

   971   if not files:

   972     result = UploadSeparatePatches(issue, rpc_server, patchset, data, options)

   973     if options.local_base:

   974       patches = result

   975 

   976   if options.local_base:

   977     vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options)

   978     if options.send_mail:

   979       rpc_server.Send("/" + issue + "/mail", payload="")

   980   return issue

   981 

   982 

   983 def main():

   984   try:

   985     RealMain(sys.argv)

   986   except KeyboardInterrupt:

   987     print

   988     StatusUpdate("Interrupted.")

   989     sys.exit(1)

   990 

   991 

   992 if __name__ == "__main__":

   993   main()