1 #!/usr/bin/python

     2 # Wrapper script around Rietveld's upload.py that groups files into

     3 # changelists.

     4 

     5 import getpass

     6 import linecache

     7 import os

     8 import random

     9 import re

    10 import string

    11 import subprocess

    12 import sys

    13 import tempfile

    14 import upload

    15 import urllib2

    16 

    17 CODEREVIEW_SETTINGS = {

    18   # Default values.

    19   "CODE_REVIEW_SERVER": "codereview.chromium.org",

    20   "CC_LIST": "chromium-reviews@googlegroups.com",

    21   "VIEW_VC": "http://src.chromium.org/viewvc/chrome?view=rev&revision=",

    22 }

    23 

    24 # Use a shell for subcommands on Windows to get a PATH search, and because svn

    25 # may be a batch file.

    26 use_shell = sys.platform.startswith("win")

    27 

    28 

    29 # globals that store the root of the current repositary and the directory where

    30 # we store information about changelists.

    31 repository_root = ""

    32 gcl_info_dir = ""

    33 

    34 

    35 def GetSVNFileInfo(file, field):

    36   """Returns a field from the svn info output for the given file."""

    37   output = RunShell(["svn", "info", file])

    38   for line in output.splitlines():

    39     search = field + ": "

    40     if line.startswith(search):

    41       return line[len(search):]

    42   return ""

    43 

    44 

    45 def GetRepositoryRoot():

    46   """Returns the top level directory of the current repository."""

    47   global repository_root

    48   if not repository_root:

    49     cur_dir_repo_root = GetSVNFileInfo(os.getcwd(), "Repository Root")

    50     if not cur_dir_repo_root:

    51       ErrorExit("gcl run outside of repository")

    52 

    53     repository_root = os.getcwd()

    54     while True:

    55       parent = os.path.dirname(repository_root)

    56       if GetSVNFileInfo(parent, "Repository Root") != cur_dir_repo_root:

    57         break

    58       repository_root = parent

    59   # Now read the code review settings for this repository.

    60   settings_file = os.path.join(repository_root, "codereview.settings")

    61   if os.path.exists(settings_file):

    62     output = ReadFile(settings_file)

    63     for line in output.splitlines():

    64       if not line or line.startswith("#"):

    65         continue

    66       key, value = line.split(": ", 1)

    67       CODEREVIEW_SETTINGS[key] = value

    68   return repository_root

    69 

    70 

    71 def GetCodeReviewSetting(key):

    72   """Returns a value for the given key for this repository."""

    73   return CODEREVIEW_SETTINGS.get(key, "")

    74 

    75 

    76 def GetInfoDir():

    77   """Returns the directory where gcl info files are stored."""

    78   global gcl_info_dir

    79   if not gcl_info_dir:

    80     gcl_info_dir = os.path.join(GetRepositoryRoot(), '.svn', 'gcl_info')

    81   return gcl_info_dir

    82 

    83 

    84 def ErrorExit(msg):

    85   """Print an error message to stderr and exit."""

    86   print >>sys.stderr, msg

    87   sys.exit(1)

    88 

    89 

    90 def RunShell(command, print_output=False):

    91   """Executes a command and returns the output."""

    92   p = subprocess.Popen(command, stdout = subprocess.PIPE,

    93                        stderr = subprocess.STDOUT, shell = use_shell,

    94                        universal_newlines=True)

    95   if print_output:

    96     output_array = []

    97     while True:

    98       line = p.stdout.readline()

    99       if not line:

   100         break

   101       if print_output:

   102         print line.strip('\n')

   103       output_array.append(line)

   104     output = "".join(output_array)

   105   else:

   106     output = p.stdout.read()

   107   p.wait()

   108   p.stdout.close()

   109   return output

   110 

   111 

   112 def ReadFile(filename):

   113   """Returns the contents of a file."""

   114   file = open(filename, 'r')

   115   result = file.read()

   116   file.close()

   117   return result

   118 

   119 

   120 def WriteFile(filename, contents):

   121   """Overwrites the file with the given contents."""

   122   file = open(filename, 'w')

   123   file.write(contents)

   124   file.close()

   125 

   126 

   127 class ChangeInfo:

   128   """Holds information about a changelist.

   129   

   130     issue: the Rietveld issue number, of "" if it hasn't been uploaded yet.

   131     description: the description.

   132     files: a list of 2 tuple containing (status, filename) of changed files,

   133            with paths being relative to the top repository directory.

   134   """

   135   def __init__(self, name="", issue="", description="", files=[]):

   136     self.name = name

   137     self.issue = issue

   138     self.description = description

   139     self.files = files

   140 

   141   def FileList(self):

   142     """Returns a list of files."""

   143     return [file[1] for file in self.files]

   144 

   145   def Save(self):

   146     """Writes the changelist information to disk."""

   147     data = SEPARATOR.join([self.issue,

   148                           "\n".join([f[0] + f[1] for f in self.files]),

   149                           self.description])

   150     WriteFile(GetChangelistInfoFile(self.name), data)

   151 

   152   def Delete(self):

   153     """Removes the changelist information from disk."""

   154     os.remove(GetChangelistInfoFile(self.name))

   155 

   156   def CloseIssue(self):

   157     """Closes the Rietveld issue for this changelist."""

   158     data = [("description", self.description),]

   159     ctype, body = upload.EncodeMultipartFormData(data, [])

   160     SendToRietveld("/" + self.issue + "/close", body, ctype)

   161 

   162   def UpdateRietveldDescription(self):

   163     """Sets the description for an issue on Rietveld."""

   164     data = [("description", self.description),]

   165     ctype, body = upload.EncodeMultipartFormData(data, [])

   166     SendToRietveld("/" + self.issue + "/description", body, ctype)

   167 

   168   

   169 SEPARATOR = "\n-----\n"

   170 # The info files have the following format:

   171 # issue_id\n

   172 # SEPARATOR\n

   173 # filepath1\n

   174 # filepath2\n

   175 # .

   176 # .

   177 # filepathn\n

   178 # SEPARATOR\n

   179 # description

   180 

   181 

   182 def GetChangelistInfoFile(changename):

   183   """Returns the file that stores information about a changelist."""

   184   if not changename or re.search(r'\W', changename):

   185     ErrorExit("Invalid changelist name: " + changename)

   186   return os.path.join(GetInfoDir(), changename)

   187 

   188 

   189 def LoadChangelistInfo(changename, fail_on_not_found=True,

   190                        update_status=False):

   191   """Gets information about a changelist.

   192   

   193   Args:

   194     fail_on_not_found: if True, this function will quit the program if the

   195       changelist doesn't exist.

   196     update_status: if True, the svn status will be updated for all the files

   197       and unchanged files will be removed.

   198   

   199   Returns: a ChangeInfo object.

   200   """

   201   info_file = GetChangelistInfoFile(changename)

   202   if not os.path.exists(info_file):

   203     if fail_on_not_found:

   204       ErrorExit("Changelist " + changename + " not found.")

   205     return ChangeInfo(changename)

   206   data = ReadFile(info_file)

   207   split_data = data.split(SEPARATOR, 2)

   208   if len(split_data) != 3:

   209     os.remove(info_file)

   210     ErrorExit("Changelist file %s was corrupt and deleted" % info_file)

   211   issue = split_data[0]

   212   files = []

   213   for line in split_data[1].splitlines():

   214     status = line[:7]

   215     file = line[7:]

   216     files.append((status, file))

   217   description = split_data[2]  

   218   save = False

   219   if update_status:

   220     for file in files:

   221       filename = os.path.join(GetRepositoryRoot(), file[1])

   222       status = RunShell(["svn", "status", filename])[:7]

   223       if not status:  # File has been reverted.

   224         save = True

   225         files.remove(file)

   226       elif status != file[0]:

   227         save = True

   228         files[files.index(file)] = (status, file[1])

   229   change_info = ChangeInfo(changename, issue, description, files)

   230   if save:

   231     change_info.Save()

   232   return change_info

   233 

   234 

   235 def GetCLs():

   236   """Returns a list of all the changelists in this repository."""

   237   return os.listdir(GetInfoDir())

   238 

   239 

   240 def GenerateChangeName():

   241   """Generate a random changelist name."""

   242   random.seed()

   243   current_cl_names = GetCLs()

   244   while True:

   245     cl_name = (random.choice(string.ascii_lowercase) +

   246                random.choice(string.digits) +

   247                random.choice(string.ascii_lowercase) +

   248                random.choice(string.digits))

   249     if cl_name not in current_cl_names:

   250       return cl_name

   251 

   252 

   253 def GetModifiedFiles():

   254   """Returns a set that maps from changelist name to (status,filename) tuples.

   255 

   256   Files not in a changelist have an empty changelist name.  Filenames are in

   257   relation to the top level directory of the current repositary.  Note that

   258   only the current directory and subdirectories are scanned, in order to

   259   improve performance while still being flexible.

   260   """

   261   files = {}

   262   

   263   # Since the files are normalized to the root folder of the repositary, figure

   264   # out what we need to add to the paths.

   265   dir_prefix = os.getcwd()[len(GetRepositoryRoot()):].strip(os.sep)

   266 

   267   # Get a list of all files in changelists.

   268   files_in_cl = {}

   269   for cl in GetCLs():

   270     change_info = LoadChangelistInfo(cl)

   271     for status, filename in change_info.files:

   272       files_in_cl[filename] = change_info.name

   273 

   274   # Get all the modified files.

   275   status = RunShell(["svn", "status"])

   276   for line in status.splitlines():

   277     if not len(line) or line[0] == "?":

   278       continue

   279     status = line[:7]

   280     filename = line[7:]

   281     if dir_prefix:

   282       filename = os.path.join(dir_prefix, filename)

   283     change_list_name = ""

   284     if filename in files_in_cl:

   285       change_list_name = files_in_cl[filename]

   286     files.setdefault(change_list_name, []).append((status, filename))

   287 

   288   return files

   289 

   290 

   291 def GetFilesNotInCL():

   292   """Returns a list of tuples (status,filename) that aren't in any changelists.

   293   

   294   See docstring of GetModifiedFiles for information about path of files and

   295   which directories are scanned.

   296   """

   297   modified_files = GetModifiedFiles()

   298   if "" not in modified_files:

   299     return []

   300   return modified_files[""]

   301 

   302 

   303 def SendToRietveld(request_path, payload=None,

   304                    content_type="application/octet-stream"):

   305   """Send a POST/GET to Rietveld.  Returns the response body."""

   306   def GetUserCredentials():

   307     """Prompts the user for a username and password."""

   308     email = raw_input("Email: ").strip()

   309     password = getpass.getpass("Password for %s: " % email)

   310     return email, password

   311 

   312   server = GetCodeReviewSetting("CODE_REVIEW_SERVER")

   313   rpc_server = upload.HttpRpcServer(server,

   314                                     GetUserCredentials,

   315                                     host_override=server,

   316                                     save_cookies=True)

   317   return rpc_server.Send(request_path, payload, content_type)

   318 

   319 

   320 def GetIssueDescription(issue):

   321   """Returns the issue description from Rietveld."""

   322   return SendToRietveld("/" + issue + "/description")

   323 

   324 

   325 def UnknownFiles(extra_args):

   326   """Runs svn status and prints unknown files.

   327 

   328   Any args in |extra_args| are passed to the tool to support giving alternate

   329   code locations.

   330   """

   331   args = ["svn", "status"]

   332   args += extra_args

   333   p = subprocess.Popen(args, stdout = subprocess.PIPE,

   334                        stderr = subprocess.STDOUT, shell = use_shell)

   335   while 1:

   336     line = p.stdout.readline()

   337     if not line:

   338       break

   339     if line[0] != '?':

   340       continue  # Not an unknown file to svn.

   341     # The lines look like this:

   342     # "?      foo.txt"

   343     # and we want just "foo.txt"

   344     print line[7:].strip()

   345   p.wait()

   346   p.stdout.close()

   347 

   348 

   349 def Opened():

   350   """Prints a list of modified files in the current directory down."""

   351   files = GetModifiedFiles()

   352   cl_keys = files.keys()

   353   cl_keys.sort()

   354   for cl_name in cl_keys:

   355     if cl_name:

   356       note = ""

   357       if len(LoadChangelistInfo(cl_name).files) != len(files[cl_name]):

   358         note = " (Note: this changelist contains files outside this directory)"

   359       print "\n--- Changelist " + cl_name + note + ":"

   360     for file in files[cl_name]:

   361       print "".join(file)

   362 

   363 

   364 def Help():

   365   print ("GCL is a wrapper for Subversion that simplifies working with groups "

   366          "of files.\n")

   367   print "Basic commands:"

   368   print "-----------------------------------------"

   369   print "   gcl change change_name"

   370   print ("      Add/remove files to a changelist.  Only scans the current "

   371          "directory and subdirectories.\n")

   372   print ("   gcl upload change_name [-r reviewer1@gmail.com,"

   373          "reviewer2@gmail.com,...] [--send_mail]")

   374   print "      Uploads the changelist to the server for review.\n"

   375   print "   gcl commit change_name"

   376   print "      Commits the changelist to the repository.\n"

   377   print "Advanced commands:"

   378   print "-----------------------------------------"

   379   print "   gcl delete change_name"

   380   print "      Deletes a changelist.\n"

   381   print "   gcl diff change_name"

   382   print "      Diffs all files in the changelist.\n"

   383   print "   gcl diff"

   384   print ("      Diffs all files in the current directory and subdirectories "

   385          "that aren't in a changelist.\n")

   386   print "   gcl changes"

   387   print "      Lists all the the changelists and the files in them.\n"

   388   print "   gcl nothave [optional directory]"

   389   print "      Lists files unknown to Subversion.\n"

   390   print "   gcl opened"

   391   print ("      Lists modified files in the current directory and "

   392          "subdirectories.\n")

   393   print "   gcl try change_name"

   394   print ("      Sends the change to the tryserver so a trybot can do a test"

   395          " run on your code.\n")

   396 

   397 

   398 def GetEditor():

   399   editor = os.environ.get("SVN_EDITOR")

   400   if not editor:

   401     editor = os.environ.get("EDITOR")

   402 

   403   if not editor:

   404     if sys.platform.startswith("win"):

   405       editor = "notepad"

   406     else:

   407       editor = "vi"

   408 

   409   return editor

   410 

   411 

   412 def GenerateDiff(files):

   413   """Returns a string containing the diff for the given file list."""

   414   diff = []

   415   for file in files:

   416     # Use svn info output instead of os.path.isdir because the latter fails

   417     # when the file is deleted.

   418     if GetSVNFileInfo(file, "Node Kind") == "directory":

   419       continue

   420     # If the user specified a custom diff command in their svn config file,

   421     # then it'll be used when we do svn diff, which we don't want to happen

   422     # since we want the unified diff.  Using --diff-cmd=diff doesn't always

   423     # work, since they can have another diff executable in their path that

   424     # gives different line endings.  So we use a bogus temp directory as the

   425     # config directory, which gets around these problems.

   426     if sys.platform.startswith("win"):

   427       parent_dir = tempfile.gettempdir()

   428     else:

   429       parent_dir = sys.path[0]  # tempdir is not secure.

   430     bogus_dir = os.path.join(parent_dir, "temp_svn_config")

   431     if not os.path.exists(bogus_dir):

   432       os.mkdir(bogus_dir)

   433     diff.append(RunShell(["svn", "diff", "--config-dir", bogus_dir, file]))

   434   return "".join(diff)

   435 

   436 

   437 def UploadCL(change_info, args):

   438   if not change_info.FileList():

   439     print "Nothing to upload, changelist is empty."

   440     return

   441 

   442   upload_arg = ["upload.py", "-y", "-l"]

   443   upload_arg.append("--server=" + GetCodeReviewSetting("CODE_REVIEW_SERVER"))

   444   upload_arg.extend(args)

   445 

   446   desc_file = ""

   447   if change_info.issue:  # Uploading a new patchset.

   448     upload_arg.append("--message=''")

   449     upload_arg.append("--issue=" + change_info.issue)

   450   else: # First time we upload.

   451     handle, desc_file = tempfile.mkstemp(text=True)

   452     os.write(handle, change_info.description)

   453     os.close(handle)

   454 

   455     upload_arg.append("--cc=" + GetCodeReviewSetting("CC_LIST"))

   456     upload_arg.append("--description_file=" + desc_file + "")

   457     if change_info.description:

   458       subject = change_info.description[:77]

   459       if subject.find("\r\n") != -1:

   460         subject = subject[:subject.find("\r\n")]

   461       if subject.find("\n") != -1:

   462         subject = subject[:subject.find("\n")]

   463       if len(change_info.description) > 77:

   464         subject = subject + "..."

   465       upload_arg.append("--message=" + subject)

   466   

   467   # Change the current working directory before calling upload.py so that it

   468   # shows the correct base.

   469   os.chdir(GetRepositoryRoot())

   470 

   471   # If we have a lot of files with long paths, then we won't be able to fit

   472   # the command to "svn diff".  Instead, we generate the diff manually for

   473   # each file and concatenate them before passing it to upload.py.

   474   issue = upload.RealMain(upload_arg, GenerateDiff(change_info.FileList()))

   475   if issue and issue != change_info.issue:

   476     change_info.issue = issue

   477     change_info.Save()

   478 

   479   if desc_file:

   480     os.remove(desc_file)

   481 

   482 

   483 def TryChange(change_info, args):

   484   """Create a diff file of change_info and send it to the try server."""

   485   try:

   486     import trychange

   487   except ImportError:

   488     ErrorExit("You need to install trychange.py to use the try server.")

   489 

   490   trychange.TryChange(args, change_info.name, change_info.FileList())

   491 

   492 

   493 def Commit(change_info):

   494   if not change_info.FileList():

   495     print "Nothing to commit, changelist is empty."

   496     return

   497 

   498   commit_cmd = ["svn", "commit"]

   499   filename = ''

   500   if change_info.issue:

   501     # Get the latest description from Rietveld.

   502     change_info.description = GetIssueDescription(change_info.issue)

   503 

   504   commit_message = change_info.description.replace('\r\n', '\n')

   505   if change_info.issue:

   506     commit_message += ('\nReview URL: http://%s/%s' %

   507                        (GetCodeReviewSetting("CODE_REVIEW_SERVER"),

   508                         change_info.issue))

   509 

   510   handle, commit_filename = tempfile.mkstemp(text=True)

   511   os.write(handle, commit_message)

   512   os.close(handle)

   513 

   514   handle, targets_filename = tempfile.mkstemp(text=True)

   515   os.write(handle, "\n".join(change_info.FileList()))

   516   os.close(handle)

   517 

   518   commit_cmd += ['--file=' + commit_filename]

   519   commit_cmd += ['--targets=' + targets_filename]

   520   # Change the current working directory before calling commit.

   521   os.chdir(GetRepositoryRoot())

   522   output = RunShell(commit_cmd, True)

   523   os.remove(commit_filename)

   524   os.remove(targets_filename)

   525   if output.find("Committed revision") != -1:

   526     change_info.Delete()

   527 

   528     if change_info.issue:

   529       revision = re.compile(".*?\nCommitted revision (\d+)",

   530                             re.DOTALL).match(output).group(1)

   531       viewvc_url = GetCodeReviewSetting("VIEW_VC")

   532       change_info.description = (change_info.description +

   533                                  "\n\nCommitted: " + viewvc_url + revision)

   534       change_info.CloseIssue()

   535 

   536 

   537 def Change(change_info):

   538   """Creates/edits a changelist."""

   539   if change_info.issue:

   540     try:

   541       description = GetIssueDescription(change_info.issue)

   542     except urllib2.HTTPError, err:

   543       if err.code == 404:

   544         # The user deleted the issue in Rietveld, so forget the old issue id.

   545         description = change_info.description

   546         change_info.issue = ""

   547         change_info.Save()

   548       else:

   549         ErrorExit("Error getting the description from Rietveld: " + err)

   550   else:

   551     description = change_info.description

   552 

   553   other_files = GetFilesNotInCL()

   554 

   555   separator1 = ("\n---All lines above this line become the description.\n"

   556                 "---Repository Root: " + GetRepositoryRoot() + "\n"

   557                 "---Paths in this changelist (" + change_info.name + "):\n")

   558   separator2 = "\n\n---Paths modified but not in any changelist:\n\n"

   559   text = (description + separator1 + '\n' +

   560           '\n'.join([f[0] + f[1] for f in change_info.files]) + separator2 +

   561           '\n'.join([f[0] + f[1] for f in other_files]) + '\n')

   562 

   563   handle, filename = tempfile.mkstemp(text=True)

   564   os.write(handle, text)

   565   os.close(handle)

   566   

   567   command = GetEditor() + " " + filename

   568   os.system(command)

   569 

   570   result = ReadFile(filename)

   571   os.remove(filename)

   572 

   573   if not result:

   574     return

   575 

   576   split_result = result.split(separator1, 1)

   577   if len(split_result) != 2:

   578     ErrorExit("Don't modify the text starting with ---!\n\n" + result)

   579 

   580   new_description = split_result[0]

   581   cl_files_text = split_result[1]

   582   if new_description != description:

   583     change_info.description = new_description

   584     if change_info.issue:

   585       # Update the Rietveld issue with the new description.

   586       change_info.UpdateRietveldDescription()

   587 

   588   new_cl_files = []

   589   for line in cl_files_text.splitlines():

   590     if not len(line):

   591       continue

   592     if line.startswith("---"):

   593       break

   594     status = line[:7]

   595     file = line[7:]

   596     new_cl_files.append((status, file))

   597   change_info.files = new_cl_files

   598 

   599   change_info.Save()

   600   print change_info.name + " changelist saved."

   601 

   602 

   603 def Changes():

   604   """Print all the changlists and their files."""

   605   for cl in GetCLs():

   606     change_info = LoadChangelistInfo(cl, True, True)

   607     print "\n--- Changelist " + change_info.name + ":"

   608     for file in change_info.files:

   609       print "".join(file)

   610 

   611 

   612 def main(argv=None):

   613   if argv is None:

   614     argv = sys.argv

   615   

   616   if len(argv) == 1:

   617     Help()

   618     return 0;

   619 

   620   # Create the directory where we store information about changelists if it

   621   # doesn't exist.

   622   if not os.path.exists(GetInfoDir()):

   623     os.mkdir(GetInfoDir())

   624 

   625   command = argv[1]

   626   if command == "opened":

   627     Opened()

   628     return 0

   629   if command == "nothave":

   630     UnknownFiles(argv[2:])

   631     return 0

   632   if command == "changes":

   633     Changes()

   634     return 0

   635   if command == "diff" and len(argv) == 2:

   636     files = GetFilesNotInCL()

   637     print GenerateDiff([os.path.join(GetRepositoryRoot(), x[1]) for x in files])

   638     return 0

   639 

   640   if len(argv) == 2:

   641     if command == "change":

   642       # Generate a random changelist name.

   643       changename = GenerateChangeName()

   644     elif command == "help":

   645       Help()

   646       return 0

   647     else:

   648       ErrorExit("Need a changelist name.")

   649   else:

   650     changename = argv[2]

   651 

   652   fail_on_not_found = command != "change"

   653   change_info = LoadChangelistInfo(changename, fail_on_not_found, True)

   654 

   655   if command == "change":

   656     Change(change_info)

   657   elif command == "upload":

   658     UploadCL(change_info, argv[3:])

   659   elif command == "commit":

   660     Commit(change_info)

   661   elif command == "delete":

   662     change_info.Delete()

   663   elif command == "try":

   664     TryChange(change_info, argv[3:])

   665   else:

   666     # Everything else that is passed into gcl we redirect to svn, after adding

   667     # the files. This allows commands such as 'gcl diff xxx' to work.

   668     args =["svn", command]

   669     root = GetRepositoryRoot()

   670     args.extend([os.path.join(root, x) for x in change_info.FileList()])

   671     RunShell(args, True)

   672   return 0

   673 

   674 

   675 if __name__ == "__main__":

   676     sys.exit(main())