scripts/release/release.py
changeset 1849 f8728d5e2e07
parent 1847 15ad1ee02dc5
child 1888 ef350db7f753
equal deleted inserted replaced
1848:a0cae3be1412 1849:f8728d5e2e07
    47 import subprocess
    47 import subprocess
    48 import sys
    48 import sys
    49 
    49 
    50 import error
    50 import error
    51 import log
    51 import log
       
    52 import subversion
    52 import util
    53 import util
    53 
    54 
    54 
    55 
    55 # Default repository URLs for Melange and the Google release
    56 # Default repository URLs for Melange and the Google release
    56 # repository.
    57 # repository.
    67   pass
    68   pass
    68 
    69 
    69 
    70 
    70 class AbortedByUser(Error):
    71 class AbortedByUser(Error):
    71   """The operation was aborted by the user."""
    72   """The operation was aborted by the user."""
    72 
       
    73 
       
    74 class ObstructionError(Error):
       
    75   """An operation was obstructed by existing data."""
       
    76 
       
    77 
       
    78 class ExpectationFailed(Error):
       
    79   """An unexpected state was encountered by an automated step."""
       
    80 
    73 
    81 
    74 
    82 class FileAccessError(Error):
    75 class FileAccessError(Error):
    83   """An error occured while accessing a file."""
    76   """An error occured while accessing a file."""
    84 
    77 
   189       f.write('\n'.join(lines))
   182       f.write('\n'.join(lines))
   190   except (IOError, OSError), e:
   183   except (IOError, OSError), e:
   191     raise FileAccessError(str(e))
   184     raise FileAccessError(str(e))
   192 
   185 
   193 
   186 
   194 class Subversion(util.Paths):
       
   195   """Wrapper for operations on a Subversion working copy.
       
   196 
       
   197   An instance of this class is bound to a specific working copy
       
   198   directory, and provides an API to perform various Subversion
       
   199   operations on this working copy.
       
   200 
       
   201   Some methods take a 'depth' argument. Depth in Subversion is a
       
   202   feature that allows the creation of arbitrarily shallow or deep
       
   203   working copies on a per-directory basis. Possible values are
       
   204   'none' (no files or directories), 'files' (only files in .),
       
   205   'immediates' (files and directories in ., directories checked out
       
   206   at depth 'none') or 'infinity' (a normal working copy with
       
   207   everything).
       
   208 
       
   209   This class also provides a few static functions that run the 'svn'
       
   210   tool against remote repositories to gather information or retrieve
       
   211   data.
       
   212 
       
   213   Note that this wrapper also doubles as a Paths object, offering an
       
   214   easy way to get or check the existence of paths in the working
       
   215   copy.
       
   216   """
       
   217 
       
   218   def __init__(self, wc_dir):
       
   219     util.Paths.__init__(self, wc_dir)
       
   220 
       
   221   def _unknownAndMissing(self, path):
       
   222     """Returns lists of unknown and missing files in the working copy.
       
   223 
       
   224     Args:
       
   225       path: The working copy path to scan.
       
   226 
       
   227     Returns:
       
   228 
       
   229       Two lists. The first is a list of all unknown paths
       
   230       (subversion has no knowledge of them), the second is a list
       
   231       of missing paths (subversion knows about them, but can't
       
   232       find them). Paths in either list are relative to the input
       
   233       path.
       
   234     """
       
   235     assert self.exists()
       
   236     unknown = []
       
   237     missing = []
       
   238     for line in self.status(path):
       
   239       if not line.strip():
       
   240         continue
       
   241       if line[0] == '?':
       
   242         unknown.append(line[7:])
       
   243       elif line[0] == '!':
       
   244         missing.append(line[7:])
       
   245     return unknown, missing
       
   246 
       
   247   def checkout(self, url, depth='infinity'):
       
   248     """Check out a working copy from the given URL.
       
   249 
       
   250     Args:
       
   251       url: The Subversion repository URL to check out.
       
   252       depth: The depth of the working copy root.
       
   253     """
       
   254     assert not self.exists()
       
   255     util.run(['svn', 'checkout', '--depth=' + depth, url, self.path()])
       
   256 
       
   257   def update(self, path='', depth=None):
       
   258     """Update a working copy path, optionally changing depth.
       
   259 
       
   260     Args:
       
   261       path: The working copy path to update.
       
   262       depth: If set, change the depth of the path before updating.
       
   263     """
       
   264     assert self.exists()
       
   265     if depth is None:
       
   266       util.run(['svn', 'update', self.path(path)])
       
   267     else:
       
   268       util.run(['svn', 'update', '--set-depth=' + depth, self.path(path)])
       
   269 
       
   270   def revert(self, path=''):
       
   271     """Recursively revert a working copy path.
       
   272 
       
   273     Note that this command is more zealous than the 'svn revert'
       
   274     command, as it will also delete any files which subversion
       
   275     does not know about.
       
   276     """
       
   277     util.run(['svn', 'revert', '-R', self.path(path)])
       
   278 
       
   279     unknown, missing = self._unknownAndMissing(path)
       
   280     unknown = [os.path.join(self.path(path), p) for p in unknown]
       
   281 
       
   282     if unknown:
       
   283       # rm -rf makes me uneasy. Verify that all paths to be deleted
       
   284       # are within the release working copy.
       
   285       for p in unknown:
       
   286         assert p.startswith(self.path())
       
   287 
       
   288       util.run(['rm', '-rf', '--'] + unknown)
       
   289 
       
   290   def ls(self, dir=''):
       
   291     """List the contents of a working copy directory.
       
   292 
       
   293     Note that this returns the contents of the directory as seen
       
   294     by the server, not constrained by the depth settings of the
       
   295     local path.
       
   296     """
       
   297     assert self.exists()
       
   298     return util.run(['svn', 'ls', self.path(dir)], capture=True)
       
   299 
       
   300   def copy(self, src, dest):
       
   301     """Copy a working copy path.
       
   302 
       
   303     The copy is only scheduled for commit, not committed.
       
   304 
       
   305     Args:
       
   306       src: The source working copy path.
       
   307       dst: The destination working copy path.
       
   308     """
       
   309     assert self.exists()
       
   310     util.run(['svn', 'cp', self.path(src), self.path(dest)])
       
   311 
       
   312   def propget(self, prop_name, path):
       
   313     """Get the value of a property on a working copy path.
       
   314 
       
   315     Args:
       
   316       prop_name: The property name, eg. 'svn:externals'.
       
   317       path: The working copy path on which the property is set.
       
   318     """
       
   319     assert self.exists()
       
   320     return util.run(['svn', 'propget', prop_name, self.path(path)],
       
   321             capture=True)
       
   322 
       
   323   def propset(self, prop_name, prop_value, path):
       
   324     """Set the value of a property on a working copy path.
       
   325 
       
   326     The property change is only scheduled for commit, not committed.
       
   327 
       
   328     Args:
       
   329       prop_name: The property name, eg. 'svn:externals'.
       
   330       prop_value: The value that should be set.
       
   331       path: The working copy path on which to set the property.
       
   332     """
       
   333     assert self.exists()
       
   334     util.run(['svn', 'propset', prop_name, prop_value, self.path(path)])
       
   335 
       
   336   def add(self, paths):
       
   337     """Schedule working copy paths for addition.
       
   338 
       
   339     The paths are only scheduled for addition, not committed.
       
   340 
       
   341     Args:
       
   342       paths: The list of working copy paths to add.
       
   343     """
       
   344     assert self.exists()
       
   345     paths = [self.path(p) for p in paths]
       
   346     util.run(['svn', 'add'] + paths)
       
   347 
       
   348   def remove(self, paths):
       
   349     """Schedule working copy paths for deletion.
       
   350 
       
   351     The paths are only scheduled for deletion, not committed.
       
   352 
       
   353     Args:
       
   354       paths: The list of working copy paths to delete.
       
   355     """
       
   356     assert self.exists()
       
   357     paths = [self.path(p) for p in paths]
       
   358     util.run(['svn', 'rm'] + paths)
       
   359 
       
   360   def status(self, path=''):
       
   361     """Return the status of a working copy path.
       
   362 
       
   363     The status returned is the verbatim output of 'svn status' on
       
   364     the path.
       
   365 
       
   366     Args:
       
   367       path: The path to examine.
       
   368     """
       
   369     assert self.exists()
       
   370     return util.run(['svn', 'status', self.path(path)], capture=True)
       
   371 
       
   372   def addRemove(self, path=''):
       
   373     """Perform an "addremove" operation a working copy path.
       
   374 
       
   375     An "addremove" runs 'svn status' and schedules all the unknown
       
   376     paths (listed as '?') for addition, and all the missing paths
       
   377     (listed as '!') for deletion. Its main use is to synchronize
       
   378     working copy state after applying a patch in unified diff
       
   379     format.
       
   380 
       
   381     Args:
       
   382       path: The path under which unknown/missing files should be
       
   383         added/removed.
       
   384     """
       
   385     assert self.exists()
       
   386     unknown, missing = self._unknownAndMissing(path)
       
   387     if unknown:
       
   388       self.add(unknown)
       
   389     if missing:
       
   390       self.remove(missing)
       
   391 
       
   392   def commit(self, message, path=''):
       
   393     """Commit scheduled changes to the source repository.
       
   394 
       
   395     Args:
       
   396       message: The commit message to use.
       
   397       path: The path to commit.
       
   398     """
       
   399     assert self.exists()
       
   400     util.run(['svn', 'commit', '-m', message, self.path(path)])
       
   401 
       
   402   @staticmethod
       
   403   def export(url, revision, dest_path):
       
   404     """Export the contents of a repository to a local path.
       
   405 
       
   406     Note that while the underlying 'svn export' only requires a
       
   407     URL, we require that both a URL and a revision be specified,
       
   408     to fully qualify the data to export.
       
   409 
       
   410     Args:
       
   411       url: The repository URL to export.
       
   412       revision: The revision to export.
       
   413       dest_path: The destination directory for the export. Note
       
   414            that this is an absolute path, NOT a working copy
       
   415            relative path.
       
   416     """
       
   417     assert os.path.isabs(dest_path)
       
   418     if os.path.exists(dest_path):
       
   419       raise ObstructionError('Cannot export to obstructed path %s' %
       
   420                    dest_path)
       
   421     util.run(['svn', 'export', '-r', str(revision), url, dest_path])
       
   422 
       
   423   @staticmethod
       
   424   def find_tag_rev(url):
       
   425     """Return the revision at which a remote tag was created.
       
   426 
       
   427     Since tags are immutable by convention, usually the HEAD of a
       
   428     tag should be the tag creation revision. However, mistakes can
       
   429     happen, so this function will walk the history of the given
       
   430     tag URL, stopping on the first revision that was created by
       
   431     copy.
       
   432 
       
   433     This detection is not foolproof. For example: it will be
       
   434     fooled by a tag that was created, deleted, and recreated by
       
   435     copy at a different revision. It is not clear what the desired
       
   436     behavior in these edge cases are, and no attempt is made to
       
   437     handle them. You should request user confirmation before using
       
   438     the result of this function.
       
   439 
       
   440     Args:
       
   441       url: The repository URL of the tag to examine.
       
   442     """
       
   443     try:
       
   444       output = util.run(['svn', 'log', '-q', '--stop-on-copy', url],
       
   445                 capture=True)
       
   446     except util.SubprocessFailed:
       
   447       raise ExpectationFailed('No tag at URL ' + url)
       
   448     first_rev_line = output[-2]
       
   449     first_rev = int(first_rev_line.split()[0][1:])
       
   450     return first_rev
       
   451 
       
   452   @staticmethod
       
   453   def diff(url, revision):
       
   454     """Retrieve a revision from a remote repository as a unified diff.
       
   455 
       
   456     Args:
       
   457       url: The repository URL on which to perform the diff.
       
   458       revision: The revision to extract at the given url.
       
   459 
       
   460     Returns:
       
   461       A string containing the changes extracted from the remote
       
   462       repository, in unified diff format suitable for application
       
   463       using 'patch'.
       
   464     """
       
   465     try:
       
   466       return util.run(['svn', 'diff', '-c', str(revision), url],
       
   467               capture=True, split_capture=False)
       
   468     except util.SubprocessFailed:
       
   469       raise ExpectationFailed('Could not get diff for r%d '
       
   470                   'from remote repository' % revision)
       
   471 
       
   472 
       
   473 #
   187 #
   474 # Decorators for use in ReleaseEnvironment.
   188 # Decorators for use in ReleaseEnvironment.
   475 #
   189 #
   476 def pristine_wc(f):
   190 def pristine_wc(f):
   477   """A decorator that cleans up the release repository."""
   191   """A decorator that cleans up the release repository."""
   485 def requires_branch(f):
   199 def requires_branch(f):
   486   """A decorator that checks that a release branch is active."""
   200   """A decorator that checks that a release branch is active."""
   487   @functools.wraps(f)
   201   @functools.wraps(f)
   488   def check_branch(self, *args, **kwargs):
   202   def check_branch(self, *args, **kwargs):
   489     if self.branch is None:
   203     if self.branch is None:
   490       raise ExpectationFailed(
   204       raise error.ExpectationFailed(
   491         'This operation requires an active release branch')
   205         'This operation requires an active release branch')
   492     return f(self, *args, **kwargs)
   206     return f(self, *args, **kwargs)
   493   return check_branch
   207   return check_branch
   494 
   208 
   495 
   209 
   514       root: The root of the release environment.
   228       root: The root of the release environment.
   515       release_repos: The URL to the Google release repository root.
   229       release_repos: The URL to the Google release repository root.
   516       upstream_repos: The URL to the Melange upstream repository root.
   230       upstream_repos: The URL to the Melange upstream repository root.
   517     """
   231     """
   518     util.Paths.__init__(self, root)
   232     util.Paths.__init__(self, root)
   519     self.wc = Subversion(self.path('google-soc'))
   233     self.wc = subversion.WorkingCopy(self.path('google-soc'))
   520     self.release_repos = release_repos.strip('/')
   234     self.release_repos = release_repos.strip('/')
   521     self.upstream_repos = upstream_repos.strip('/')
   235     self.upstream_repos = upstream_repos.strip('/')
   522 
   236 
   523     if not self.wc.exists():
   237     if not self.wc.exists():
   524       self._InitializeWC()
   238       self._InitializeWC()
   619   @pristine_wc
   333   @pristine_wc
   620   def switchToBranch(self):
   334   def switchToBranch(self):
   621     """Switch to another Melange release branch"""
   335     """Switch to another Melange release branch"""
   622     branches = self._listBranches()
   336     branches = self._listBranches()
   623     if not branches:
   337     if not branches:
   624       raise ExpectationFailed(
   338       raise error.ExpectationFailed(
   625         'No branches available. Please import one.')
   339         'No branches available. Please import one.')
   626 
   340 
   627     choice = getChoice('Available release branches:',
   341     choice = getChoice('Available release branches:',
   628                'Your choice?',
   342                'Your choice?',
   629                branches,
   343                branches,
   675     for i, line in enumerate(tmpl):
   389     for i, line in enumerate(tmpl):
   676       if 'http://code.google.com/p/soc/source/browse/tags/' in line:
   390       if 'http://code.google.com/p/soc/source/browse/tags/' in line:
   677         tmpl[i] = line.replace('/p/soc/', '/p/soc-google/')
   391         tmpl[i] = line.replace('/p/soc/', '/p/soc-google/')
   678         break
   392         break
   679     else:
   393     else:
   680       raise ExpectationFailed(
   394       raise error.ExpectationFailed(
   681         'No source code link found in base.html')
   395         'No source code link found in base.html')
   682     linesToFile(tmpl_file, tmpl)
   396     linesToFile(tmpl_file, tmpl)
   683 
   397 
   684     self.wc.commit(
   398     self.wc.commit(
   685       'Customize the Melange release link in the sidebar menu')
   399       'Customize the Melange release link in the sidebar menu')
   694     branch_dir = 'branches/' + release
   408     branch_dir = 'branches/' + release
   695     if self.wc.exists(branch_dir):
   409     if self.wc.exists(branch_dir):
   696       raise ObstructionError('Release %s already imported' % release)
   410       raise ObstructionError('Release %s already imported' % release)
   697 
   411 
   698     tag_url = '%s/tags/%s' % (self.upstream_repos, release)
   412     tag_url = '%s/tags/%s' % (self.upstream_repos, release)
   699     release_rev = Subversion.find_tag_rev(tag_url)
   413     release_rev = subversion.find_tag_rev(tag_url)
   700 
   414 
   701     if confirm('Confirm import of release %s, tagged at r%d?' %
   415     if confirm('Confirm import of release %s, tagged at r%d?' %
   702            (release, release_rev)):
   416            (release, release_rev)):
   703       # Add an entry to the vendor externals for the Melange
   417       # Add an entry to the vendor externals for the Melange
   704       # release.
   418       # release.
   707       self.wc.propset('svn:externals', '\n'.join(externals), 'vendor/soc')
   421       self.wc.propset('svn:externals', '\n'.join(externals), 'vendor/soc')
   708       self.wc.commit('Add svn:externals entry to pull in Melange '
   422       self.wc.commit('Add svn:externals entry to pull in Melange '
   709                'release %s at r%d.' % (release, release_rev))
   423                'release %s at r%d.' % (release, release_rev))
   710 
   424 
   711       # Export the tag into the release repository's branches
   425       # Export the tag into the release repository's branches
   712       Subversion.export(tag_url, release_rev, self.wc.path(branch_dir))
   426       subversion.export(tag_url, release_rev, self.wc.path(branch_dir))
   713 
   427 
   714       # Add and commit the branch add (very long operation!)
   428       # Add and commit the branch add (very long operation!)
   715       self.wc.add([branch_dir])
   429       self.wc.add([branch_dir])
   716       self.wc.commit('Branch of Melange release %s' % release,
   430       self.wc.commit('Branch of Melange release %s' % release,
   717                branch_dir)
   431                branch_dir)
   730   def cherryPickChange(self):
   444   def cherryPickChange(self):
   731     """Cherry-pick a change from the Melange trunk"""
   445     """Cherry-pick a change from the Melange trunk"""
   732     rev = getNumber('Revision number to cherry-pick:')
   446     rev = getNumber('Revision number to cherry-pick:')
   733     bug = getNumber('Issue fixed by this change:')
   447     bug = getNumber('Issue fixed by this change:')
   734 
   448 
   735     diff = self.wc.diff(self.upstream_repos + '/trunk', rev)
   449     diff = subversion.diff(self.upstream_repos + '/trunk', rev)
   736     if not diff.strip():
   450     if not diff.strip():
   737       raise ExpectationFailed(
   451       raise error.ExpectationFailed(
   738         'Retrieved diff is empty. '
   452         'Retrieved diff is empty. '
   739         'Did you accidentally cherry-pick a branch change?')
   453         'Did you accidentally cherry-pick a branch change?')
   740     util.run(['patch', '-p0'], cwd=self.wc.path(self.branch_dir),
   454     util.run(['patch', '-p0'], cwd=self.wc.path(self.branch_dir),
   741          stdin=diff)
   455          stdin=diff)
   742     self.wc.addRemove(self.branch_dir)
   456     self.wc.addRemove(self.branch_dir)
   809       except (KeyboardInterrupt, AbortedByUser):
   523       except (KeyboardInterrupt, AbortedByUser):
   810         log.info('Exiting.')
   524         log.info('Exiting.')
   811         return
   525         return
   812       try:
   526       try:
   813         self.MENU_ORDER[choice](self)
   527         self.MENU_ORDER[choice](self)
   814       except Error, e:
   528       except error.Error, e:
   815         log.error(str(e))
   529         log.error(str(e))
   816       else:
   530       else:
   817         done.append(choice)
   531         done.append(choice)
   818         last_choice = choice
   532         last_choice = choice
   819 
   533