scripts/release/release.py
changeset 1981 8cfb054b73b2
parent 1888 ef350db7f753
equal deleted inserted replaced
1980:db7c98580008 1981:8cfb054b73b2
    17 from __future__ import with_statement
    17 from __future__ import with_statement
    18 
    18 
    19 """Google Summer of Code Melange release script.
    19 """Google Summer of Code Melange release script.
    20 
    20 
    21 This script provides automation for the various tasks involved in
    21 This script provides automation for the various tasks involved in
    22 pushing a new release of Melange to the official Google Summer of Code
    22 releasing a new version of Melange and pushing it to various app
    23 app engine instance.
    23 engine instances.
    24 
    24 
    25 It does not provide a turnkey autopilot solution. Notably, each stage
    25 It does not provide a turnkey autopilot solution. Notably, each stage
    26 of the release process must be started by a human operator, and some
    26 of the release process must be started by a human operator, and some
    27 commands will request confirmation or extra details before
    27 commands will request confirmation or extra details before
    28 proceeding. It is not a replacement for a cautious human
    28 proceeding. It is not a replacement for a cautious human operator.
    29 operator.
    29 
    30 
    30 Note that this script requires Python 2.5 or better (for various
    31 Note that this script requires:
    31 language features)
    32  - Python 2.5 or better (for various language features)
       
    33 
       
    34  - Subversion 1.5.0 or better (for working copy depth control, which
       
    35      cuts down checkout/update times by several orders of
       
    36      magnitude).
       
    37 """
    32 """
    38 
    33 
    39 __authors__ = [
    34 __authors__ = [
    40     # alphabetical order by last name, please
    35     # alphabetical order by last name, please
    41     '"David Anderson" <dave@natulte.net>',
    36     '"David Anderson" <dave@natulte.net>',
    47 import re
    42 import re
    48 import subprocess
    43 import subprocess
    49 import sys
    44 import sys
    50 
    45 
    51 import error
    46 import error
       
    47 import io
    52 import log
    48 import log
    53 import subversion
    49 import subversion
    54 import util
    50 import util
    55 
    51 
    56 
    52 
    57 # Default repository URLs for Melange and the Google release
    53 # Default repository URLs for Melange and the Google release
    58 # repository.
    54 # repository.
    59 MELANGE_REPOS = 'http://soc.googlecode.com/svn'
    55 MELANGE_REPOS = 'http://soc.googlecode.com/svn'
    60 GOOGLE_SOC_REPOS = 'https://soc-google.googlecode.com/svn'
       
    61 
    56 
    62 
    57 
    63 # Regular expression matching an apparently well formed Melange
    58 # Regular expression matching an apparently well formed Melange
    64 # release number.
    59 # release number.
    65 MELANGE_RELEASE_RE = re.compile(r'\d-\d-\d{8}p\d+')
    60 MELANGE_RELEASE_RE = re.compile(r'\d-\d-\d{8}p\d+')
    66 
    61 
    67 
    62 
    68 class Error(error.Error):
    63 class Error(error.Error):
    69   pass
    64   pass
    70 
       
    71 
       
    72 class AbortedByUser(Error):
       
    73   """The operation was aborted by the user."""
       
    74   pass
       
    75 
       
    76 
       
    77 class FileAccessError(Error):
       
    78   """An error occured while accessing a file."""
       
    79   pass
       
    80 
       
    81 
       
    82 def getString(prompt):
       
    83   """Prompt for and return a string."""
       
    84   prompt += ' '
       
    85   log.stdout.write(prompt)
       
    86   log.stdout.flush()
       
    87 
       
    88   response = sys.stdin.readline()
       
    89   log.terminal_echo(prompt + response.strip())
       
    90   if not response:
       
    91     raise AbortedByUser('Aborted by ctrl+D')
       
    92 
       
    93   return response.strip()
       
    94 
       
    95 
       
    96 def confirm(prompt, default=False):
       
    97   """Ask a yes/no question and return the answer.
       
    98 
       
    99   Will reprompt the user until one of "yes", "no", "y" or "n" is
       
   100   entered. The input is case insensitive.
       
   101 
       
   102   Args:
       
   103     prompt: The question to ask the user.
       
   104     default: The answer to return if the user just hits enter.
       
   105 
       
   106   Returns:
       
   107     True if the user answered affirmatively, False otherwise.
       
   108   """
       
   109   if default:
       
   110     question = prompt + ' [Yn]'
       
   111   else:
       
   112     question = prompt + ' [yN]'
       
   113   while True:
       
   114     answer = getString(question)
       
   115     if not answer:
       
   116       return default
       
   117     elif answer in ('y', 'yes'):
       
   118       return True
       
   119     elif answer in ('n', 'no'):
       
   120       return False
       
   121     else:
       
   122       log.error('Please answer yes or no.')
       
   123 
       
   124 
       
   125 def getNumber(prompt):
       
   126   """Prompt for and return a number.
       
   127 
       
   128   Will reprompt the user until a number is entered.
       
   129   """
       
   130   while True:
       
   131     value_str = getString(prompt)
       
   132     try:
       
   133       return int(value_str)
       
   134     except ValueError:
       
   135       log.error('Please enter a number. You entered "%s".' % value_str)
       
   136 
       
   137 
       
   138 def getChoice(intro, prompt, choices, done=None, suggest=None):
       
   139   """Prompt for and return a choice from a menu.
       
   140 
       
   141   Will reprompt the user until a valid menu entry is chosen.
       
   142 
       
   143   Args:
       
   144     intro: Text to print verbatim before the choice menu.
       
   145     prompt: The prompt to print right before accepting input.
       
   146     choices: The list of string choices to display.
       
   147     done: If not None, the list of indices of previously
       
   148       selected/completed choices.
       
   149     suggest: If not None, the index of the choice to highlight as
       
   150       the suggested choice.
       
   151 
       
   152   Returns:
       
   153     The index in the choices list of the selection the user made.
       
   154   """
       
   155   done = set(done or [])
       
   156   while True:
       
   157     print intro
       
   158     print
       
   159     for i, entry in enumerate(choices):
       
   160       done_text = ' (done)' if i in done else ''
       
   161       indent = '--> ' if i == suggest else '    '
       
   162       print '%s%2d. %s%s' % (indent, i+1, entry, done_text)
       
   163     print
       
   164     choice = getNumber(prompt)
       
   165     if 0 < choice <= len(choices):
       
   166       return choice-1
       
   167     log.error('%d is not a valid choice between %d and %d' %
       
   168               (choice, 1, len(choices)))
       
   169     print
       
   170 
       
   171 
       
   172 def fileToLines(path):
       
   173   """Read a file and return it as a list of lines."""
       
   174   try:
       
   175     with file(path) as f:
       
   176       return f.read().split('\n')
       
   177   except (IOError, OSError), e:
       
   178     raise FileAccessError(str(e))
       
   179 
       
   180 
       
   181 def linesToFile(path, lines):
       
   182   """Write a list of lines to a file."""
       
   183   try:
       
   184     with file(path, 'w') as f:
       
   185       f.write('\n'.join(lines))
       
   186   except (IOError, OSError), e:
       
   187     raise FileAccessError(str(e))
       
   188 
    65 
   189 
    66 
   190 #
    67 #
   191 # Decorators for use in ReleaseEnvironment.
    68 # Decorators for use in ReleaseEnvironment.
   192 #
    69 #
   241       self._InitializeWC()
   118       self._InitializeWC()
   242     else:
   119     else:
   243       self.wc.revert()
   120       self.wc.revert()
   244 
   121 
   245       if self.exists(self.BRANCH_FILE):
   122       if self.exists(self.BRANCH_FILE):
   246         branch = fileToLines(self.path(self.BRANCH_FILE))[0]
   123         branch = io.fileToLines(self.path(self.BRANCH_FILE))[0]
   247         self._switchBranch(branch)
   124         self._switchBranch(branch)
   248       else:
   125       else:
   249         self._switchBranch(None)
   126         self._switchBranch(None)
   250 
   127 
   251   def _InitializeWC(self):
   128   def _InitializeWC(self):
   310       self.branch_dir = None
   187       self.branch_dir = None
   311       log.info('No release branch available')
   188       log.info('No release branch available')
   312     else:
   189     else:
   313       self.wc.update()
   190       self.wc.update()
   314       assert self.wc.exists('branches/' + release)
   191       assert self.wc.exists('branches/' + release)
   315       linesToFile(self.path(self.BRANCH_FILE), [release])
   192       io.linesToFile(self.path(self.BRANCH_FILE), [release])
   316       self.branch = release
   193       self.branch = release
   317       self.branch_dir = 'branches/' + release
   194       self.branch_dir = 'branches/' + release
   318       self.wc.update(self.branch_dir, depth='infinity')
   195       self.wc.update(self.branch_dir, depth='infinity')
   319       log.info('Working on branch ' + self.branch)
   196       log.info('Working on branch ' + self.branch)
   320 
   197 
   338     branches = self._listBranches()
   215     branches = self._listBranches()
   339     if not branches:
   216     if not branches:
   340       raise error.ExpectationFailed(
   217       raise error.ExpectationFailed(
   341         'No branches available. Please import one.')
   218         'No branches available. Please import one.')
   342 
   219 
   343     choice = getChoice('Available release branches:',
   220     choice = io.getChoice('Available release branches:',
   344                'Your choice?',
   221                           'Your choice?',
   345                branches,
   222                           branches,
   346                suggest=len(branches)-1)
   223                           suggest=len(branches)-1)
   347     self._switchBranch(branches[choice])
   224     self._switchBranch(branches[choice])
   348 
   225 
   349   def _addAppYaml(self):
   226   def _addAppYaml(self):
   350     """Create a Google production app.yaml configuration.
   227     """Create a Google production app.yaml configuration.
   351 
   228 
   357       raise ObstructionError('app/app.yaml exists already')
   234       raise ObstructionError('app/app.yaml exists already')
   358 
   235 
   359     yaml_path = self._branchPath('app/app.yaml')
   236     yaml_path = self._branchPath('app/app.yaml')
   360     self.wc.copy(yaml_path + '.template', yaml_path)
   237     self.wc.copy(yaml_path + '.template', yaml_path)
   361 
   238 
   362     yaml = fileToLines(self.wc.path(yaml_path))
   239     yaml = io.fileToLines(self.wc.path(yaml_path))
   363     out = []
   240     out = []
   364     for i, line in enumerate(yaml):
   241     for i, line in enumerate(yaml):
   365       stripped_line = line.strip()
   242       stripped_line = line.strip()
   366       if 'TODO' in stripped_line:
   243       if 'TODO' in stripped_line:
   367         continue
   244         continue
   370       elif stripped_line.startswith('version:'):
   247       elif stripped_line.startswith('version:'):
   371         out.append(line.lstrip() + 'g0')
   248         out.append(line.lstrip() + 'g0')
   372         out.append('# * initial Google fork of Melange ' + self.branch)
   249         out.append('# * initial Google fork of Melange ' + self.branch)
   373       else:
   250       else:
   374         out.append(line)
   251         out.append(line)
   375     linesToFile(self.wc.path(yaml_path), out)
   252     io.linesToFile(self.wc.path(yaml_path), out)
   376 
   253 
   377     self.wc.commit('Create app.yaml with Google patch version g0 '
   254     self.wc.commit('Create app.yaml with Google patch version g0 '
   378              'in branch ' + self.branch)
   255              'in branch ' + self.branch)
   379 
   256 
   380   def _applyGooglePatches(self):
   257   def _applyGooglePatches(self):
   384     """
   261     """
   385     # Edit the base template to point users to the Google fork
   262     # Edit the base template to point users to the Google fork
   386     # of the Melange codebase instead of the vanilla release.
   263     # of the Melange codebase instead of the vanilla release.
   387     tmpl_file = self.wc.path(
   264     tmpl_file = self.wc.path(
   388       self._branchPath('app/soc/templates/soc/base.html'))
   265       self._branchPath('app/soc/templates/soc/base.html'))
   389     tmpl = fileToLines(tmpl_file)
   266     tmpl = io.fileToLines(tmpl_file)
   390     for i, line in enumerate(tmpl):
   267     for i, line in enumerate(tmpl):
   391       if 'http://code.google.com/p/soc/source/browse/tags/' in line:
   268       if 'http://code.google.com/p/soc/source/browse/tags/' in line:
   392         tmpl[i] = line.replace('/p/soc/', '/p/soc-google/')
   269         tmpl[i] = line.replace('/p/soc/', '/p/soc-google/')
   393         break
   270         break
   394     else:
   271     else:
   395       raise error.ExpectationFailed(
   272       raise error.ExpectationFailed(
   396         'No source code link found in base.html')
   273         'No source code link found in base.html')
   397     linesToFile(tmpl_file, tmpl)
   274     io.linesToFile(tmpl_file, tmpl)
   398 
   275 
   399     self.wc.commit(
   276     self.wc.commit(
   400       'Customize the Melange release link in the sidebar menu')
   277       'Customize the Melange release link in the sidebar menu')
   401 
   278 
   402   @pristine_wc
   279   @pristine_wc
   403   def importTag(self):
   280   def importTag(self):
   404     """Import a new Melange release"""
   281     """Import a new Melange release"""
   405     release = getString('Enter the Melange release to import:')
   282     release = io.getString('Enter the Melange release to import:')
   406     if not release:
   283     if not release:
   407       AbortedByUser('No release provided, import aborted')
   284       error.AbortedByUser('No release provided, import aborted')
   408 
   285 
   409     branch_dir = 'branches/' + release
   286     branch_dir = 'branches/' + release
   410     if self.wc.exists(branch_dir):
   287     if self.wc.exists(branch_dir):
   411       raise ObstructionError('Release %s already imported' % release)
   288       raise ObstructionError('Release %s already imported' % release)
   412 
   289 
   413     tag_url = '%s/tags/%s' % (self.upstream_repos, release)
   290     tag_url = '%s/tags/%s' % (self.upstream_repos, release)
   414     release_rev = subversion.find_tag_rev(tag_url)
   291     release_rev = subversion.find_tag_rev(tag_url)
   415 
   292 
   416     if confirm('Confirm import of release %s, tagged at r%d?' %
   293     if io.confirm('Confirm import of release %s, tagged at r%d?' %
   417            (release, release_rev)):
   294                   (release, release_rev)):
   418       # Add an entry to the vendor externals for the Melange
   295       # Add an entry to the vendor externals for the Melange
   419       # release.
   296       # release.
   420       externals = self.wc.propget('svn:externals', 'vendor/soc')
   297       externals = self.wc.propget('svn:externals', 'vendor/soc')
   421       externals.append('%s -r %d %s' % (release, release_rev, tag_url))
   298       externals.append('%s -r %d %s' % (release, release_rev, tag_url))
   422       self.wc.propset('svn:externals', '\n'.join(externals), 'vendor/soc')
   299       self.wc.propset('svn:externals', '\n'.join(externals), 'vendor/soc')
   441 
   318 
   442   @requires_branch
   319   @requires_branch
   443   @pristine_wc
   320   @pristine_wc
   444   def cherryPickChange(self):
   321   def cherryPickChange(self):
   445     """Cherry-pick a change from the Melange trunk"""
   322     """Cherry-pick a change from the Melange trunk"""
   446     rev = getNumber('Revision number to cherry-pick:')
   323     rev = io.getNumber('Revision number to cherry-pick:')
   447     bug = getNumber('Issue fixed by this change:')
   324     bug = io.getNumber('Issue fixed by this change:')
   448 
   325 
   449     diff = subversion.diff(self.upstream_repos + '/trunk', rev)
   326     diff = subversion.diff(self.upstream_repos + '/trunk', rev)
   450     if not diff.strip():
   327     if not diff.strip():
   451       raise error.ExpectationFailed(
   328       raise error.ExpectationFailed(
   452         'Retrieved diff is empty. '
   329         'Retrieved diff is empty. '
   455     self.wc.addRemove(self.branch_dir)
   332     self.wc.addRemove(self.branch_dir)
   456 
   333 
   457     yaml_path = self.wc.path(self._branchPath('app/app.yaml'))
   334     yaml_path = self.wc.path(self._branchPath('app/app.yaml'))
   458     out = []
   335     out = []
   459     updated_patchlevel = False
   336     updated_patchlevel = False
   460     for line in fileToLines(yaml_path):
   337     for line in io.fileToLines(yaml_path):
   461       if line.strip().startswith('version: '):
   338       if line.strip().startswith('version: '):
   462         version = line.strip().split()[-1]
   339         version = line.strip().split()[-1]
   463         base, patch = line.rsplit('g', 1)
   340         base, patch = line.rsplit('g', 1)
   464         new_version = '%sg%d' % (base, int(patch) + 1)
   341         new_version = '%sg%d' % (base, int(patch) + 1)
   465         message = ('Cherry-picked r%d from /p/soc/ to fix issue %d' %
   342         message = ('Cherry-picked r%d from /p/soc/ to fix issue %d' %
   472 
   349 
   473     if not updated_patchlevel:
   350     if not updated_patchlevel:
   474       log.error('Failed to update Google patch revision')
   351       log.error('Failed to update Google patch revision')
   475       log.error('Cherry-picking failed')
   352       log.error('Cherry-picking failed')
   476 
   353 
   477     linesToFile(yaml_path, out)
   354     io.linesToFile(yaml_path, out)
   478 
   355 
   479     log.info('Check the diff about to be committed with:')
   356     log.info('Check the diff about to be committed with:')
   480     log.info('svn diff ' + self.wc.path(self.branch_dir))
   357     log.info('svn diff ' + self.wc.path(self.branch_dir))
   481     if not confirm('Commit this change?'):
   358     if not io.confirm('Commit this change?'):
   482       raise AbortedByUser('Cherry-pick aborted')
   359       raise error.AbortedByUser('Cherry-pick aborted')
   483     self.wc.commit(message)
   360     self.wc.commit(message)
   484     log.info('Cherry-picked r%d from the Melange trunk.' % rev)
   361     log.info('Cherry-picked r%d from the Melange trunk.' % rev)
   485 
   362 
   486   MENU_ORDER = [
   363   MENU_ORDER = [
   487     update,
   364     update,
   514         last_command = None
   391         last_command = None
   515       suggested_next = self.MENU_ORDER.index(
   392       suggested_next = self.MENU_ORDER.index(
   516         self.MENU_SUGGESTIONS[last_command])
   393         self.MENU_SUGGESTIONS[last_command])
   517 
   394 
   518       try:
   395       try:
   519         choice = getChoice('Main menu:', 'Your choice?',
   396         choice = io.getChoice('Main menu:', 'Your choice?',
   520           self.MENU_STRINGS, done=done, suggest=suggested_next)
   397                               self.MENU_STRINGS, done=done,
   521       except (KeyboardInterrupt, AbortedByUser):
   398                               suggest=suggested_next)
       
   399       except (KeyboardInterrupt, error.AbortedByUser):
   522         log.info('Exiting.')
   400         log.info('Exiting.')
   523         return
   401         return
   524       try:
   402       try:
   525         self.MENU_ORDER[choice](self)
   403         self.MENU_ORDER[choice](self)
   526       except error.Error, e:
   404       except error.Error, e: