1 ##############################################################################

     2 #

     3 # Copyright (c) 2005-2009 Zope Corporation and Contributors.

     4 # All Rights Reserved.

     5 #

     6 # This software is subject to the provisions of the Zope Public License,

     7 # Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.

     8 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED

     9 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

    10 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS

    11 # FOR A PARTICULAR PURPOSE.

    12 #

    13 ##############################################################################

    14 """Buildout main script

    15 """

    16 

    17 from rmtree import rmtree

    18 try:

    19     from hashlib import md5

    20 except ImportError:

    21     # Python 2.4 and older

    22     from md5 import md5

    23 

    24 import ConfigParser

    25 import copy

    26 import distutils.errors

    27 import glob

    28 import itertools

    29 import logging

    30 import os

    31 import pkg_resources

    32 import re

    33 import shutil

    34 import sys

    35 import tempfile

    36 import UserDict

    37 import warnings

    38 import zc.buildout

    39 import zc.buildout.download

    40 import zc.buildout.easy_install

    41 

    42 

    43 realpath = zc.buildout.easy_install.realpath

    44 

    45 pkg_resources_loc = pkg_resources.working_set.find(

    46     pkg_resources.Requirement.parse('setuptools')).location

    47 

    48 _isurl = re.compile('([a-zA-Z0-9+.-]+)://').match

    49 

    50 is_jython = sys.platform.startswith('java')

    51 

    52 if is_jython:

    53     import subprocess

    54 

    55 _sys_executable_has_broken_dash_S = (

    56     zc.buildout.easy_install._has_broken_dash_S(sys.executable))

    57 

    58 class MissingOption(zc.buildout.UserError, KeyError):

    59     """A required option was missing.

    60     """

    61 

    62 class MissingSection(zc.buildout.UserError, KeyError):

    63     """A required section is missing.

    64     """

    65 

    66     def __str__(self):

    67         return "The referenced section, %r, was not defined." % self[0]

    68 

    69 

    70 def _annotate_section(section, note):

    71     for key in section:

    72         section[key] = (section[key], note)

    73     return section

    74 

    75 def _annotate(data, note):

    76     for key in data:

    77         data[key] = _annotate_section(data[key], note)

    78     return data

    79 

    80 def _print_annotate(data):

    81     sections = data.keys()

    82     sections.sort()

    83     print

    84     print "Annotated sections"

    85     print "="*len("Annotated sections")

    86     for section in sections:

    87         print

    88         print '[%s]' % section

    89         keys = data[section].keys()

    90         keys.sort()

    91         for key in keys:

    92             value, notes = data[section][key]

    93             keyvalue = "%s= %s" % (key, value)

    94             print keyvalue

    95             line = '   '

    96             for note in notes.split():

    97                 if note == '[+]':

    98                     line = '+= '

    99                 elif note == '[-]':

   100                     line = '-= '

   101                 else:

   102                     print line, note

   103                     line = '   '

   104     print

   105 

   106 

   107 def _unannotate_section(section):

   108     for key in section:

   109         value, note = section[key]

   110         section[key] = value

   111     return section

   112 

   113 def _unannotate(data):

   114     for key in data:

   115         data[key] = _unannotate_section(data[key])

   116     return data

   117 

   118 _buildout_default_options = _annotate_section({

   119     'accept-buildout-test-releases': 'false',

   120     'allow-hosts': '*',

   121     'allow-picked-versions': 'true',

   122     'allowed-eggs-from-site-packages': '*',

   123     'bin-directory': 'bin',

   124     'develop-eggs-directory': 'develop-eggs',

   125     'eggs-directory': 'eggs',

   126     'executable': sys.executable,

   127     'exec-sitecustomize': 'true',

   128     'find-links': '',

   129     'include-site-packages': 'true',

   130     'install-from-cache': 'false',

   131     'installed': '.installed.cfg',

   132     'log-format': '',

   133     'log-level': 'INFO',

   134     'newest': 'true',

   135     'offline': 'false',

   136     'parts-directory': 'parts',

   137     'prefer-final': 'false',

   138     'python': 'buildout',

   139     'relative-paths': 'false',

   140     'socket-timeout': '',

   141     'unzip': 'false',

   142     'use-dependency-links': 'true',

   143     }, 'DEFAULT_VALUE')

   144 

   145 

   146 class Buildout(UserDict.DictMixin):

   147 

   148     def __init__(self, config_file, cloptions,

   149                  user_defaults=True, windows_restart=False, command=None):

   150 

   151         __doing__ = 'Initializing.'

   152 

   153         self.__windows_restart = windows_restart

   154 

   155         # default options

   156         data = dict(buildout=_buildout_default_options.copy())

   157         self._buildout_dir = os.getcwd()

   158 

   159         if not _isurl(config_file):

   160             config_file = os.path.abspath(config_file)

   161             base = os.path.dirname(config_file)

   162             if not os.path.exists(config_file):

   163                 if command == 'init':

   164                     print 'Creating %r.' % config_file

   165                     open(config_file, 'w').write('[buildout]\nparts = \n')

   166                 elif command == 'setup':

   167                     # Sigh. This model of a buildout instance

   168                     # with methods is breaking down. :(

   169                     config_file = None

   170                     data['buildout']['directory'] = ('.', 'COMPUTED_VALUE')

   171                 else:

   172                     raise zc.buildout.UserError(

   173                         "Couldn't open %s" % config_file)

   174 

   175             if config_file:

   176                 data['buildout']['directory'] = (os.path.dirname(config_file),

   177                     'COMPUTED_VALUE')

   178         else:

   179             base = None

   180 

   181 

   182         cloptions = dict(

   183             (section, dict((option, (value, 'COMMAND_LINE_VALUE'))

   184                            for (_, option, value) in v))

   185             for (section, v) in itertools.groupby(sorted(cloptions),

   186                                                   lambda v: v[0])

   187             )

   188         override = cloptions.get('buildout', {}).copy()

   189 

   190         # load user defaults, which override defaults

   191         if user_defaults:

   192             user_config = os.path.join(os.path.expanduser('~'),

   193                                        '.buildout', 'default.cfg')

   194             if os.path.exists(user_config):

   195                 _update(data, _open(os.path.dirname(user_config), user_config,

   196                                     [], data['buildout'].copy(), override))

   197 

   198         # load configuration files

   199         if config_file:

   200             _update(data, _open(os.path.dirname(config_file), config_file, [],

   201                                 data['buildout'].copy(), override))

   202 

   203         # apply command-line options

   204         _update(data, cloptions)

   205 

   206         self._annotated = copy.deepcopy(data)

   207         self._raw = _unannotate(data)

   208         self._data = {}

   209         self._parts = []

   210         # provide some defaults before options are parsed

   211         # because while parsing options those attributes might be

   212         # used already (Gottfried Ganssauge)

   213         buildout_section = data['buildout']

   214 

   215         # Try to make sure we have absolute paths for standard

   216         # directories. We do this before doing substitutions, in case

   217         # a one of these gets read by another section.  If any

   218         # variable references are used though, we leave it as is in

   219         # _buildout_path.

   220         if 'directory' in buildout_section:

   221             self._buildout_dir = buildout_section['directory']

   222             for name in ('bin', 'parts', 'eggs', 'develop-eggs'):

   223                 d = self._buildout_path(buildout_section[name+'-directory'])

   224                 buildout_section[name+'-directory'] = d

   225 

   226         # Attributes on this buildout object shouldn't be used by

   227         # recipes in their __init__.  It can cause bugs, because the

   228         # recipes will be instantiated below (``options = self['buildout']``)

   229         # before this has completed initializing.  These attributes are

   230         # left behind for legacy support but recipe authors should

   231         # beware of using them.  A better practice is for a recipe to

   232         # use the buildout['buildout'] options.

   233         links = buildout_section['find-links']

   234         self._links = links and links.split() or ()

   235         allow_hosts = buildout_section['allow-hosts'].split('\n')

   236         self._allow_hosts = tuple([host.strip() for host in allow_hosts

   237                                    if host.strip() != ''])

   238         self._logger = logging.getLogger('zc.buildout')

   239         self.offline = (buildout_section['offline'] == 'true')

   240         self.newest = (buildout_section['newest'] == 'true')

   241         self.accept_buildout_test_releases = (

   242             buildout_section['accept-buildout-test-releases'] == 'true')

   243 

   244         ##################################################################

   245         ## WARNING!!!

   246         ## ALL ATTRIBUTES MUST HAVE REASONABLE DEFAULTS AT THIS POINT

   247         ## OTHERWISE ATTRIBUTEERRORS MIGHT HAPPEN ANY TIME FROM RECIPES.

   248         ## RECIPES SHOULD GENERALLY USE buildout['buildout'] OPTIONS, NOT

   249         ## BUILDOUT ATTRIBUTES.

   250         ##################################################################

   251         # initialize some attrs and buildout directories.

   252         options = self['buildout']

   253 

   254         # now reinitialize

   255         links = options.get('find-links', '')

   256         self._links = links and links.split() or ()

   257 

   258         allow_hosts = options['allow-hosts'].split('\n')

   259         self._allow_hosts = tuple([host.strip() for host in allow_hosts

   260                                    if host.strip() != ''])

   261 

   262         self._buildout_dir = options['directory']

   263 

   264         # Make sure we have absolute paths for standard directories.  We do this

   265         # a second time here in case someone overrode these in their configs.

   266         for name in ('bin', 'parts', 'eggs', 'develop-eggs'):

   267             d = self._buildout_path(options[name+'-directory'])

   268             options[name+'-directory'] = d

   269 

   270         if options['installed']:

   271             options['installed'] = os.path.join(options['directory'],

   272                                                 options['installed'])

   273 

   274         self._setup_logging()

   275 

   276         versions = options.get('versions')

   277         if versions:

   278             zc.buildout.easy_install.default_versions(dict(self[versions]))

   279 

   280 

   281         self.offline = options.get_bool('offline')

   282         if self.offline:

   283             options['newest'] = 'false'

   284         self.newest = options.get_bool('newest')

   285         zc.buildout.easy_install.prefer_final(

   286             options.get_bool('prefer-final'))

   287         self.accept_buildout_test_releases = options.get_bool(

   288             'accept-buildout-test-releases')

   289         zc.buildout.easy_install.use_dependency_links(

   290             options.get_bool('use-dependency-links'))

   291         zc.buildout.easy_install.allow_picked_versions(

   292             options.get_bool('allow-picked-versions'))

   293         zc.buildout.easy_install.install_from_cache(

   294             options.get_bool('install-from-cache'))

   295         zc.buildout.easy_install.always_unzip(options.get_bool('unzip'))

   296         allowed_eggs = tuple(name.strip() for name in options[

   297             'allowed-eggs-from-site-packages'].split('\n'))

   298         self.include_site_packages = options.get_bool('include-site-packages')

   299         self.exec_sitecustomize = options.get_bool('exec-sitecustomize')

   300         if (_sys_executable_has_broken_dash_S and

   301             (not self.include_site_packages or allowed_eggs != ('*',))):

   302             # We can't do this if the executable has a broken -S.

   303             warnings.warn(zc.buildout.easy_install.BROKEN_DASH_S_WARNING)

   304             self.include_site_packages = True

   305         zc.buildout.easy_install.allowed_eggs_from_site_packages(allowed_eggs)

   306         zc.buildout.easy_install.include_site_packages(

   307             self.include_site_packages)

   308 

   309         download_cache = options.get('download-cache')

   310         if download_cache:

   311             download_cache = os.path.join(options['directory'], download_cache)

   312             if not os.path.isdir(download_cache):

   313                 raise zc.buildout.UserError(

   314                     'The specified download cache:\n'

   315                     '%r\n'

   316                     "Doesn't exist.\n"

   317                     % download_cache)

   318             download_cache = os.path.join(download_cache, 'dist')

   319             if not os.path.isdir(download_cache):

   320                 os.mkdir(download_cache)

   321 

   322             zc.buildout.easy_install.download_cache(download_cache)

   323 

   324         # "Use" each of the defaults so they aren't reported as unused options.

   325         for name in _buildout_default_options:

   326             options[name]

   327 

   328         # Do the same for extends-cache which is not among the defaults but

   329         # wasn't recognized as having been used since it was used before

   330         # tracking was turned on.

   331         options.get('extends-cache')

   332 

   333         os.chdir(options['directory'])

   334 

   335     def _buildout_path(self, name):

   336         if '${' in name:

   337             return name

   338         return os.path.join(self._buildout_dir, name)

   339 

   340     def bootstrap(self, args):

   341         __doing__ = 'Bootstrapping.'

   342 

   343         self._setup_directories()

   344 

   345         options = self['buildout']

   346 

   347         # Get a base working set for our distributions that corresponds to the

   348         # stated desires in the configuration.

   349         distributions = ['setuptools', 'zc.buildout']

   350         if options.get('offline') == 'true':

   351             ws = zc.buildout.easy_install.working_set(

   352                 distributions, options['executable'],

   353                 [options['develop-eggs-directory'],

   354                  options['eggs-directory']],

   355                 prefer_final=not self.accept_buildout_test_releases,

   356                 )

   357         else:

   358             ws = zc.buildout.easy_install.install(

   359                 distributions, options['eggs-directory'],

   360                 links=self._links,

   361                 index=options.get('index'),

   362                 executable=options['executable'],

   363                 path=[options['develop-eggs-directory']],

   364                 newest=self.newest,

   365                 allow_hosts=self._allow_hosts,

   366                 prefer_final=not self.accept_buildout_test_releases,

   367                 )

   368 

   369         # Now copy buildout and setuptools eggs, and record destination eggs:

   370         entries = []

   371         for name in 'setuptools', 'zc.buildout':

   372             r = pkg_resources.Requirement.parse(name)

   373             dist = ws.find(r)

   374             if dist.precedence == pkg_resources.DEVELOP_DIST:

   375                 dest = os.path.join(self['buildout']['develop-eggs-directory'],

   376                                     name+'.egg-link')

   377                 open(dest, 'w').write(dist.location)

   378                 entries.append(dist.location)

   379             else:

   380                 dest = os.path.join(self['buildout']['eggs-directory'],

   381                                     os.path.basename(dist.location))

   382                 entries.append(dest)

   383                 if not os.path.exists(dest):

   384                     if os.path.isdir(dist.location):

   385                         shutil.copytree(dist.location, dest)

   386                     else:

   387                         shutil.copy2(dist.location, dest)

   388 

   389         # Create buildout script.

   390         # Ideally the (possibly) new version of buildout would get a

   391         # chance to write the script.  Not sure how to do that.

   392         ws = pkg_resources.WorkingSet(entries)

   393         ws.require('zc.buildout')

   394         partsdir = os.path.join(options['parts-directory'], 'buildout')

   395         if not os.path.exists(partsdir):

   396             os.mkdir(partsdir)

   397         # (Honor the relative-paths option.)

   398         relative_paths = options.get('relative-paths', 'false')

   399         if relative_paths == 'true':

   400             relative_paths = options['directory']

   401         else:

   402             assert relative_paths == 'false'

   403             relative_paths = ''

   404         if (self.accept_buildout_test_releases and

   405             self._annotated['buildout']['accept-buildout-test-releases'][1] ==

   406             'COMMAND_LINE_VALUE'):

   407             # Bootstrap was called with '--accept-buildout-test-releases'.

   408             # Continue to honor that setting.

   409             script_initialization = _early_release_initialization_code

   410         else:

   411             script_initialization = ''

   412         zc.buildout.easy_install.sitepackage_safe_scripts(

   413             options['bin-directory'], ws, options['executable'], partsdir,

   414             reqs=['zc.buildout'], relative_paths=relative_paths,

   415             include_site_packages=self.include_site_packages,

   416             script_initialization=script_initialization,

   417             exec_sitecustomize=self.exec_sitecustomize,

   418             )

   419 

   420     init = bootstrap

   421 

   422     def install(self, install_args):

   423         __doing__ = 'Installing.'

   424 

   425         self._load_extensions()

   426         self._setup_directories()

   427 

   428         # Add develop-eggs directory to path so that it gets searched

   429         # for eggs:

   430         sys.path.insert(0, self['buildout']['develop-eggs-directory'])

   431 

   432         # Check for updates. This could cause the process to be restarted.

   433         self._maybe_upgrade()

   434 

   435         # load installed data

   436         (installed_part_options, installed_exists

   437          )= self._read_installed_part_options()

   438 

   439         # Remove old develop eggs

   440         self._uninstall(

   441             installed_part_options['buildout'].get(

   442                 'installed_develop_eggs', '')

   443             )

   444 

   445         # Build develop eggs

   446         installed_develop_eggs = self._develop()

   447         installed_part_options['buildout']['installed_develop_eggs'

   448                                            ] = installed_develop_eggs

   449 

   450         if installed_exists:

   451             self._update_installed(

   452                 installed_develop_eggs=installed_develop_eggs)

   453 

   454         # get configured and installed part lists

   455         conf_parts = self['buildout']['parts']

   456         conf_parts = conf_parts and conf_parts.split() or []

   457         installed_parts = installed_part_options['buildout']['parts']

   458         installed_parts = installed_parts and installed_parts.split() or []

   459 

   460         if install_args:

   461             install_parts = install_args

   462             uninstall_missing = False

   463         else:

   464             install_parts = conf_parts

   465             uninstall_missing = True

   466 

   467         # load and initialize recipes

   468         [self[part]['recipe'] for part in install_parts]

   469         if not install_args:

   470             install_parts = self._parts

   471 

   472         if self._log_level < logging.DEBUG:

   473             sections = list(self)

   474             sections.sort()

   475             print

   476             print 'Configuration data:'

   477             for section in self._data:

   478                 _save_options(section, self[section], sys.stdout)

   479             print

   480 

   481 

   482         # compute new part recipe signatures

   483         self._compute_part_signatures(install_parts)

   484 

   485         # uninstall parts that are no-longer used or whose configs

   486         # have changed

   487         for part in reversed(installed_parts):

   488             if part in install_parts:

   489                 old_options = installed_part_options[part].copy()

   490                 installed_files = old_options.pop('__buildout_installed__')

   491                 new_options = self.get(part)

   492                 if old_options == new_options:

   493                     # The options are the same, but are all of the

   494                     # installed files still there?  If not, we should

   495                     # reinstall.

   496                     if not installed_files:

   497                         continue

   498                     for f in installed_files.split('\n'):

   499                         if not os.path.exists(self._buildout_path(f)):

   500                             break

   501                     else:

   502                         continue

   503 

   504                 # output debugging info

   505                 if self._logger.getEffectiveLevel() < logging.DEBUG:

   506                     for k in old_options:

   507                         if k not in new_options:

   508                             self._logger.debug("Part %s, dropped option %s.",

   509                                                part, k)

   510                         elif old_options[k] != new_options[k]:

   511                             self._logger.debug(

   512                                 "Part %s, option %s changed:\n%r != %r",

   513                                 part, k, new_options[k], old_options[k],

   514                                 )

   515                     for k in new_options:

   516                         if k not in old_options:

   517                             self._logger.debug("Part %s, new option %s.",

   518                                                part, k)

   519 

   520             elif not uninstall_missing:

   521                 continue

   522 

   523             self._uninstall_part(part, installed_part_options)

   524             installed_parts = [p for p in installed_parts if p != part]

   525 

   526             if installed_exists:

   527                 self._update_installed(parts=' '.join(installed_parts))

   528 

   529         # Check for unused buildout options:

   530         _check_for_unused_options_in_section(self, 'buildout')

   531 

   532         # install new parts

   533         for part in install_parts:

   534             signature = self[part].pop('__buildout_signature__')

   535             saved_options = self[part].copy()

   536             recipe = self[part].recipe

   537             if part in installed_parts: # update

   538                 need_to_save_installed = False

   539                 __doing__ = 'Updating %s.', part

   540                 self._logger.info(*__doing__)

   541                 old_options = installed_part_options[part]

   542                 old_installed_files = old_options['__buildout_installed__']

   543 

   544                 try:

   545                     update = recipe.update

   546                 except AttributeError:

   547                     update = recipe.install

   548                     self._logger.warning(

   549                         "The recipe for %s doesn't define an update "

   550                         "method. Using its install method.",

   551                         part)

   552 

   553                 try:

   554                     installed_files = self[part]._call(update)

   555                 except:

   556                     installed_parts.remove(part)

   557                     self._uninstall(old_installed_files)

   558                     if installed_exists:

   559                         self._update_installed(

   560                             parts=' '.join(installed_parts))

   561                     raise

   562 

   563                 old_installed_files = old_installed_files.split('\n')

   564                 if installed_files is None:

   565                     installed_files = old_installed_files

   566                 else:

   567                     if isinstance(installed_files, str):

   568                         installed_files = [installed_files]

   569                     else:

   570                         installed_files = list(installed_files)

   571 

   572                     need_to_save_installed = [

   573                         p for p in installed_files

   574                         if p not in old_installed_files]

   575 

   576                     if need_to_save_installed:

   577                         installed_files = (old_installed_files

   578                                            + need_to_save_installed)

   579 

   580             else: # install

   581                 need_to_save_installed = True

   582                 __doing__ = 'Installing %s.', part

   583                 self._logger.info(*__doing__)

   584                 installed_files = self[part]._call(recipe.install)

   585                 if installed_files is None:

   586                     self._logger.warning(

   587                         "The %s install returned None.  A path or "

   588                         "iterable of paths should be returned.",

   589                         part)

   590                     installed_files = ()

   591                 elif isinstance(installed_files, str):

   592                     installed_files = [installed_files]

   593                 else:

   594                     installed_files = list(installed_files)

   595 

   596             installed_part_options[part] = saved_options

   597             saved_options['__buildout_installed__'

   598                           ] = '\n'.join(installed_files)

   599             saved_options['__buildout_signature__'] = signature

   600 

   601             installed_parts = [p for p in installed_parts if p != part]

   602             installed_parts.append(part)

   603             _check_for_unused_options_in_section(self, part)

   604 

   605             if need_to_save_installed:

   606                 installed_part_options['buildout']['parts'] = (

   607                     ' '.join(installed_parts))

   608                 self._save_installed_options(installed_part_options)

   609                 installed_exists = True

   610             else:

   611                 assert installed_exists

   612                 self._update_installed(parts=' '.join(installed_parts))

   613 

   614         if installed_develop_eggs:

   615             if not installed_exists:

   616                 self._save_installed_options(installed_part_options)

   617         elif (not installed_parts) and installed_exists:

   618             os.remove(self['buildout']['installed'])

   619 

   620         self._unload_extensions()

   621 

   622     def _update_installed(self, **buildout_options):

   623         installed = self['buildout']['installed']

   624         f = open(installed, 'a')

   625         f.write('\n[buildout]\n')

   626         for option, value in buildout_options.items():

   627             _save_option(option, value, f)

   628         f.close()

   629 

   630     def _uninstall_part(self, part, installed_part_options):

   631         # uninstall part

   632         __doing__ = 'Uninstalling %s.', part

   633         self._logger.info(*__doing__)

   634 

   635         # run uninstall recipe

   636         recipe, entry = _recipe(installed_part_options[part])

   637         try:

   638             uninstaller = _install_and_load(

   639                 recipe, 'zc.buildout.uninstall', entry, self)

   640             self._logger.info('Running uninstall recipe.')

   641             uninstaller(part, installed_part_options[part])

   642         except (ImportError, pkg_resources.DistributionNotFound), v:

   643             pass

   644 

   645         # remove created files and directories

   646         self._uninstall(

   647             installed_part_options[part]['__buildout_installed__'])

   648 

   649     def _setup_directories(self):

   650         __doing__ = 'Setting up buildout directories'

   651 

   652         # Create buildout directories

   653         for name in ('bin', 'parts', 'eggs', 'develop-eggs'):

   654             d = self['buildout'][name+'-directory']

   655             if not os.path.exists(d):

   656                 self._logger.info('Creating directory %r.', d)

   657                 os.mkdir(d)

   658 

   659     def _develop(self):

   660         """Install sources by running setup.py develop on them

   661         """

   662         __doing__ = 'Processing directories listed in the develop option'

   663 

   664         develop = self['buildout'].get('develop')

   665         if not develop:

   666             return ''

   667 

   668         dest = self['buildout']['develop-eggs-directory']

   669         old_files = os.listdir(dest)

   670 

   671         env = dict(os.environ, PYTHONPATH=pkg_resources_loc)

   672         here = os.getcwd()

   673         try:

   674             try:

   675                 for setup in develop.split():

   676                     setup = self._buildout_path(setup)

   677                     files = glob.glob(setup)

   678                     if not files:

   679                         self._logger.warn("Couldn't develop %r (not found)",

   680                                           setup)

   681                     else:

   682                         files.sort()

   683                     for setup in files:

   684                         self._logger.info("Develop: %r", setup)

   685                         __doing__ = 'Processing develop directory %r.', setup

   686                         zc.buildout.easy_install.develop(setup, dest)

   687             except:

   688                 # if we had an error, we need to roll back changes, by

   689                 # removing any files we created.

   690                 self._sanity_check_develop_eggs_files(dest, old_files)

   691                 self._uninstall('\n'.join(

   692                     [os.path.join(dest, f)

   693                      for f in os.listdir(dest)

   694                      if f not in old_files

   695                      ]))

   696                 raise

   697 

   698             else:

   699                 self._sanity_check_develop_eggs_files(dest, old_files)

   700                 return '\n'.join([os.path.join(dest, f)

   701                                   for f in os.listdir(dest)

   702                                   if f not in old_files

   703                                   ])

   704 

   705         finally:

   706             os.chdir(here)

   707 

   708 

   709     def _sanity_check_develop_eggs_files(self, dest, old_files):

   710         for f in os.listdir(dest):

   711             if f in old_files:

   712                 continue

   713             if not (os.path.isfile(os.path.join(dest, f))

   714                     and f.endswith('.egg-link')):

   715                 self._logger.warning(

   716                     "Unexpected entry, %r, in develop-eggs directory.", f)

   717 

   718     def _compute_part_signatures(self, parts):

   719         # Compute recipe signature and add to options

   720         for part in parts:

   721             options = self.get(part)

   722             if options is None:

   723                 options = self[part] = {}

   724             recipe, entry = _recipe(options)

   725             req = pkg_resources.Requirement.parse(recipe)

   726             sig = _dists_sig(pkg_resources.working_set.resolve([req]))

   727             options['__buildout_signature__'] = ' '.join(sig)

   728 

   729     def _read_installed_part_options(self):

   730         old = self['buildout']['installed']

   731         if old and os.path.isfile(old):

   732             parser = ConfigParser.RawConfigParser()

   733             parser.optionxform = lambda s: s

   734             parser.read(old)

   735             result = {}

   736             for section in parser.sections():

   737                 options = {}

   738                 for option, value in parser.items(section):

   739                     if '%(' in value:

   740                         for k, v in _spacey_defaults:

   741                             value = value.replace(k, v)

   742                     options[option] = value

   743                 result[section] = Options(self, section, options)

   744 

   745             return result, True

   746         else:

   747             return ({'buildout': Options(self, 'buildout', {'parts': ''})},

   748                     False,

   749                     )

   750 

   751     def _uninstall(self, installed):

   752         for f in installed.split('\n'):

   753             if not f:

   754                 continue

   755             f = self._buildout_path(f)

   756             if os.path.isdir(f):

   757                 rmtree(f)

   758             elif os.path.isfile(f):

   759                 try:

   760                     os.remove(f)

   761                 except OSError:

   762                     if not (

   763                         sys.platform == 'win32' and

   764                         (realpath(os.path.join(os.path.dirname(sys.argv[0]),

   765                                                'buildout.exe'))

   766                          ==

   767                          realpath(f)

   768                          )

   769                         # Sigh. This is the exectable used to run the buildout

   770                         # and, of course, it's in use. Leave it.

   771                         ):

   772                         raise

   773 

   774     def _install(self, part):

   775         options = self[part]

   776         recipe, entry = _recipe(options)

   777         recipe_class = pkg_resources.load_entry_point(

   778             recipe, 'zc.buildout', entry)

   779         installed = recipe_class(self, part, options).install()

   780         if installed is None:

   781             installed = []

   782         elif isinstance(installed, basestring):

   783             installed = [installed]

   784         base = self._buildout_path('')

   785         installed = [d.startswith(base) and d[len(base):] or d

   786                      for d in installed]

   787         return ' '.join(installed)

   788 

   789 

   790     def _save_installed_options(self, installed_options):

   791         installed = self['buildout']['installed']

   792         if not installed:

   793             return

   794         f = open(installed, 'w')

   795         _save_options('buildout', installed_options['buildout'], f)

   796         for part in installed_options['buildout']['parts'].split():

   797             print >>f

   798             _save_options(part, installed_options[part], f)

   799         f.close()

   800 

   801     def _error(self, message, *args):

   802         raise zc.buildout.UserError(message % args)

   803 

   804     def _setup_logging(self):

   805         root_logger = logging.getLogger()

   806         self._logger = logging.getLogger('zc.buildout')

   807         handler = logging.StreamHandler(sys.stdout)

   808         log_format = self['buildout']['log-format']

   809         if not log_format:

   810             # No format specified. Use different formatter for buildout

   811             # and other modules, showing logger name except for buildout

   812             log_format = '%(name)s: %(message)s'

   813             buildout_handler = logging.StreamHandler(sys.stdout)

   814             buildout_handler.setFormatter(logging.Formatter('%(message)s'))

   815             self._logger.propagate = False

   816             self._logger.addHandler(buildout_handler)

   817 

   818         handler.setFormatter(logging.Formatter(log_format))

   819         root_logger.addHandler(handler)

   820 

   821         level = self['buildout']['log-level']

   822         if level in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):

   823             level = getattr(logging, level)

   824         else:

   825             try:

   826                 level = int(level)

   827             except ValueError:

   828                 self._error("Invalid logging level %s", level)

   829         verbosity = self['buildout'].get('verbosity', 0)

   830         try:

   831             verbosity = int(verbosity)

   832         except ValueError:

   833             self._error("Invalid verbosity %s", verbosity)

   834 

   835         level -= verbosity

   836         root_logger.setLevel(level)

   837         self._log_level = level

   838 

   839     def _maybe_upgrade(self):

   840         # See if buildout or setuptools need to be upgraded.

   841         # If they do, do the upgrade and restart the buildout process.

   842         __doing__ = 'Checking for upgrades.'

   843 

   844         if not self.newest:

   845             return

   846 

   847         options = self['buildout']

   848 

   849         specs = ['zc.buildout']

   850         if zc.buildout.easy_install.is_distribute:

   851             specs.append('distribute')

   852         else:

   853             specs.append('setuptools')

   854         ws = zc.buildout.easy_install.install(

   855             [

   856             (spec + ' ' + options.get(spec+'-version', '')).strip()

   857             for spec in specs

   858             ],

   859             options['eggs-directory'],

   860             links = options.get('find-links', '').split(),

   861             index = options.get('index'),

   862             path = [options['develop-eggs-directory']],

   863             allow_hosts = self._allow_hosts,

   864             prefer_final=not self.accept_buildout_test_releases,

   865             )

   866 

   867         upgraded = []

   868         for project in 'zc.buildout', 'setuptools':

   869             req = pkg_resources.Requirement.parse(project)

   870             project_location = pkg_resources.working_set.find(req).location

   871             if ws.find(req).location != project_location:

   872                 upgraded.append(ws.find(req))

   873 

   874         if not upgraded:

   875             return

   876 

   877         __doing__ = 'Upgrading.'

   878 

   879         should_run = realpath(

   880             os.path.join(os.path.abspath(options['bin-directory']),

   881                          'buildout')

   882             )

   883         if sys.platform == 'win32':

   884             should_run += '-script.py'

   885 

   886         if (realpath(os.path.abspath(sys.argv[0])) != should_run):

   887             self._logger.debug("Running %r.", realpath(sys.argv[0]))

   888             self._logger.debug("Local buildout is %r.", should_run)

   889             self._logger.warn("Not upgrading because not running a local "

   890                               "buildout command.")

   891             return

   892 

   893         if sys.platform == 'win32' and not self.__windows_restart:

   894             args = map(zc.buildout.easy_install._safe_arg, sys.argv)

   895             args.insert(1, '-W')

   896             if not __debug__:

   897                 args.insert(0, '-O')

   898             args.insert(0, zc.buildout.easy_install._safe_arg (sys.executable))

   899             os.execv(sys.executable, args)

   900 

   901         self._logger.info("Upgraded:\n  %s;\nrestarting.",

   902                           ",\n  ".join([("%s version %s"

   903                                        % (dist.project_name, dist.version)

   904                                        )

   905                                       for dist in upgraded

   906                                       ]

   907                                      ),

   908                           )

   909 

   910         # the new dist is different, so we've upgraded.

   911         # Update the scripts and return True

   912         # Ideally the new version of buildout would get a chance to write the

   913         # script.  Not sure how to do that.

   914         partsdir = os.path.join(options['parts-directory'], 'buildout')

   915         if os.path.exists(partsdir):

   916             # This is primarily for unit tests, in which .py files change too

   917             # fast for Python to know to regenerate the .pyc/.pyo files.

   918             shutil.rmtree(partsdir)

   919         os.mkdir(partsdir)

   920         if (self.accept_buildout_test_releases and

   921             self._annotated['buildout']['accept-buildout-test-releases'][1] ==

   922             'COMMAND_LINE_VALUE'):

   923             # Bootstrap was called with '--accept-buildout-test-releases'.

   924             # Continue to honor that setting.

   925             script_initialization = _early_release_initialization_code

   926         else:

   927             script_initialization = ''

   928         # (Honor the relative-paths option.)

   929         relative_paths = options.get('relative-paths', 'false')

   930         if relative_paths == 'true':

   931             relative_paths = options['directory']

   932         else:

   933             assert relative_paths == 'false'

   934             relative_paths = ''

   935         zc.buildout.easy_install.sitepackage_safe_scripts(

   936             options['bin-directory'], ws, options['executable'], partsdir,

   937             reqs=['zc.buildout'], relative_paths=relative_paths,

   938             include_site_packages=self.include_site_packages,

   939             script_initialization=script_initialization,

   940             exec_sitecustomize=self.exec_sitecustomize,

   941             )

   942 

   943         # Restart

   944         args = map(zc.buildout.easy_install._safe_arg, sys.argv)

   945         if not __debug__:

   946             args.insert(0, '-O')

   947         args.insert(0, zc.buildout.easy_install._safe_arg(sys.executable))

   948         # We want to make sure that our new site.py is used for rerunning

   949         # buildout, so we put the partsdir in PYTHONPATH for our restart.

   950         # This overrides any set PYTHONPATH, but since we generally are

   951         # trying to run with a completely "clean" python (only the standard

   952         # library) then that should be fine.

   953         env = os.environ.copy()

   954         env['PYTHONPATH'] = partsdir

   955         if is_jython:

   956             sys.exit(

   957                 subprocess.Popen(

   958                     [sys.executable] + list(args), env=env).wait())

   959         else:

   960             sys.exit(os.spawnve(os.P_WAIT, sys.executable, args, env))

   961 

   962     def _load_extensions(self):

   963         __doing__ = 'Loading extensions.'

   964         specs = self['buildout'].get('extensions', '').split()

   965         if specs:

   966             path = [self['buildout']['develop-eggs-directory']]

   967             if self.offline:

   968                 dest = None

   969                 path.append(self['buildout']['eggs-directory'])

   970             else:

   971                 dest = self['buildout']['eggs-directory']

   972                 if not os.path.exists(dest):

   973                     self._logger.info('Creating directory %r.', dest)

   974                     os.mkdir(dest)

   975 

   976             zc.buildout.easy_install.install(

   977                 specs, dest, path=path,

   978                 working_set=pkg_resources.working_set,

   979                 links = self['buildout'].get('find-links', '').split(),

   980                 index = self['buildout'].get('index'),

   981                 newest=self.newest, allow_hosts=self._allow_hosts,

   982                 prefer_final=not self.accept_buildout_test_releases)

   983 

   984             # Clear cache because extensions might now let us read pages we

   985             # couldn't read before.

   986             zc.buildout.easy_install.clear_index_cache()

   987 

   988             for ep in pkg_resources.iter_entry_points('zc.buildout.extension'):

   989                 ep.load()(self)

   990 

   991     def _unload_extensions(self):

   992         __doing__ = 'Unloading extensions.'

   993         specs = self['buildout'].get('extensions', '').split()

   994         if specs:

   995             for ep in pkg_resources.iter_entry_points(

   996                 'zc.buildout.unloadextension'):

   997                 ep.load()(self)

   998 

   999     def setup(self, args):

  1000         if not args:

  1001             raise zc.buildout.UserError(

  1002                 "The setup command requires the path to a setup script or \n"

  1003                 "directory containing a setup script, and its arguments."

  1004                 )

  1005         setup = args.pop(0)

  1006         if os.path.isdir(setup):

  1007             setup = os.path.join(setup, 'setup.py')

  1008 

  1009         self._logger.info("Running setup script %r.", setup)

  1010         setup = os.path.abspath(setup)

  1011 

  1012         fd, tsetup = tempfile.mkstemp()

  1013         exe = zc.buildout.easy_install._safe_arg(sys.executable)

  1014         try:

  1015             os.write(fd, zc.buildout.easy_install.runsetup_template % dict(

  1016                 setuptools=pkg_resources_loc,

  1017                 setupdir=os.path.dirname(setup),

  1018                 setup=setup,

  1019                 __file__ = setup,

  1020                 ))

  1021             if is_jython:

  1022                 arg_list = list()

  1023 

  1024                 for a in args:

  1025                     arg_list.append(zc.buildout.easy_install._safe_arg(a))

  1026 

  1027                 subprocess.Popen([exe] + list(tsetup) + arg_list).wait()

  1028 

  1029             else:

  1030                 os.spawnl(os.P_WAIT, sys.executable, exe, tsetup,

  1031                         *[zc.buildout.easy_install._safe_arg(a)

  1032                             for a in args])

  1033         finally:

  1034             os.close(fd)

  1035             os.remove(tsetup)

  1036 

  1037     runsetup = setup # backward compat.

  1038 

  1039     def annotate(self, args):

  1040         _print_annotate(self._annotated)

  1041 

  1042     def __getitem__(self, section):

  1043         __doing__ = 'Getting section %s.', section

  1044         try:

  1045             return self._data[section]

  1046         except KeyError:

  1047             pass

  1048 

  1049         try:

  1050             data = self._raw[section]

  1051         except KeyError:

  1052             raise MissingSection(section)

  1053 

  1054         options = Options(self, section, data)

  1055         self._data[section] = options

  1056         options._initialize()

  1057         return options

  1058 

  1059     def __setitem__(self, key, value):

  1060         raise NotImplementedError('__setitem__')

  1061 

  1062     def __delitem__(self, key):

  1063         raise NotImplementedError('__delitem__')

  1064 

  1065     def keys(self):

  1066         return self._raw.keys()

  1067 

  1068     def __iter__(self):

  1069         return iter(self._raw)

  1070 

  1071 

  1072 def _install_and_load(spec, group, entry, buildout):

  1073     __doing__ = 'Loading recipe %r.', spec

  1074     try:

  1075         req = pkg_resources.Requirement.parse(spec)

  1076 

  1077         buildout_options = buildout['buildout']

  1078         if pkg_resources.working_set.find(req) is None:

  1079             __doing__ = 'Installing recipe %s.', spec

  1080             if buildout.offline:

  1081                 dest = None

  1082                 path = [buildout_options['develop-eggs-directory'],

  1083                         buildout_options['eggs-directory'],

  1084                         ]

  1085             else:

  1086                 dest = buildout_options['eggs-directory']

  1087                 path = [buildout_options['develop-eggs-directory']]

  1088 

  1089             zc.buildout.easy_install.install(

  1090                 [spec], dest,

  1091                 links=buildout._links,

  1092                 index=buildout_options.get('index'),

  1093                 path=path,

  1094                 working_set=pkg_resources.working_set,

  1095                 newest=buildout.newest,

  1096                 allow_hosts=buildout._allow_hosts,

  1097                 prefer_final=not buildout.accept_buildout_test_releases)

  1098 

  1099         __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry

  1100         return pkg_resources.load_entry_point(

  1101             req.project_name, group, entry)

  1102 

  1103     except Exception, v:

  1104         buildout._logger.log(

  1105             1,

  1106             "Could't load %s entry point %s\nfrom %s:\n%s.",

  1107             group, entry, spec, v)

  1108         raise

  1109 

  1110 

  1111 class Options(UserDict.DictMixin):

  1112 

  1113     def __init__(self, buildout, section, data):

  1114         self.buildout = buildout

  1115         self.name = section

  1116         self._raw = data

  1117         self._cooked = {}

  1118         self._data = {}

  1119 

  1120     def _initialize(self):

  1121         name = self.name

  1122         __doing__ = 'Initializing section %s.', name

  1123 

  1124         if '<' in self._raw:

  1125             self._raw = self._do_extend_raw(name, self._raw, [])

  1126 

  1127         # force substitutions

  1128         for k, v in self._raw.items():

  1129             if '${' in v:

  1130                 self._dosub(k, v)

  1131 

  1132         if self.name == 'buildout':

  1133             return # buildout section can never be a part

  1134 

  1135         recipe = self.get('recipe')

  1136         if not recipe: