1 #############################################################################

     2 #

     3 # Copyright (c) 2005 Zope Corporation and Contributors.

     4 # All Rights Reserved.

     5 #

     6 # This software is subject to the provisions of the Zope Public License,

     7 # Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.

     8 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED

     9 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

    10 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS

    11 # FOR A PARTICULAR PURPOSE.

    12 #

    13 ##############################################################################

    14 """Python easy_install API

    15 

    16 This module provides a high-level Python API for installing packages.

    17 It doesn't install scripts.  It uses setuptools and requires it to be

    18 installed.

    19 """

    20 

    21 import distutils.errors

    22 import fnmatch

    23 import glob

    24 import logging

    25 import os

    26 import pkg_resources

    27 import py_compile

    28 import re

    29 import setuptools.archive_util

    30 import setuptools.command.setopt

    31 import setuptools.package_index

    32 import shutil

    33 import subprocess

    34 import sys

    35 import tempfile

    36 import warnings

    37 import zc.buildout

    38 import zipimport

    39 

    40 _oprp = getattr(os.path, 'realpath', lambda path: path)

    41 def realpath(path):

    42     return os.path.normcase(os.path.abspath(_oprp(path)))

    43 

    44 default_index_url = os.environ.get(

    45     'buildout-testing-index-url',

    46     'http://pypi.python.org/simple',

    47     )

    48 

    49 logger = logging.getLogger('zc.buildout.easy_install')

    50 

    51 url_match = re.compile('[a-z0-9+.-]+://').match

    52 

    53 is_win32 = sys.platform == 'win32'

    54 is_jython = sys.platform.startswith('java')

    55 is_distribute = (

    56     pkg_resources.Requirement.parse('setuptools').key=='distribute')

    57 

    58 BROKEN_DASH_S_WARNING = (

    59     'Buildout has been asked to exclude or limit site-packages so that '

    60     'builds can be repeatable when using a system Python.  However, '

    61     'the chosen Python executable has a broken implementation of -S (see '

    62     'https://bugs.launchpad.net/virtualenv/+bug/572545 for an example '

    63     "problem) and this breaks buildout's ability to isolate site-packages.  "

    64     "If the executable already has a clean site-packages (e.g., "

    65     "using virtualenv's ``--no-site-packages`` option) you may be getting "

    66     'equivalent repeatability.  To silence this warning, use the -s argument '

    67     'to the buildout script.  Alternatively, use a Python executable with a '

    68     'working -S (such as a standard Python binary).')

    69 

    70 if is_jython:

    71     import java.lang.System

    72     jython_os_name = (java.lang.System.getProperties()['os.name']).lower()

    73 

    74 setuptools_loc = pkg_resources.working_set.find(

    75     pkg_resources.Requirement.parse('setuptools')

    76     ).location

    77 

    78 # Include buildout and setuptools eggs in paths.  We prevent dupes just to

    79 # keep from duplicating any log messages about them.

    80 buildout_loc = pkg_resources.working_set.find(

    81     pkg_resources.Requirement.parse('zc.buildout')).location

    82 buildout_and_setuptools_path = [setuptools_loc]

    83 if os.path.normpath(setuptools_loc) != os.path.normpath(buildout_loc):

    84     buildout_and_setuptools_path.append(buildout_loc)

    85 

    86 def _has_broken_dash_S(executable):

    87     """Detect https://bugs.launchpad.net/virtualenv/+bug/572545 ."""

    88     # The first attempt here was to simply have the executable attempt to import

    89     # ConfigParser and return the return code. That worked except for tests on

    90     # Windows, where the return code was wrong for the fake Python executable

    91     # generated by the virtualenv.txt test, apparently because setuptools' .exe

    92     # file does not pass the -script.py's returncode back properly, at least in

    93     # some circumstances. Therefore...print statements.

    94     stdout, stderr = subprocess.Popen(

    95         [executable, '-Sc',

    96          'try:\n'

    97          '    import ConfigParser\n'

    98          'except ImportError:\n'

    99          '    print 1\n'

   100          'else:\n'

   101          '    print 0\n'],

   102         stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()

   103     return bool(int(stdout.strip()))

   104 

   105 def _get_system_paths(executable):

   106     """Return lists of standard lib and site paths for executable.

   107     """

   108     # We want to get a list of the site packages, which is not easy.

   109     # The canonical way to do this is to use

   110     # distutils.sysconfig.get_python_lib(), but that only returns a

   111     # single path, which does not reflect reality for many system

   112     # Pythons, which have multiple additions.  Instead, we start Python

   113     # with -S, which does not import site.py and set up the extra paths

   114     # like site-packages or (Ubuntu/Debian) dist-packages and

   115     # python-support. We then compare that sys.path with the normal one

   116     # (minus user packages if this is Python 2.6, because we don't

   117     # support those (yet?).  The set of the normal one minus the set of

   118     # the ones in ``python -S`` is the set of packages that are

   119     # effectively site-packages.

   120     #

   121     # The given executable might not be the current executable, so it is

   122     # appropriate to do another subprocess to figure out what the

   123     # additional site-package paths are. Moreover, even if this

   124     # executable *is* the current executable, this code might be run in

   125     # the context of code that has manipulated the sys.path--for

   126     # instance, to add local zc.buildout or setuptools eggs.

   127     def get_sys_path(*args, **kwargs):

   128         cmd = [executable]

   129         cmd.extend(args)

   130         cmd.extend([

   131             "-c", "import sys, os;"

   132             "print repr([os.path.normpath(p) for p in sys.path if p])"])

   133         # Windows needs some (as yet to be determined) part of the real env.

   134         env = os.environ.copy()

   135         # We need to make sure that PYTHONPATH, which will often be set

   136         # to include a custom buildout-generated site.py, is not set, or

   137         # else we will not get an accurate sys.path for the executable.

   138         env.pop('PYTHONPATH', None)

   139         env.update(kwargs)

   140         _proc = subprocess.Popen(

   141             cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)

   142         stdout, stderr = _proc.communicate();

   143         if _proc.returncode:

   144             raise RuntimeError(

   145                 'error trying to get system packages:\n%s' % (stderr,))

   146         res = eval(stdout.strip())

   147         try:

   148             res.remove('.')

   149         except ValueError:

   150             pass

   151         return res

   152     stdlib = get_sys_path('-S') # stdlib only

   153     no_user_paths = get_sys_path(PYTHONNOUSERSITE='x')

   154     site_paths = [p for p in no_user_paths if p not in stdlib]

   155     return (stdlib, site_paths)

   156 

   157 def _get_version_info(executable):

   158     cmd = [executable, '-Sc',

   159            'import sys; print(repr(tuple(x for x in sys.version_info)))']

   160     _proc = subprocess.Popen(

   161         cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

   162     stdout, stderr = _proc.communicate();

   163     if _proc.returncode:

   164         raise RuntimeError(

   165             'error trying to get system packages:\n%s' % (stderr,))

   166     return eval(stdout.strip())

   167 

   168 

   169 class IncompatibleVersionError(zc.buildout.UserError):

   170     """A specified version is incompatible with a given requirement.

   171     """

   172 

   173 _versions = {sys.executable: '%d.%d' % sys.version_info[:2]}

   174 def _get_version(executable):

   175     try:

   176         return _versions[executable]

   177     except KeyError:

   178         cmd = _safe_arg(executable) + ' -V'

   179         p = subprocess.Popen(cmd,

   180                              shell=True,

   181                              stdin=subprocess.PIPE,

   182                              stdout=subprocess.PIPE,

   183                              stderr=subprocess.STDOUT,

   184                              close_fds=not is_win32)

   185         i, o = (p.stdin, p.stdout)

   186         i.close()

   187         version = o.read().strip()

   188         o.close()

   189         pystring, version = version.split()

   190         assert pystring == 'Python'

   191         version = re.match('(\d[.]\d)([.].*\d)?$', version).group(1)

   192         _versions[executable] = version

   193         return version

   194 

   195 FILE_SCHEME = re.compile('file://', re.I).match

   196 

   197 

   198 class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):

   199     """Will allow urls that are local to the system.

   200 

   201     No matter what is allow_hosts.

   202     """

   203     def url_ok(self, url, fatal=False):

   204         if FILE_SCHEME(url):

   205             return True

   206         return setuptools.package_index.PackageIndex.url_ok(self, url, False)

   207 

   208 

   209 _indexes = {}

   210 def _get_index(executable, index_url, find_links, allow_hosts=('*',),

   211                path=None):

   212     # If path is None, the index will use sys.path.  If you provide an empty

   213     # path ([]), it will complain uselessly about missing index pages for

   214     # packages found in the paths that you expect to use.  Therefore, this path

   215     # is always the same as the _env path in the Installer.

   216     key = executable, index_url, tuple(find_links)

   217     index = _indexes.get(key)

   218     if index is not None:

   219         return index

   220 

   221     if index_url is None:

   222         index_url = default_index_url

   223     index = AllowHostsPackageIndex(

   224         index_url, hosts=allow_hosts, search_path=path,

   225         python=_get_version(executable)

   226         )

   227 

   228     if find_links:

   229         index.add_find_links(find_links)

   230 

   231     _indexes[key] = index

   232     return index

   233 

   234 clear_index_cache = _indexes.clear

   235 

   236 if is_win32:

   237     # work around spawn lamosity on windows

   238     # XXX need safe quoting (see the subprocess.list2cmdline) and test

   239     def _safe_arg(arg):

   240         return '"%s"' % arg

   241 else:

   242     _safe_arg = str

   243 

   244 # The following string is used to run easy_install in

   245 # Installer._call_easy_install.  It is usually started with python -S

   246 # (that is, don't import site at start).  That flag, and all of the code

   247 # in this snippet above the last two lines, exist to work around a

   248 # relatively rare problem.  If

   249 #

   250 # - your buildout configuration is trying to install a package that is within

   251 #   a namespace package, and

   252 #

   253 # - you use a Python that has a different version of this package

   254 #   installed in in its site-packages using

   255 #   --single-version-externally-managed (that is, using the mechanism

   256 #   sometimes used by system packagers:

   257 #   http://peak.telecommunity.com/DevCenter/setuptools#install-command ), and

   258 #

   259 # - the new package tries to do sys.path tricks in the setup.py to get a

   260 #   __version__,

   261 #

   262 # then the older package will be loaded first, making the setup version

   263 # the wrong number. While very arguably packages simply shouldn't do

   264 # the sys.path tricks, some do, and we don't want buildout to fall over

   265 # when they do.

   266 #

   267 # The namespace packages installed in site-packages with

   268 # --single-version-externally-managed use a mechanism that cause them to

   269 # be processed when site.py is imported  (see

   270 # http://mail.python.org/pipermail/distutils-sig/2009-May/011730.html

   271 # for another description of the problem).  Simply starting Python with

   272 # -S addresses the problem in Python 2.4 and 2.5, but Python 2.6's

   273 # distutils imports a value from the site module, so we unfortunately

   274 # have to do more drastic surgery in the _easy_install_preface code below.

   275 #

   276 # Here's an example of the .pth files created by setuptools when using that

   277 # flag:

   278 #

   279 # import sys,new,os;

   280 # p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('<NAMESPACE>',));

   281 # ie = os.path.exists(os.path.join(p,'__init__.py'));

   282 # m = not ie and sys.modules.setdefault('<NAMESPACE>',new.module('<NAMESPACE>'));

   283 # mp = (m or []) and m.__dict__.setdefault('__path__',[]);

   284 # (p not in mp) and mp.append(p)

   285 #

   286 # The code, below, then, runs under -S, indicating that site.py should

   287 # not be loaded initially.  It gets the initial sys.path under these

   288 # circumstances, and then imports site (because Python 2.6's distutils

   289 # will want it, as mentioned above). It then reinstates the old sys.path

   290 # value. Then it removes namespace packages (created by the setuptools

   291 # code above) from sys.modules.  It identifies namespace packages by

   292 # iterating over every loaded module.  It first looks if there is a

   293 # __path__, so it is a package; and then it sees if that __path__ does

   294 # not have an __init__.py.  (Note that PEP 382,

   295 # http://www.python.org/dev/peps/pep-0382, makes it possible to have a

   296 # namespace package that has an __init__.py, but also should make it

   297 # unnecessary for site.py to preprocess these packages, so it should be

   298 # fine, as far as can be guessed as of this writing.)  Finally, it

   299 # imports easy_install and runs it.

   300 _easy_install_preface = '''\

   301 import sys,os;\

   302 p = sys.path[:];\

   303 import site;\

   304 sys.path[:] = p;\

   305 [sys.modules.pop(k) for k, v in sys.modules.items()\

   306  if hasattr(v, '__path__') and len(v.__path__)==1 and\

   307  not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))];'''

   308 _easy_install_cmd = (

   309     'from setuptools.command.easy_install import main;main()')

   310 

   311 

   312 class Installer:

   313 

   314     _versions = {}

   315     _download_cache = None

   316     _install_from_cache = False

   317     _prefer_final = True

   318     _use_dependency_links = True

   319     _allow_picked_versions = True

   320     _always_unzip = False

   321     _include_site_packages = True

   322     _allowed_eggs_from_site_packages = ('*',)

   323 

   324     def __init__(self,

   325                  dest=None,

   326                  links=(),

   327                  index=None,

   328                  executable=sys.executable,

   329                  always_unzip=None,

   330                  path=None,

   331                  newest=True,

   332                  versions=None,

   333                  use_dependency_links=None,

   334                  allow_hosts=('*',),

   335                  include_site_packages=None,

   336                  allowed_eggs_from_site_packages=None,

   337                  prefer_final=None,

   338                  ):

   339         self._dest = dest

   340         self._allow_hosts = allow_hosts

   341 

   342         if self._install_from_cache:

   343             if not self._download_cache:

   344                 raise ValueError("install_from_cache set to true with no"

   345                                  " download cache")

   346             links = ()

   347             index = 'file://' + self._download_cache

   348 

   349         if use_dependency_links is not None:

   350             self._use_dependency_links = use_dependency_links

   351         if prefer_final is not None:

   352             self._prefer_final = prefer_final

   353         self._links = links = list(_fix_file_links(links))

   354         if self._download_cache and (self._download_cache not in links):

   355             links.insert(0, self._download_cache)

   356 

   357         self._index_url = index

   358         self._executable = executable

   359         self._has_broken_dash_S = _has_broken_dash_S(self._executable)

   360         if always_unzip is not None:

   361             self._always_unzip = always_unzip

   362         path = (path and path[:] or [])

   363         if include_site_packages is not None:

   364             self._include_site_packages = include_site_packages

   365         if allowed_eggs_from_site_packages is not None:

   366             self._allowed_eggs_from_site_packages = tuple(

   367                 allowed_eggs_from_site_packages)

   368         if self._has_broken_dash_S:

   369             if (not self._include_site_packages or

   370                 self._allowed_eggs_from_site_packages != ('*',)):

   371                 # We can't do this if the executable has a broken -S.

   372                 warnings.warn(BROKEN_DASH_S_WARNING)

   373                 self._include_site_packages = True

   374                 self._allowed_eggs_from_site_packages = ('*',)

   375             self._easy_install_cmd = _easy_install_cmd

   376         else:

   377             self._easy_install_cmd = _easy_install_preface + _easy_install_cmd

   378         self._easy_install_cmd = _safe_arg(self._easy_install_cmd)

   379         stdlib, self._site_packages = _get_system_paths(executable)

   380         version_info = _get_version_info(executable)

   381         if version_info == sys.version_info:

   382             # Maybe we can add the buildout and setuptools path.  If we

   383             # are including site_packages, we only have to include the extra

   384             # bits here, so we don't duplicate.  On the other hand, if we

   385             # are not including site_packages, we only want to include the

   386             # parts that are not in site_packages, so the code is the same.

   387             path.extend(

   388                 set(buildout_and_setuptools_path).difference(

   389                     self._site_packages))

   390         if self._include_site_packages:

   391             path.extend(self._site_packages)

   392         if dest is not None and dest not in path:

   393             path.insert(0, dest)

   394         self._path = path

   395         if self._dest is None:

   396             newest = False

   397         self._newest = newest

   398         self._env = pkg_resources.Environment(path,

   399                                               python=_get_version(executable))

   400         self._index = _get_index(executable, index, links, self._allow_hosts,

   401                                  self._path)

   402 

   403         if versions is not None:

   404             self._versions = versions

   405 

   406     _allowed_eggs_from_site_packages_regex = None

   407     def allow_site_package_egg(self, name):

   408         if (not self._include_site_packages or

   409             not self._allowed_eggs_from_site_packages):

   410             # If the answer is a blanket "no," perform a shortcut.

   411             return False

   412         if self._allowed_eggs_from_site_packages_regex is None:

   413             pattern = '(%s)' % (

   414                 '|'.join(

   415                     fnmatch.translate(name)

   416                     for name in self._allowed_eggs_from_site_packages),

   417                 )

   418             self._allowed_eggs_from_site_packages_regex = re.compile(pattern)

   419         return bool(self._allowed_eggs_from_site_packages_regex.match(name))

   420 

   421     def _satisfied(self, req, source=None):

   422         # We get all distributions that match the given requirement.  If we are

   423         # not supposed to include site-packages for the given egg, we also

   424         # filter those out. Even if include_site_packages is False and so we

   425         # have excluded site packages from the _env's paths (see

   426         # Installer.__init__), we need to do the filtering here because an

   427         # .egg-link, such as one for setuptools or zc.buildout installed by

   428         # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a

   429         # path in our _site_packages.

   430         dists = [dist for dist in self._env[req.project_name] if (

   431                     dist in req and (

   432                         dist.location not in self._site_packages or

   433                         self.allow_site_package_egg(dist.project_name))

   434                     )

   435                 ]

   436         if not dists:

   437             logger.debug('We have no distributions for %s that satisfies %r.',

   438                          req.project_name, str(req))

   439 

   440             return None, self._obtain(req, source)

   441 

   442         # Note that dists are sorted from best to worst, as promised by

   443         # env.__getitem__

   444 

   445         for dist in dists:

   446             if (dist.precedence == pkg_resources.DEVELOP_DIST and

   447                 dist.location not in self._site_packages):

   448                 # System eggs are sometimes installed as develop eggs.

   449                 # Those are not the kind of develop eggs we are looking for

   450                 # here: we want ones that the buildout itself has locally as

   451                 # develop eggs.

   452                 logger.debug('We have a develop egg: %s', dist)

   453                 return dist, None

   454 

   455         # Special common case, we have a specification for a single version:

   456         specs = req.specs

   457         if len(specs) == 1 and specs[0][0] == '==':

   458             logger.debug('We have the distribution that satisfies %r.',

   459                          str(req))

   460             return dists[0], None

   461 

   462         if self._prefer_final:

   463             fdists = [dist for dist in dists

   464                       if _final_version(dist.parsed_version)

   465                       ]

   466             if fdists:

   467                 # There are final dists, so only use those

   468                 dists = fdists

   469 

   470         if not self._newest:

   471             # We don't need the newest, so we'll use the newest one we

   472             # find, which is the first returned by

   473             # Environment.__getitem__.

   474             return dists[0], None

   475 

   476         best_we_have = dists[0] # Because dists are sorted from best to worst

   477 

   478         # We have some installed distros.  There might, theoretically, be

   479         # newer ones.  Let's find out which ones are available and see if

   480         # any are newer.  We only do this if we're willing to install

   481         # something, which is only true if dest is not None:

   482 

   483         if self._dest is not None:

   484             best_available = self._obtain(req, source)

   485         else:

   486             best_available = None

   487 

   488         if best_available is None:

   489             # That's a bit odd.  There aren't any distros available.

   490             # We should use the best one we have that meets the requirement.

   491             logger.debug(

   492                 'There are no distros available that meet %r.\n'

   493                 'Using our best, %s.',

   494                 str(req), best_available)

   495             return best_we_have, None

   496 

   497         if self._prefer_final:

   498             if _final_version(best_available.parsed_version):

   499                 if _final_version(best_we_have.parsed_version):

   500                     if (best_we_have.parsed_version

   501                         <

   502                         best_available.parsed_version

   503                         ):

   504                         return None, best_available

   505                 else:

   506                     return None, best_available

   507             else:

   508                 if (not _final_version(best_we_have.parsed_version)

   509                     and

   510                     (best_we_have.parsed_version

   511                      <

   512                      best_available.parsed_version

   513                      )

   514                     ):

   515                     return None, best_available

   516         else:

   517             if (best_we_have.parsed_version

   518                 <

   519                 best_available.parsed_version

   520                 ):

   521                 return None, best_available

   522 

   523         logger.debug(

   524             'We have the best distribution that satisfies %r.',

   525             str(req))

   526         return best_we_have, None

   527 

   528     def _load_dist(self, dist):

   529         dists = pkg_resources.Environment(

   530             dist.location,

   531             python=_get_version(self._executable),

   532             )[dist.project_name]

   533         assert len(dists) == 1

   534         return dists[0]

   535 

   536     def _call_easy_install(self, spec, ws, dest, dist):

   537 

   538         tmp = tempfile.mkdtemp(dir=dest)

   539         try:

   540             path = setuptools_loc

   541 

   542             args = ('-c', self._easy_install_cmd, '-mUNxd', _safe_arg(tmp))

   543             if not self._has_broken_dash_S:

   544                 args = ('-S',) + args

   545             if self._always_unzip:

   546                 args += ('-Z', )

   547             level = logger.getEffectiveLevel()

   548             if level > 0:

   549                 args += ('-q', )

   550             elif level < 0:

   551                 args += ('-v', )

   552 

   553             args += (_safe_arg(spec), )

   554 

   555             if level <= logging.DEBUG:

   556                 logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',

   557                              self._executable, '" "'.join(args), path)

   558 

   559             if is_jython:

   560                 extra_env = dict(os.environ, PYTHONPATH=path)

   561             else:

   562                 args += (dict(os.environ, PYTHONPATH=path), )

   563 

   564             sys.stdout.flush() # We want any pending output first

   565 

   566             if is_jython:

   567                 exit_code = subprocess.Popen(

   568                 [_safe_arg(self._executable)] + list(args),

   569                 env=extra_env).wait()

   570             else:

   571                 exit_code = os.spawnle(

   572                     os.P_WAIT, self._executable, _safe_arg (self._executable),

   573                     *args)

   574 

   575             dists = []

   576             env = pkg_resources.Environment(

   577                 [tmp],

   578                 python=_get_version(self._executable),

   579                 )

   580             for project in env:

   581                 dists.extend(env[project])

   582 

   583             if exit_code:

   584                 logger.error(

   585                     "An error occurred when trying to install %s. "

   586                     "Look above this message for any errors that "

   587                     "were output by easy_install.",

   588                     dist)

   589 

   590             if not dists:

   591                 raise zc.buildout.UserError("Couldn't install: %s" % dist)

   592 

   593             if len(dists) > 1:

   594                 logger.warn("Installing %s\n"

   595                             "caused multiple distributions to be installed:\n"

   596                             "%s\n",

   597                             dist, '\n'.join(map(str, dists)))

   598             else:

   599                 d = dists[0]

   600                 if d.project_name != dist.project_name:

   601                     logger.warn("Installing %s\n"

   602                                 "Caused installation of a distribution:\n"

   603                                 "%s\n"

   604                                 "with a different project name.",

   605                                 dist, d)

   606                 if d.version != dist.version:

   607                     logger.warn("Installing %s\n"

   608                                 "Caused installation of a distribution:\n"

   609                                 "%s\n"

   610                                 "with a different version.",

   611                                 dist, d)

   612 

   613             result = []

   614             for d in dists:

   615                 newloc = os.path.join(dest, os.path.basename(d.location))

   616                 if os.path.exists(newloc):

   617                     if os.path.isdir(newloc):

   618                         shutil.rmtree(newloc)

   619                     else:

   620                         os.remove(newloc)

   621                 os.rename(d.location, newloc)

   622 

   623                 [d] = pkg_resources.Environment(

   624                     [newloc],

   625                     python=_get_version(self._executable),

   626                     )[d.project_name]

   627 

   628                 result.append(d)

   629 

   630             return result

   631 

   632         finally:

   633             shutil.rmtree(tmp)

   634 

   635     def _obtain(self, requirement, source=None):

   636         # initialize out index for this project:

   637         index = self._index

   638 

   639         if index.obtain(requirement) is None:

   640             # Nothing is available.

   641             return None

   642 

   643         # Filter the available dists for the requirement and source flag.  If

   644         # we are not supposed to include site-packages for the given egg, we

   645         # also filter those out. Even if include_site_packages is False and so

   646         # we have excluded site packages from the _env's paths (see

   647         # Installer.__init__), we need to do the filtering here because an

   648         # .egg-link, such as one for setuptools or zc.buildout installed by

   649         # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a

   650         # path in our _site_packages.

   651         dists = [dist for dist in index[requirement.project_name] if (

   652                     dist in requirement and (

   653                         dist.location not in self._site_packages or

   654                         self.allow_site_package_egg(dist.project_name))

   655                     and (

   656                         (not source) or

   657                         (dist.precedence == pkg_resources.SOURCE_DIST))

   658                     )

   659                  ]

   660 

   661         # If we prefer final dists, filter for final and use the

   662         # result if it is non empty.

   663         if self._prefer_final:

   664             fdists = [dist for dist in dists

   665                       if _final_version(dist.parsed_version)

   666                       ]

   667             if fdists:

   668                 # There are final dists, so only use those

   669                 dists = fdists

   670 

   671         # Now find the best one:

   672         best = []

   673         bestv = ()

   674         for dist in dists:

   675             distv = dist.parsed_version

   676             if distv > bestv:

   677                 best = [dist]

   678                 bestv = distv

   679             elif distv == bestv:

   680                 best.append(dist)

   681 

   682         if not best:

   683             return None

   684 

   685         if len(best) == 1:

   686             return best[0]

   687 

   688         if self._download_cache:

   689             for dist in best:

   690                 if (realpath(os.path.dirname(dist.location))

   691                     ==

   692                     self._download_cache

   693                     ):

   694                     return dist

   695 

   696         best.sort()

   697         return best[-1]

   698 

   699     def _fetch(self, dist, tmp, download_cache):

   700         if (download_cache

   701             and (realpath(os.path.dirname(dist.location)) == download_cache)

   702             ):

   703             return dist

   704 

   705         new_location = self._index.download(dist.location, tmp)

   706         if (download_cache

   707             and (realpath(new_location) == realpath(dist.location))

   708             and os.path.isfile(new_location)

   709             ):

   710             # setuptools avoids making extra copies, but we want to copy

   711             # to the download cache

   712             shutil.copy2(new_location, tmp)

   713             new_location = os.path.join(tmp, os.path.basename(new_location))

   714 

   715         return dist.clone(location=new_location)

   716 

   717     def _get_dist(self, requirement, ws, always_unzip):

   718 

   719         __doing__ = 'Getting distribution for %r.', str(requirement)

   720 

   721         # Maybe an existing dist is already the best dist that satisfies the

   722         # requirement

   723         dist, avail = self._satisfied(requirement)

   724 

   725         if dist is None:

   726             if self._dest is not None:

   727                 logger.info(*__doing__)

   728 

   729             # Retrieve the dist:

   730             if avail is None:

   731                 raise MissingDistribution(requirement, ws)

   732 

   733             # We may overwrite distributions, so clear importer

   734             # cache.

   735             sys.path_importer_cache.clear()

   736 

   737             tmp = self._download_cache

   738             if tmp is None:

   739                 tmp = tempfile.mkdtemp('get_dist')

   740 

   741             try:

   742                 dist = self._fetch(avail, tmp, self._download_cache)

   743 

   744                 if dist is None:

   745                     raise zc.buildout.UserError(

   746                         "Couldn't download distribution %s." % avail)

   747 

   748                 if dist.precedence == pkg_resources.EGG_DIST:

   749                     # It's already an egg, just fetch it into the dest

   750 

   751                     newloc = os.path.join(

   752                         self._dest, os.path.basename(dist.location))

   753 

   754                     if os.path.isdir(dist.location):

   755                         # we got a directory. It must have been

   756                         # obtained locally.  Just copy it.

   757                         shutil.copytree(dist.location, newloc)

   758                     else:

   759 

   760                         if self._always_unzip:

   761                             should_unzip = True

   762                         else:

   763                             metadata = pkg_resources.EggMetadata(

   764                                 zipimport.zipimporter(dist.location)

   765                                 )

   766                             should_unzip = (

   767                                 metadata.has_metadata('not-zip-safe')

   768                                 or

   769                                 not metadata.has_metadata('zip-safe')

   770                                 )

   771 

   772                         if should_unzip:

   773                             setuptools.archive_util.unpack_archive(

   774                                 dist.location, newloc)

   775                         else:

   776                             shutil.copyfile(dist.location, newloc)

   777 

   778                     redo_pyc(newloc)

   779 

   780                     # Getting the dist from the environment causes the

   781                     # distribution meta data to be read.  Cloning isn't

   782                     # good enough.

   783                     dists = pkg_resources.Environment(

   784                         [newloc],

   785                         python=_get_version(self._executable),

   786                         )[dist.project_name]

   787                 else:

   788                     # It's some other kind of dist.  We'll let easy_install

   789                     # deal with it:

   790                     dists = self._call_easy_install(

   791                         dist.location, ws, self._dest, dist)

   792                     for dist in dists:

   793                         redo_pyc(dist.location)

   794 

   795             finally:

   796                 if tmp != self._download_cache:

   797                     shutil.rmtree(tmp)

   798 

   799             self._env.scan([self._dest])

   800             dist = self._env.best_match(requirement, ws)

   801             logger.info("Got %s.", dist)

   802 

   803         else:

   804             dists = [dist]

   805 

   806         for dist in dists:

   807             if (dist.has_metadata('dependency_links.txt')

   808                 and not self._install_from_cache

   809                 and self._use_dependency_links

   810                 ):

   811                 for link in dist.get_metadata_lines('dependency_links.txt'):

   812                     link = link.strip()

   813                     if link not in self._links:

   814                         logger.debug('Adding find link %r from %s', link, dist)

   815                         self._links.append(link)

   816                         self._index = _get_index(self._executable,

   817                                                  self._index_url, self._links,

   818                                                  self._allow_hosts, self._path)

   819 

   820         for dist in dists:

   821             # Check whether we picked a version and, if we did, report it:

   822             if not (

   823                 dist.precedence == pkg_resources.DEVELOP_DIST

   824                 or

   825                 (len(requirement.specs) == 1

   826                  and

   827                  requirement.specs[0][0] == '==')

   828                 ):

   829                 logger.debug('Picked: %s = %s',

   830                              dist.project_name, dist.version)

   831                 if not self._allow_picked_versions:

   832                     raise zc.buildout.UserError(

   833                         'Picked: %s = %s' % (dist.project_name, dist.version)

   834                         )

   835 

   836         return dists

   837 

   838     def _maybe_add_setuptools(self, ws, dist):

   839         if dist.has_metadata('namespace_packages.txt'):

   840             for r in dist.requires():

   841                 if r.project_name in ('setuptools', 'distribute'):

   842                     break

   843             else:

   844                 # We have a namespace package but no requirement for setuptools

   845                 if dist.precedence == pkg_resources.DEVELOP_DIST:

   846                     logger.warn(

   847                         "Develop distribution: %s\n"

   848                         "uses namespace packages but the distribution "

   849                         "does not require setuptools.",

   850                         dist)

   851                 requirement = self._constrain(

   852                     pkg_resources.Requirement.parse('setuptools')

   853                     )

   854                 if ws.find(requirement) is None:

   855                     for dist in self._get_dist(requirement, ws, False):

   856                         ws.add(dist)

   857 

   858 

   859     def _constrain(self, requirement):

   860         if is_distribute and requirement.key == 'setuptools':

   861             requirement = pkg_resources.Requirement.parse('distribute')

   862         version = self._versions.get(requirement.project_name)

   863         if version:

   864             if version not in requirement:

   865                 logger.error("The version, %s, is not consistent with the "

   866                              "requirement, %r.", version, str(requirement))

   867                 raise IncompatibleVersionError("Bad version", version)

   868 

   869             requirement = pkg_resources.Requirement.parse(

   870                 "%s[%s] ==%s" % (requirement.project_name,

   871                                ','.join(requirement.extras),

   872                                version))

   873 

   874         return requirement

   875 

   876     def install(self, specs, working_set=None):

   877 

   878         logger.debug('Installing %s.', repr(specs)[1:-1])

   879 

   880         path = self._path

   881         destination = self._dest

   882         if destination is not None and destination not in path:

   883             path.insert(0, destination)

   884 

   885         requirements = [self._constrain(pkg_resources.Requirement.parse(spec))

   886                         for spec in specs]

   887 

   888 

   889 

   890         if working_set is None:

   891             ws = pkg_resources.WorkingSet([])

   892         else:

   893             ws = working_set

   894 

   895         for requirement in requirements:

   896             for dist in self._get_dist(requirement, ws, self._always_unzip):

   897                 ws.add(dist)

   898                 self._maybe_add_setuptools(ws, dist)

   899 

   900         # OK, we have the requested distributions and they're in the working

   901         # set, but they may have unmet requirements.  We'll resolve these

   902         # requirements. This is code modified from

   903         # pkg_resources.WorkingSet.resolve.  We can't reuse that code directly

   904         # because we have to constrain our requirements (see

   905         # versions_section_ignored_for_dependency_in_favor_of_site_packages in

   906         # zc.buildout.tests).

   907         requirements.reverse() # Set up the stack.

   908         processed = {}  # This is a set of processed requirements.

   909         best = {}  # This is a mapping of key -> dist.

   910         # Note that we don't use the existing environment, because we want

   911         # to look for new eggs unless what we have is the best that

   912         # matches the requirement.

   913         env = pkg_resources.Environment(ws.entries)

   914         while requirements:

   915             # Process dependencies breadth-first.

   916             req = self._constrain(requirements.pop(0))

   917             if req in processed:

   918                 # Ignore cyclic or redundant dependencies.

   919                 continue

   920             dist = best.get(req.key)

   921             if dist is None:

   922                 # Find the best distribution and add it to the map.

   923                 dist = ws.by_key.get(req.key)

   924                 if dist is None:

   925                     try:

   926                         dist = best[req.key] = env.best_match(req, ws)

   927                     except pkg_resources.VersionConflict, err:

   928                         raise VersionConflict(err, ws)

   929                     if dist is None or (

   930                         dist.location in self._site_packages and not

   931                         self.allow_site_package_egg(dist.project_name)):

   932                         # If we didn't find a distribution in the

   933                         # environment, or what we found is from site

   934                         # packages and not allowed to be there, try

   935                         # again.

   936                         if destination:

   937                             logger.debug('Getting required %r', str(req))

   938                         else:

   939                             logger.debug('Adding required %r', str(req))

   940                         _log_requirement(ws, req)

   941                         for dist in self._get_dist(req,

   942                                                    ws, self._always_unzip):

   943                             ws.add(dist)

   944                             self._maybe_add_setuptools(ws, dist)

   945             if dist not in req:

   946                 # Oops, the "best" so far conflicts with a dependency.

   947                 raise VersionConflict(

   948                     pkg_resources.VersionConflict(dist, req), ws)

   949             requirements.extend(dist.requires(req.extras)[::-1])

   950             processed[req] = True

   951             if dist.location in self._site_packages:

   952                 logger.debug('Egg from site-packages: %s', dist)

   953         return ws

   954 

   955     def build(self, spec, build_ext):

   956 

   957         requirement = self._constrain(pkg_resources.Requirement.parse(spec))

   958 

   959         dist, avail = self._satisfied(requirement, 1)

   960         if dist is not None:

   961             return [dist.location]

   962 

   963         # Retrieve the dist:

   964         if avail is None:

   965             raise zc.buildout.UserError(

   966                 "Couldn't find a source distribution for %r."

   967                 % str(requirement))

   968 

   969         logger.debug('Building %r', spec)

   970 

   971         tmp = self._download_cache

   972         if tmp is None:

   973             tmp = tempfile.mkdtemp('get_dist')

   974 

   975         try:

   976             dist = self._fetch(avail, tmp, self._download_cache)

   977 

   978             build_tmp = tempfile.mkdtemp('build')

   979             try:

   980                 setuptools.archive_util.unpack_archive(dist.location,

   981                                                        build_tmp)

   982                 if os.path.exists(os.path.join(build_tmp, 'setup.py')):

   983                     base = build_tmp

   984                 else:

   985                     setups = glob.glob(

   986                         os.path.join(build_tmp, '*', 'setup.py'))

   987                     if not setups:

   988                         raise distutils.errors.DistutilsError(

   989                             "Couldn't find a setup script in %s"

   990                             % os.path.basename(dist.location)

   991                             )

   992                     if len(setups) > 1:

   993                         raise distutils.errors.DistutilsError(

   994                             "Multiple setup scripts in %s"

   995                             % os.path.basename(dist.location)

   996                             )

   997                     base = os.path.dirname(setups[0])

   998 

   999                 setup_cfg = os.path.join(base, 'setup.cfg')

  1000                 if not os.path.exists(setup_cfg):

  1001                     f = open(setup_cfg, 'w')

  1002                     f.close()

  1003                 setuptools.command.setopt.edit_config(

  1004                     setup_cfg, dict(build_ext=build_ext))

  1005 

  1006                 dists = self._call_easy_install(

  1007                     base, pkg_resources.WorkingSet(),

  1008                     self._dest, dist)

  1009 

  1010                 for dist in dists:

  1011                     redo_pyc(dist.location)

  1012 

  1013                 return [dist.location for dist in dists]

  1014             finally:

  1015                 shutil.rmtree(build_tmp)

  1016 

  1017         finally:

  1018             if tmp != self._download_cache:

  1019                 shutil.rmtree(tmp)

  1020 

  1021 def default_versions(versions=None):

  1022     old = Installer._versions

  1023     if versions is not None:

  1024         Installer._versions = versions

  1025     return old

  1026 

  1027 def download_cache(path=-1):

  1028     old = Installer._download_cache

  1029     if path != -1:

  1030         if path:

  1031             path = realpath(path)

  1032         Installer._download_cache = path

  1033     return old

  1034 

  1035 def install_from_cache(setting=None):

  1036     old = Installer._install_from_cache

  1037     if setting is not None:

  1038         Installer._install_from_cache = bool(setting)

  1039     return old

  1040 

  1041 def prefer_final(setting=None):

  1042     old = Installer._prefer_final

  1043     if setting is not None:

  1044         Installer._prefer_final = bool(setting)

  1045     return old

  1046 

  1047 def include_site_packages(setting=None):

  1048     old = Installer._include_site_packages

  1049     if setting is not None:

  1050         Installer._include_site_packages = bool(setting)

  1051     return old

  1052 

  1053 def allowed_eggs_from_site_packages(setting=None):

  1054     old = Installer._allowed_eggs_from_site_packages

  1055     if setting is not None:

  1056         Installer._allowed_eggs_from_site_packages = tuple(setting)

  1057     return old

  1058 

  1059 def use_dependency_links(setting=None):

  1060     old = Installer._use_dependency_links

  1061     if setting is not None:

  1062         Installer._use_dependency_links = bool(setting)

  1063     return old

  1064 

  1065 def allow_picked_versions(setting=None):

  1066     old = Installer._allow_picked_versions

  1067     if setting is not None:

  1068         Installer._allow_picked_versions = bool(setting)

  1069     return old

  1070 

  1071 def always_unzip(setting=None):

  1072     old = Installer._always_unzip

  1073     if setting is not None:

  1074         Installer._always_unzip = bool(setting)

  1075     return old

  1076 

  1077 def install(specs, dest,

  1078             links=(), index=None,

  1079             executable=sys.executable, always_unzip=None,

  1080             path=None, working_set=None, newest=True, versions=None,

  1081             use_dependency_links=None, allow_hosts=('*',),

  1082             include_site_packages=None, allowed_eggs_from_site_packages=None,

  1083             prefer_final=None):

  1084     installer = Installer(

  1085         dest, links, index, executable, always_unzip, path, newest,

  1086         versions, use_dependency_links, allow_hosts=allow_hosts,

  1087         include_site_packages=include_site_packages,

  1088         allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,

  1089         prefer_final=prefer_final)

  1090     return installer.install(specs, working_set)

  1091 

  1092 

  1093 def build(spec, dest, build_ext,

  1094           links=(), index=None,

  1095           executable=sys.executable,

  1096           path=None, newest=True, versions=None, allow_hosts=('*',),

  1097           include_site_packages=None, allowed_eggs_from_site_packages=None):

  1098     installer = Installer(

  1099         dest, links, index, executable, True, path, newest, versions,

  1100         allow_hosts=allow_hosts,

  1101         include_site_packages=include_site_packages,

  1102         allowed_eggs_from_site_packages=allowed_eggs_from_site_packages)

  1103     return installer.build(spec, build_ext)

  1104 

  1105 

  1106 

  1107 def _rm(*paths):

  1108     for path in paths:

  1109         if os.path.isdir(path):

  1110             shutil.rmtree(path)

  1111         elif os.path.exists(path):

  1112             os.remove(path)

  1113 

  1114 def _copyeggs(src, dest, suffix, undo):

  1115     result = []

  1116     undo.append(lambda : _rm(*result))

  1117     for name in os.listdir(src):

  1118         if name.endswith(suffix):

  1119             new = os.path.join(dest, name)

  1120             _rm(new)

  1121             os.rename(os.path.join(src, name), new)

  1122             result.append(new)

  1123 

  1124     assert len(result) == 1, str(result)

  1125     undo.pop()

  1126 

  1127     return result[0]

  1128 

  1129 def develop(setup, dest,

  1130             build_ext=None,

  1131             executable=sys.executable):

  1132 

  1133     if os.path.isdir(setup):

  1134         directory = setup

  1135         setup = os.path.join(directory, 'setup.py')

  1136     else: