author | Nishanth Amuluru <nishanth@fossee.in> |
Tue, 11 Jan 2011 17:40:14 +0530 | |
changeset 145 | 4252da60a4ef |
parent 69 | c6bca38c1cbf |
permissions | -rw-r--r-- |
69
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
1 |
############################################################################## |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
2 |
# |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
3 |
# Copyright (c) 2009 Zope Corporation and Contributors. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
4 |
# All Rights Reserved. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
5 |
# |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
6 |
# This software is subject to the provisions of the Zope Public License, |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
7 |
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
8 |
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
9 |
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
10 |
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
11 |
# FOR A PARTICULAR PURPOSE. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
12 |
# |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
13 |
############################################################################## |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
14 |
"""Buildout download infrastructure""" |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
15 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
16 |
try: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
17 |
from hashlib import md5 |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
18 |
except ImportError: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
19 |
from md5 import new as md5 |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
20 |
from zc.buildout.easy_install import realpath |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
21 |
import logging |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
22 |
import os |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
23 |
import os.path |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
24 |
import re |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
25 |
import shutil |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
26 |
import tempfile |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
27 |
import urllib |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
28 |
import urlparse |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
29 |
import zc.buildout |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
30 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
31 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
32 |
class URLOpener(urllib.FancyURLopener): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
33 |
http_error_default = urllib.URLopener.http_error_default |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
34 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
35 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
36 |
class ChecksumError(zc.buildout.UserError): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
37 |
pass |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
38 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
39 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
40 |
url_opener = URLOpener() |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
41 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
42 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
43 |
class Download(object): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
44 |
"""Configurable download utility. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
45 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
46 |
Handles the download cache and offline mode. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
47 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
48 |
Download(options=None, cache=None, namespace=None, hash_name=False) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
49 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
50 |
options: mapping of buildout options (e.g. a ``buildout`` config section) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
51 |
cache: path to the download cache (excluding namespaces) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
52 |
namespace: namespace directory to use inside the cache |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
53 |
hash_name: whether to use a hash of the URL as cache file name |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
54 |
logger: an optional logger to receive download-related log messages |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
55 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
56 |
""" |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
57 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
58 |
def __init__(self, options={}, cache=-1, namespace=None, |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
59 |
offline=-1, fallback=False, hash_name=False, logger=None): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
60 |
self.directory = options.get('directory', '') |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
61 |
self.cache = cache |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
62 |
if cache == -1: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
63 |
self.cache = options.get('download-cache') |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
64 |
self.namespace = namespace |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
65 |
self.offline = offline |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
66 |
if offline == -1: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
67 |
self.offline = (options.get('offline') == 'true' |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
68 |
or options.get('install-from-cache') == 'true') |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
69 |
self.fallback = fallback |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
70 |
self.hash_name = hash_name |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
71 |
self.logger = logger or logging.getLogger('zc.buildout') |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
72 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
73 |
@property |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
74 |
def download_cache(self): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
75 |
if self.cache is not None: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
76 |
return realpath(os.path.join(self.directory, self.cache)) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
77 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
78 |
@property |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
79 |
def cache_dir(self): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
80 |
if self.download_cache is not None: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
81 |
return os.path.join(self.download_cache, self.namespace or '') |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
82 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
83 |
def __call__(self, url, md5sum=None, path=None): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
84 |
"""Download a file according to the utility's configuration. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
85 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
86 |
url: URL to download |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
87 |
md5sum: MD5 checksum to match |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
88 |
path: where to place the downloaded file |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
89 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
90 |
Returns the path to the downloaded file. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
91 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
92 |
""" |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
93 |
if self.cache: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
94 |
local_path, is_temp = self.download_cached(url, md5sum) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
95 |
else: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
96 |
local_path, is_temp = self.download(url, md5sum, path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
97 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
98 |
return locate_at(local_path, path), is_temp |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
99 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
100 |
def download_cached(self, url, md5sum=None): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
101 |
"""Download a file from a URL using the cache. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
102 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
103 |
This method assumes that the cache has been configured. Optionally, it |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
104 |
raises a ChecksumError if a cached copy of a file has an MD5 mismatch, |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
105 |
but will not remove the copy in that case. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
106 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
107 |
""" |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
108 |
if not os.path.exists(self.download_cache): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
109 |
raise zc.buildout.UserError( |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
110 |
'The directory:\n' |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
111 |
'%r\n' |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
112 |
"to be used as a download cache doesn't exist.\n" |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
113 |
% self.download_cache) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
114 |
cache_dir = self.cache_dir |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
115 |
if not os.path.exists(cache_dir): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
116 |
os.mkdir(cache_dir) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
117 |
cache_key = self.filename(url) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
118 |
cached_path = os.path.join(cache_dir, cache_key) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
119 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
120 |
self.logger.debug('Searching cache at %s' % cache_dir) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
121 |
if os.path.isfile(cached_path): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
122 |
is_temp = False |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
123 |
if self.fallback: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
124 |
try: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
125 |
_, is_temp = self.download(url, md5sum, cached_path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
126 |
except ChecksumError: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
127 |
raise |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
128 |
except Exception: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
129 |
pass |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
130 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
131 |
if not check_md5sum(cached_path, md5sum): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
132 |
raise ChecksumError( |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
133 |
'MD5 checksum mismatch for cached download ' |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
134 |
'from %r at %r' % (url, cached_path)) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
135 |
self.logger.debug('Using cache file %s' % cached_path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
136 |
else: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
137 |
self.logger.debug('Cache miss; will cache %s as %s' % |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
138 |
(url, cached_path)) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
139 |
_, is_temp = self.download(url, md5sum, cached_path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
140 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
141 |
return cached_path, is_temp |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
142 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
143 |
def download(self, url, md5sum=None, path=None): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
144 |
"""Download a file from a URL to a given or temporary path. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
145 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
146 |
An online resource is always downloaded to a temporary file and moved |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
147 |
to the specified path only after the download is complete and the |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
148 |
checksum (if given) matches. If path is None, the temporary file is |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
149 |
returned and the client code is responsible for cleaning it up. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
150 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
151 |
""" |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
152 |
if re.match(r"^[A-Za-z]:\\", url): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
153 |
url = 'file:' + url |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
154 |
parsed_url = urlparse.urlparse(url, 'file') |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
155 |
url_scheme, _, url_path = parsed_url[:3] |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
156 |
if url_scheme == 'file': |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
157 |
self.logger.debug('Using local resource %s' % url) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
158 |
if not check_md5sum(url_path, md5sum): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
159 |
raise ChecksumError( |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
160 |
'MD5 checksum mismatch for local resource at %r.' % |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
161 |
url_path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
162 |
return locate_at(url_path, path), False |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
163 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
164 |
if self.offline: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
165 |
raise zc.buildout.UserError( |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
166 |
"Couldn't download %r in offline mode." % url) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
167 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
168 |
self.logger.info('Downloading %s' % url) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
169 |
urllib._urlopener = url_opener |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
170 |
handle, tmp_path = tempfile.mkstemp(prefix='buildout-') |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
171 |
try: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
172 |
try: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
173 |
tmp_path, headers = urllib.urlretrieve(url, tmp_path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
174 |
if not check_md5sum(tmp_path, md5sum): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
175 |
raise ChecksumError( |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
176 |
'MD5 checksum mismatch downloading %r' % url) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
177 |
finally: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
178 |
os.close(handle) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
179 |
except: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
180 |
os.remove(tmp_path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
181 |
raise |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
182 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
183 |
if path: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
184 |
shutil.move(tmp_path, path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
185 |
return path, False |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
186 |
else: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
187 |
return tmp_path, True |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
188 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
189 |
def filename(self, url): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
190 |
"""Determine a file name from a URL according to the configuration. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
191 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
192 |
""" |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
193 |
if self.hash_name: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
194 |
return md5(url).hexdigest() |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
195 |
else: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
196 |
if re.match(r"^[A-Za-z]:\\", url): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
197 |
url = 'file:' + url |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
198 |
parsed = urlparse.urlparse(url, 'file') |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
199 |
url_path = parsed[2] |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
200 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
201 |
if parsed[0] == 'file': |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
202 |
while True: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
203 |
url_path, name = os.path.split(url_path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
204 |
if name: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
205 |
return name |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
206 |
if not url_path: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
207 |
break |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
208 |
else: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
209 |
for name in reversed(url_path.split('/')): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
210 |
if name: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
211 |
return name |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
212 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
213 |
url_host, url_port = parsed[-2:] |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
214 |
return '%s:%s' % (url_host, url_port) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
215 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
216 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
217 |
def check_md5sum(path, md5sum): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
218 |
"""Tell whether the MD5 checksum of the file at path matches. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
219 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
220 |
No checksum being given is considered a match. |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
221 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
222 |
""" |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
223 |
if md5sum is None: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
224 |
return True |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
225 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
226 |
f = open(path, 'rb') |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
227 |
checksum = md5() |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
228 |
try: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
229 |
chunk = f.read(2**16) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
230 |
while chunk: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
231 |
checksum.update(chunk) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
232 |
chunk = f.read(2**16) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
233 |
return checksum.hexdigest() == md5sum |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
234 |
finally: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
235 |
f.close() |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
236 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
237 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
238 |
def remove(path): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
239 |
if os.path.exists(path): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
240 |
os.remove(path) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
241 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
242 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
243 |
def locate_at(source, dest): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
244 |
if dest is None or realpath(dest) == realpath(source): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
245 |
return source |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
246 |
|
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
247 |
try: |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
248 |
os.link(source, dest) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
249 |
except (AttributeError, OSError): |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
250 |
shutil.copyfile(source, dest) |
c6bca38c1cbf
Added buildout stuff and made changes accordingly
Nishanth Amuluru <nishanth@fossee.in>
parents:
diff
changeset
|
251 |
return dest |