| ... |
... |
@@ -5,16 +5,81 @@ import urllib.request |
|
5
|
5
|
import urllib.error
|
|
6
|
6
|
import contextlib
|
|
7
|
7
|
import shutil
|
|
|
8
|
+import netrc
|
|
8
|
9
|
|
|
9
|
10
|
from buildstream import Source, SourceError, Consistency
|
|
10
|
11
|
from buildstream import utils
|
|
11
|
12
|
|
|
12
|
13
|
|
|
|
14
|
+class NetrcFTPOpener(urllib.request.FTPHandler):
|
|
|
15
|
+
|
|
|
16
|
+ def __init__(self, netrc_config):
|
|
|
17
|
+ self.netrc = netrc_config
|
|
|
18
|
+
|
|
|
19
|
+ def _split(self, netloc):
|
|
|
20
|
+ userpass, hostport = urllib.parse.splituser(netloc)
|
|
|
21
|
+ host, port = urllib.parse.splitport(hostport)
|
|
|
22
|
+ if userpass:
|
|
|
23
|
+ user, passwd = urllib.parse.splitpasswd(userpass)
|
|
|
24
|
+ else:
|
|
|
25
|
+ user = ''
|
|
|
26
|
+ passwd = ''
|
|
|
27
|
+ return host, port, user, passwd
|
|
|
28
|
+
|
|
|
29
|
+ def _unsplit(self, host, port, user, passwd):
|
|
|
30
|
+ if port:
|
|
|
31
|
+ host = '{}:{}'.format(host, port)
|
|
|
32
|
+ if user:
|
|
|
33
|
+ if passwd:
|
|
|
34
|
+ user = '{}:{}'.format(user, passwd)
|
|
|
35
|
+ host = '{}@{}'.format(user, host)
|
|
|
36
|
+
|
|
|
37
|
+ return host
|
|
|
38
|
+
|
|
|
39
|
+ def ftp_open(self, req):
|
|
|
40
|
+ host, port, user, passwd = self._split(req.host)
|
|
|
41
|
+
|
|
|
42
|
+ if (not user or not passwd) and self.netrc:
|
|
|
43
|
+ entry = self.netrc.authenticators(host)
|
|
|
44
|
+ if entry:
|
|
|
45
|
+ entry_login, _, entry_password = entry
|
|
|
46
|
+ if not user:
|
|
|
47
|
+ user = entry_login
|
|
|
48
|
+ if not passwd:
|
|
|
49
|
+ passwd = entry_password
|
|
|
50
|
+
|
|
|
51
|
+ req.host = self._unsplit(host, port, user, passwd)
|
|
|
52
|
+
|
|
|
53
|
+ return super().ftp_open(req)
|
|
|
54
|
+
|
|
|
55
|
+
|
|
|
56
|
+class NetrcPasswordManager:
|
|
|
57
|
+
|
|
|
58
|
+ def __init__(self, netrc_config):
|
|
|
59
|
+ self.netrc = netrc_config
|
|
|
60
|
+
|
|
|
61
|
+ def add_password(self, realm, uri, user, passwd):
|
|
|
62
|
+ pass
|
|
|
63
|
+
|
|
|
64
|
+ def find_user_password(self, realm, authuri):
|
|
|
65
|
+ if not self.netrc:
|
|
|
66
|
+ return None, None
|
|
|
67
|
+ parts = urllib.parse.urlsplit(authuri)
|
|
|
68
|
+ entry = self.netrc.authenticators(parts.hostname)
|
|
|
69
|
+ if not entry:
|
|
|
70
|
+ return None, None
|
|
|
71
|
+ else:
|
|
|
72
|
+ login, _, password = entry
|
|
|
73
|
+ return login, password
|
|
|
74
|
+
|
|
|
75
|
+
|
|
13
|
76
|
class DownloadableFileSource(Source):
|
|
14
|
77
|
# pylint: disable=attribute-defined-outside-init
|
|
15
|
78
|
|
|
16
|
79
|
COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
|
|
17
|
80
|
|
|
|
81
|
+ __urlopener = None
|
|
|
82
|
+
|
|
18
|
83
|
def configure(self, node):
|
|
19
|
84
|
self.original_url = self.node_get_member(node, str, 'url')
|
|
20
|
85
|
self.ref = self.node_get_member(node, str, 'ref', None)
|
| ... |
... |
@@ -118,7 +183,8 @@ class DownloadableFileSource(Source): |
|
118
|
183
|
if etag and self.get_consistency() == Consistency.CACHED:
|
|
119
|
184
|
request.add_header('If-None-Match', etag)
|
|
120
|
185
|
|
|
121
|
|
- with contextlib.closing(urllib.request.urlopen(request)) as response:
|
|
|
186
|
+ opener = self.__get_urlopener()
|
|
|
187
|
+ with contextlib.closing(opener.open(request)) as response:
|
|
122
|
188
|
info = response.info()
|
|
123
|
189
|
|
|
124
|
190
|
etag = info['ETag'] if 'ETag' in info else None
|
| ... |
... |
@@ -164,3 +230,19 @@ class DownloadableFileSource(Source): |
|
164
|
230
|
|
|
165
|
231
|
def _get_mirror_file(self, sha=None):
|
|
166
|
232
|
return os.path.join(self._get_mirror_dir(), sha or self.ref)
|
|
|
233
|
+
|
|
|
234
|
+ def __get_urlopener(self):
|
|
|
235
|
+ if not DownloadableFileSource.__urlopener:
|
|
|
236
|
+ try:
|
|
|
237
|
+ netrc_config = netrc.netrc()
|
|
|
238
|
+ except FileNotFoundError:
|
|
|
239
|
+ DownloadableFileSource.__urlopener = urllib.request.build_opener()
|
|
|
240
|
+ except netrc.NetrcParseError as e:
|
|
|
241
|
+ self.warn('{}: While reading .netrc: {}'.format(self, e))
|
|
|
242
|
+ return urllib.request.build_opener()
|
|
|
243
|
+ else:
|
|
|
244
|
+ netrc_pw_mgr = NetrcPasswordManager(netrc_config)
|
|
|
245
|
+ http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr)
|
|
|
246
|
+ ftp_handler = NetrcFTPOpener(netrc_config)
|
|
|
247
|
+ DownloadableFileSource.__urlopener = urllib.request.build_opener(http_auth, ftp_handler)
|
|
|
248
|
+ return DownloadableFileSource.__urlopener
|
