[Notes] [Git][BuildStream/buildstream][valentindavid/netrc] Add support for .netrc in remote/tar/zip plugins



Title: GitLab

Valentin David pushed to branch valentindavid/netrc at BuildStream / buildstream

Commits:

1 changed file:

Changes:

  • buildstream/plugins/sources/_downloadablefilesource.py
    ... ... @@ -5,16 +5,81 @@ import urllib.request
    5 5
     import urllib.error
    
    6 6
     import contextlib
    
    7 7
     import shutil
    
    8
    +import netrc
    
    8 9
     
    
    9 10
     from buildstream import Source, SourceError, Consistency
    
    10 11
     from buildstream import utils
    
    11 12
     
    
    12 13
     
    
    14
    +class NetrcFTPOpener(urllib.request.FTPHandler):
    
    15
    +
    
    16
    +    def __init__(self, netrc_config):
    
    17
    +        self.netrc = netrc_config
    
    18
    +
    
    19
    +    def _split(self, netloc):
    
    20
    +        userpass, hostport = urllib.parse.splituser(netloc)
    
    21
    +        host, port = urllib.parse.splitport(hostport)
    
    22
    +        if userpass:
    
    23
    +            user, passwd = urllib.parse.splitpasswd(userpass)
    
    24
    +        else:
    
    25
    +            user = ''
    
    26
    +            passwd = ''
    
    27
    +        return host, port, user, passwd
    
    28
    +
    
    29
    +    def _unsplit(self, host, port, user, passwd):
    
    30
    +        if port:
    
    31
    +            host = '{}:{}'.format(host, port)
    
    32
    +        if user:
    
    33
    +            if passwd:
    
    34
    +                user = '{}:{}'.format(user, passwd)
    
    35
    +            host = '{}@{}'.format(user, host)
    
    36
    +
    
    37
    +        return host
    
    38
    +
    
    39
    +    def ftp_open(self, req):
    
    40
    +        host, port, user, passwd = self._split(req.host)
    
    41
    +
    
    42
    +        if (not user or not passwd) and self.netrc:
    
    43
    +            entry = self.netrc.authenticators(host)
    
    44
    +            if entry:
    
    45
    +                entry_login, _, entry_password = entry
    
    46
    +                if not user:
    
    47
    +                    user = entry_login
    
    48
    +                if not passwd:
    
    49
    +                    passwd = entry_password
    
    50
    +
    
    51
    +        req.host = self._unsplit(host, port, user, passwd)
    
    52
    +
    
    53
    +        return super().ftp_open(req)
    
    54
    +
    
    55
    +
    
    56
    +class NetrcPasswordManager:
    
    57
    +
    
    58
    +    def __init__(self, netrc_config):
    
    59
    +        self.netrc = netrc_config
    
    60
    +
    
    61
    +    def add_password(self, realm, uri, user, passwd):
    
    62
    +        pass
    
    63
    +
    
    64
    +    def find_user_password(self, realm, authuri):
    
    65
    +        if not self.netrc:
    
    66
    +            return None, None
    
    67
    +        parts = urllib.parse.urlsplit(authuri)
    
    68
    +        entry = self.netrc.authenticators(parts.hostname)
    
    69
    +        if not entry:
    
    70
    +            return None, None
    
    71
    +        else:
    
    72
    +            login, _, password = entry
    
    73
    +            return login, password
    
    74
    +
    
    75
    +
    
    13 76
     class DownloadableFileSource(Source):
    
    14 77
         # pylint: disable=attribute-defined-outside-init
    
    15 78
     
    
    16 79
         COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
    
    17 80
     
    
    81
    +    __urlopener = None
    
    82
    +
    
    18 83
         def configure(self, node):
    
    19 84
             self.original_url = self.node_get_member(node, str, 'url')
    
    20 85
             self.ref = self.node_get_member(node, str, 'ref', None)
    
    ... ... @@ -118,7 +183,8 @@ class DownloadableFileSource(Source):
    118 183
                         if etag and self.get_consistency() == Consistency.CACHED:
    
    119 184
                             request.add_header('If-None-Match', etag)
    
    120 185
     
    
    121
    -                with contextlib.closing(urllib.request.urlopen(request)) as response:
    
    186
    +                opener = self.__get_urlopener()
    
    187
    +                with contextlib.closing(opener.open(request)) as response:
    
    122 188
                         info = response.info()
    
    123 189
     
    
    124 190
                         etag = info['ETag'] if 'ETag' in info else None
    
    ... ... @@ -164,3 +230,19 @@ class DownloadableFileSource(Source):
    164 230
     
    
    165 231
         def _get_mirror_file(self, sha=None):
    
    166 232
             return os.path.join(self._get_mirror_dir(), sha or self.ref)
    
    233
    +
    
    234
    +    def __get_urlopener(self):
    
    235
    +        if not DownloadableFileSource.__urlopener:
    
    236
    +            try:
    
    237
    +                netrc_config = netrc.netrc()
    
    238
    +            except FileNotFoundError:
    
    239
    +                _urlopener = urllib.request.build_opener()
    
    240
    +            except netrc.NetrcParseError as e:
    
    241
    +                self.warn('{}: While reading .netrc: {}'.format(self, e))
    
    242
    +                return urllib.request.build_opener()
    
    243
    +            else:
    
    244
    +                netrc_pw_mgr = NetrcPasswordManager(netrc_config)
    
    245
    +                http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr)
    
    246
    +                ftp_handler = NetrcFTPOpener(netrc_config)
    
    247
    +                DownloadableFileSource.__urlopener = urllib.request.build_opener(http_auth, ftp_handler)
    
    248
    +        return DownloadableFileSource.__urlopener



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]