[Notes] [Git][BuildStream/buildstream][jonathan/expose-downloadablefilesource] Make DownloadableFileSource clearly define public and private fields



Title: GitLab

Jonathan Maw pushed to branch jonathan/expose-downloadablefilesource at BuildStream / buildstream

Commits:

5 changed files:

Changes:

  • buildstream/downloadablefilesource.py
    ... ... @@ -2,6 +2,13 @@
    2 2
     DownloadableFileSource - Abstract class for downloading files
    
    3 3
     =============================================================
    
    4 4
     A base abstract class for source implementations which download a file.
    
    5
    +
    
    6
    +Derived classes must write their own stage() implementation, using the
    
    7
    +public APIs exposed in this class.
    
    8
    +
    
    9
    +Derived classes must also chain up to the parent method in their get_unique_key()
    
    10
    +implementations.
    
    11
    +
    
    5 12
     """
    
    6 13
     
    
    7 14
     import os
    
    ... ... @@ -19,50 +26,58 @@ class DownloadableFileSource(Source):
    19 26
     
    
    20 27
         COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
    
    21 28
     
    
    29
    +    @property
    
    30
    +    def url(self):
    
    31
    +        return self._url
    
    32
    +
    
    33
    +    #####################################
    
    34
    +    # Implementations of abstract methods
    
    35
    +    #####################################
    
    36
    +
    
    22 37
         def configure(self, node):
    
    23
    -        self.original_url = self.node_get_member(node, str, 'url')
    
    24
    -        self.ref = self.node_get_member(node, str, 'ref', None)
    
    25
    -        self.url = self.translate_url(self.original_url)
    
    26
    -        self._warn_deprecated_etag(node)
    
    38
    +        self._original_url = self.node_get_member(node, str, 'url')
    
    39
    +        self._ref = self.node_get_member(node, str, 'ref', None)
    
    40
    +        self._url = self.translate_url(self._original_url)
    
    41
    +        self.__warn_deprecated_etag(node)
    
    27 42
     
    
    28 43
         def preflight(self):
    
    29 44
             return
    
    30 45
     
    
    31 46
         def get_unique_key(self):
    
    32
    -        return [self.original_url, self.ref]
    
    47
    +        return [self._original_url, self._ref]
    
    33 48
     
    
    34 49
         def get_consistency(self):
    
    35
    -        if self.ref is None:
    
    50
    +        if self._ref is None:
    
    36 51
                 return Consistency.INCONSISTENT
    
    37 52
     
    
    38
    -        if os.path.isfile(self._get_mirror_file()):
    
    53
    +        if os.path.isfile(self.get_mirror_file()):
    
    39 54
                 return Consistency.CACHED
    
    40 55
     
    
    41 56
             else:
    
    42 57
                 return Consistency.RESOLVED
    
    43 58
     
    
    44 59
         def load_ref(self, node):
    
    45
    -        self.ref = self.node_get_member(node, str, 'ref', None)
    
    46
    -        self._warn_deprecated_etag(node)
    
    60
    +        self._ref = self.node_get_member(node, str, 'ref', None)
    
    61
    +        self.__warn_deprecated_etag(node)
    
    47 62
     
    
    48 63
         def get_ref(self):
    
    49
    -        return self.ref
    
    64
    +        return self._ref
    
    50 65
     
    
    51 66
         def set_ref(self, ref, node):
    
    52
    -        node['ref'] = self.ref = ref
    
    67
    +        node['ref'] = self._ref = ref
    
    53 68
     
    
    54 69
         def track(self):
    
    55 70
             # there is no 'track' field in the source to determine what/whether
    
    56 71
             # or not to update refs, because tracking a ref is always a conscious
    
    57 72
             # decision by the user.
    
    58
    -        with self.timed_activity("Tracking {}".format(self.url),
    
    73
    +        with self.timed_activity("Tracking {}".format(self._url),
    
    59 74
                                      silent_nested=True):
    
    60
    -            new_ref = self._ensure_mirror()
    
    75
    +            new_ref = self.ensure_mirror()
    
    61 76
     
    
    62
    -            if self.ref and self.ref != new_ref:
    
    77
    +            if self._ref and self._ref != new_ref:
    
    63 78
                     detail = "When tracking, new ref differs from current ref:\n" \
    
    64
    -                    + "  Tracked URL: {}\n".format(self.url) \
    
    65
    -                    + "  Current ref: {}\n".format(self.ref) \
    
    79
    +                    + "  Tracked URL: {}\n".format(self._url) \
    
    80
    +                    + "  Current ref: {}\n".format(self._ref) \
    
    66 81
                         + "  New ref: {}\n".format(new_ref)
    
    67 82
                     self.warn("Potential man-in-the-middle attack!", detail=detail)
    
    68 83
     
    
    ... ... @@ -74,49 +89,33 @@ class DownloadableFileSource(Source):
    74 89
             # file to be already cached because Source.fetch() will
    
    75 90
             # not be called if the source is already Consistency.CACHED.
    
    76 91
             #
    
    77
    -        if os.path.isfile(self._get_mirror_file()):
    
    92
    +        if os.path.isfile(self.get_mirror_file()):
    
    78 93
                 return  # pragma: nocover
    
    79 94
     
    
    80 95
             # Download the file, raise hell if the sha256sums don't match,
    
    81 96
             # and mirror the file otherwise.
    
    82
    -        with self.timed_activity("Fetching {}".format(self.url), silent_nested=True):
    
    83
    -            sha256 = self._ensure_mirror()
    
    84
    -            if sha256 != self.ref:
    
    97
    +        with self.timed_activity("Fetching {}".format(self._url), silent_nested=True):
    
    98
    +            sha256 = self.ensure_mirror()
    
    99
    +            if sha256 != self._ref:
    
    85 100
                     raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!"
    
    86
    -                                  .format(self.url, sha256, self.ref))
    
    87
    -
    
    88
    -    def _warn_deprecated_etag(self, node):
    
    89
    -        etag = self.node_get_member(node, str, 'etag', None)
    
    90
    -        if etag:
    
    91
    -            provenance = self.node_provenance(node, member_name='etag')
    
    92
    -            self.warn('{} "etag" is deprecated and ignored.'.format(provenance))
    
    93
    -
    
    94
    -    def _get_etag(self, ref):
    
    95
    -        etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
    
    96
    -        if os.path.exists(etagfilename):
    
    97
    -            with open(etagfilename, 'r') as etagfile:
    
    98
    -                return etagfile.read()
    
    99
    -
    
    100
    -        return None
    
    101
    -
    
    102
    -    def _store_etag(self, ref, etag):
    
    103
    -        etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
    
    104
    -        with utils.save_file_atomic(etagfilename) as etagfile:
    
    105
    -            etagfile.write(etag)
    
    101
    +                                  .format(self._url, sha256, self._ref))
    
    102
    +    ################
    
    103
    +    # Public methods
    
    104
    +    ################
    
    106 105
     
    
    107
    -    def _ensure_mirror(self):
    
    106
    +    def ensure_mirror(self):
    
    108 107
             # Downloads from the url and caches it according to its sha256sum.
    
    109 108
             try:
    
    110 109
                 with self.tempdir() as td:
    
    111
    -                default_name = os.path.basename(self.url)
    
    112
    -                request = urllib.request.Request(self.url)
    
    110
    +                default_name = os.path.basename(self._url)
    
    111
    +                request = urllib.request.Request(self._url)
    
    113 112
                     request.add_header('Accept', '*/*')
    
    114 113
     
    
    115 114
                     # We do not use etag in case what we have in cache is
    
    116 115
                     # not matching ref in order to be able to recover from
    
    117 116
                     # corrupted download.
    
    118
    -                if self.ref:
    
    119
    -                    etag = self._get_etag(self.ref)
    
    117
    +                if self._ref:
    
    118
    +                    etag = self.__get_etag(self._ref)
    
    120 119
     
    
    121 120
                         # Do not re-download the file if the ETag matches.
    
    122 121
                         if etag and self.get_consistency() == Consistency.CACHED:
    
    ... ... @@ -134,17 +133,17 @@ class DownloadableFileSource(Source):
    134 133
                             shutil.copyfileobj(response, dest)
    
    135 134
     
    
    136 135
                     # Make sure url-specific mirror dir exists.
    
    137
    -                if not os.path.isdir(self._get_mirror_dir()):
    
    138
    -                    os.makedirs(self._get_mirror_dir())
    
    136
    +                if not os.path.isdir(self.get_mirror_dir()):
    
    137
    +                    os.makedirs(self.get_mirror_dir())
    
    139 138
     
    
    140 139
                     # Store by sha256sum
    
    141 140
                     sha256 = utils.sha256sum(local_file)
    
    142 141
                     # Even if the file already exists, move the new file over.
    
    143 142
                     # In case the old file was corrupted somehow.
    
    144
    -                os.rename(local_file, self._get_mirror_file(sha256))
    
    143
    +                os.rename(local_file, self.get_mirror_file(sha256))
    
    145 144
     
    
    146 145
                     if etag:
    
    147
    -                    self._store_etag(sha256, etag)
    
    146
    +                    self.__store_etag(sha256, etag)
    
    148 147
                     return sha256
    
    149 148
     
    
    150 149
             except urllib.error.HTTPError as e:
    
    ... ... @@ -152,17 +151,40 @@ class DownloadableFileSource(Source):
    152 151
                     # 304 Not Modified.
    
    153 152
                     # Because we use etag only for matching ref, currently specified ref is what
    
    154 153
                     # we would have downloaded.
    
    155
    -                return self.ref
    
    154
    +                return self._ref
    
    156 155
                 raise SourceError("{}: Error mirroring {}: {}"
    
    157
    -                              .format(self, self.url, e), temporary=True) from e
    
    156
    +                              .format(self, self._url, e), temporary=True) from e
    
    158 157
     
    
    159 158
             except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError) as e:
    
    160 159
                 raise SourceError("{}: Error mirroring {}: {}"
    
    161
    -                              .format(self, self.url, e), temporary=True) from e
    
    160
    +                              .format(self, self._url, e), temporary=True) from e
    
    162 161
     
    
    163
    -    def _get_mirror_dir(self):
    
    162
    +    def get_mirror_dir(self):
    
    164 163
             return os.path.join(self.get_mirror_directory(),
    
    165
    -                            utils.url_directory_name(self.original_url))
    
    164
    +                            utils.url_directory_name(self._original_url))
    
    165
    +
    
    166
    +    def get_mirror_file(self, sha=None):
    
    167
    +        return os.path.join(self.get_mirror_dir(), sha or self._ref)
    
    168
    +
    
    169
    +    #######################
    
    170
    +    # Local Private methods
    
    171
    +    #######################
    
    172
    +
    
    173
    +    def __warn_deprecated_etag(self, node):
    
    174
    +        etag = self.node_get_member(node, str, 'etag', None)
    
    175
    +        if etag:
    
    176
    +            provenance = self.node_provenance(node, member_name='etag')
    
    177
    +            self.warn('{} "etag" is deprecated and ignored.'.format(provenance))
    
    166 178
     
    
    167
    -    def _get_mirror_file(self, sha=None):
    
    168
    -        return os.path.join(self._get_mirror_dir(), sha or self.ref)
    179
    +    def __get_etag(self, ref):
    
    180
    +        etagfilename = os.path.join(self.get_mirror_dir(), '{}.etag'.format(ref))
    
    181
    +        if os.path.exists(etagfilename):
    
    182
    +            with open(etagfilename, 'r') as etagfile:
    
    183
    +                return etagfile.read()
    
    184
    +
    
    185
    +        return None
    
    186
    +
    
    187
    +    def __store_etag(self, ref, etag):
    
    188
    +        etagfilename = os.path.join(self.get_mirror_dir(), '{}.etag'.format(ref))
    
    189
    +        with utils.save_file_atomic(etagfilename) as etagfile:
    
    190
    +            etagfile.write(etag)

  • buildstream/plugins/sources/deb.py
    ... ... @@ -70,7 +70,7 @@ class DebSource(TarSource):
    70 70
         @contextmanager
    
    71 71
         def _get_tar(self):
    
    72 72
             with ExitStack() as context:
    
    73
    -            deb_file = context.enter_context(open(self._get_mirror_file(), 'rb'))
    
    73
    +            deb_file = context.enter_context(open(self.get_mirror_file(), 'rb'))
    
    74 74
                 arpy_archive = arpy.Archive(fileobj=deb_file)
    
    75 75
                 arpy_archive.read_all_headers()
    
    76 76
                 data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0]
    

  • buildstream/plugins/sources/remote.py
    ... ... @@ -81,7 +81,7 @@ class RemoteSource(DownloadableFileSource):
    81 81
             dest = os.path.join(directory, self.filename)
    
    82 82
             with self.timed_activity("Staging remote file to {}".format(dest)):
    
    83 83
     
    
    84
    -            utils.safe_copy(self._get_mirror_file(), dest)
    
    84
    +            utils.safe_copy(self.get_mirror_file(), dest)
    
    85 85
     
    
    86 86
                 # To prevent user's umask introducing variability here, explicitly set
    
    87 87
                 # file modes.
    

  • buildstream/plugins/sources/tar.py
    ... ... @@ -88,7 +88,7 @@ class TarSource(DownloadableFileSource):
    88 88
             assert self.host_lzip
    
    89 89
             with TemporaryFile() as lzip_stdout:
    
    90 90
                 with ExitStack() as context:
    
    91
    -                lzip_file = context.enter_context(open(self._get_mirror_file(), 'r'))
    
    91
    +                lzip_file = context.enter_context(open(self.get_mirror_file(), 'r'))
    
    92 92
                     self.call([self.host_lzip, '-d'],
    
    93 93
                               stdin=lzip_file,
    
    94 94
                               stdout=lzip_stdout)
    
    ... ... @@ -103,7 +103,7 @@ class TarSource(DownloadableFileSource):
    103 103
                     with tarfile.open(fileobj=lzip_dec, mode='r:') as tar:
    
    104 104
                         yield tar
    
    105 105
             else:
    
    106
    -            with tarfile.open(self._get_mirror_file()) as tar:
    
    106
    +            with tarfile.open(self.get_mirror_file()) as tar:
    
    107 107
                     yield tar
    
    108 108
     
    
    109 109
         def stage(self, directory):
    

  • buildstream/plugins/sources/zip.py
    ... ... @@ -84,7 +84,7 @@ class ZipSource(DownloadableFileSource):
    84 84
             noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
    
    85 85
     
    
    86 86
             try:
    
    87
    -            with zipfile.ZipFile(self._get_mirror_file()) as archive:
    
    87
    +            with zipfile.ZipFile(self.get_mirror_file()) as archive:
    
    88 88
                     base_dir = None
    
    89 89
                     if self.base_dir:
    
    90 90
                         base_dir = self._find_base_dir(archive, self.base_dir)
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]