GNOME.org
 

[Notes] [Git][BuildStream/buildstream][jonathan/expose-downloadablefilesource] Make DownloadableFileSource clearly define public and private fields

Title: GitLab

Jonathan Maw pushed to branch jonathan/expose-downloadablefilesource at BuildStream / buildstream

Commits:

5 changed files:

Changes:

  • buildstream/downloadablefilesource.py
    ... ... @@ -2,6 +2,13 @@
    2 2 
     DownloadableFileSource - Abstract class for downloading files
    3 3 
     =============================================================
    4 4 
     A base abstract class for source implementations which download a file.
    5 
    +
    6 
    +Derived classes must write their own stage() implementation, using the
    7 
    +public APIs exposed in this class.
    8 
    +
    9 
    +Derived classes must also chain up to the parent method in their get_unique_key()
    10 
    +implementations.
    11 
    +
    5 12 
     """
    6 13
    7 14 
     import os
    ... ... @@ -19,50 +26,58 @@ class DownloadableFileSource(Source):
    19 26
    20 27 
         COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
    21 28
    29 
    +    @property
    30 
    +    def url(self):
    31 
    +        return self._url
    32 
    +
    33 
    +    #####################################
    34 
    +    # Implementations of abstract methods
    35 
    +    #####################################
    36 
    +
    22 37 
         def configure(self, node):
    23 
    -        self.original_url = self.node_get_member(node, str, 'url')
    24 
    -        self.ref = self.node_get_member(node, str, 'ref', None)
    25 
    -        self.url = self.translate_url(self.original_url)
    26 
    -        self._warn_deprecated_etag(node)
    38 
    +        self._original_url = self.node_get_member(node, str, 'url')
    39 
    +        self._ref = self.node_get_member(node, str, 'ref', None)
    40 
    +        self._url = self.translate_url(self._original_url)
    41 
    +        self.__warn_deprecated_etag(node)
    27 42
    28 43 
         def preflight(self):
    29 44 
             return
    30 45
    31 46 
         def get_unique_key(self):
    32 
    -        return [self.original_url, self.ref]
    47 
    +        return [self._original_url, self._ref]
    33 48
    34 49 
         def get_consistency(self):
    35 
    -        if self.ref is None:
    50 
    +        if self._ref is None:
    36 51 
                 return Consistency.INCONSISTENT
    37 52
    38 
    -        if os.path.isfile(self._get_mirror_file()):
    53 
    +        if os.path.isfile(self.get_mirror_file()):
    39 54 
                 return Consistency.CACHED
    40 55
    41 56 
             else:
    42 57 
                 return Consistency.RESOLVED
    43 58
    44 59 
         def load_ref(self, node):
    45 
    -        self.ref = self.node_get_member(node, str, 'ref', None)
    46 
    -        self._warn_deprecated_etag(node)
    60 
    +        self._ref = self.node_get_member(node, str, 'ref', None)
    61 
    +        self.__warn_deprecated_etag(node)
    47 62
    48 63 
         def get_ref(self):
    49 
    -        return self.ref
    64 
    +        return self._ref
    50 65
    51 66 
         def set_ref(self, ref, node):
    52 
    -        node['ref'] = self.ref = ref
    67 
    +        node['ref'] = self._ref = ref
    53 68
    54 69 
         def track(self):
    55 70 
             # there is no 'track' field in the source to determine what/whether
    56 71 
             # or not to update refs, because tracking a ref is always a conscious
    57 72 
             # decision by the user.
    58 
    -        with self.timed_activity("Tracking {}".format(self.url),
    73 
    +        with self.timed_activity("Tracking {}".format(self._url),
    59 74 
                                      silent_nested=True):
    60 
    -            new_ref = self._ensure_mirror()
    75 
    +            new_ref = self.ensure_mirror()
    61 76
    62 
    -            if self.ref and self.ref != new_ref:
    77 
    +            if self._ref and self._ref != new_ref:
    63 78 
                     detail = "When tracking, new ref differs from current ref:\n" \
    64 
    -                    + "  Tracked URL: {}\n".format(self.url) \
    65 
    -                    + "  Current ref: {}\n".format(self.ref) \
    79 
    +                    + "  Tracked URL: {}\n".format(self._url) \
    80 
    +                    + "  Current ref: {}\n".format(self._ref) \
    66 81 
                         + "  New ref: {}\n".format(new_ref)
    67 82 
                     self.warn("Potential man-in-the-middle attack!", detail=detail)
    68 83
    ... ... @@ -74,49 +89,33 @@ class DownloadableFileSource(Source):
    74 89 
             # file to be already cached because Source.fetch() will
    75 90 
             # not be called if the source is already Consistency.CACHED.
    76 91 
             #
    77 
    -        if os.path.isfile(self._get_mirror_file()):
    92 
    +        if os.path.isfile(self.get_mirror_file()):
    78 93 
                 return  # pragma: nocover
    79 94
    80 95 
             # Download the file, raise hell if the sha256sums don't match,
    81 96 
             # and mirror the file otherwise.
    82 
    -        with self.timed_activity("Fetching {}".format(self.url), silent_nested=True):
    83 
    -            sha256 = self._ensure_mirror()
    84 
    -            if sha256 != self.ref:
    97 
    +        with self.timed_activity("Fetching {}".format(self._url), silent_nested=True):
    98 
    +            sha256 = self.ensure_mirror()
    99 
    +            if sha256 != self._ref:
    85 100 
                     raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!"
    86 
    -                                  .format(self.url, sha256, self.ref))
    87 
    -
    88 
    -    def _warn_deprecated_etag(self, node):
    89 
    -        etag = self.node_get_member(node, str, 'etag', None)
    90 
    -        if etag:
    91 
    -            provenance = self.node_provenance(node, member_name='etag')
    92 
    -            self.warn('{} "etag" is deprecated and ignored.'.format(provenance))
    93 
    -
    94 
    -    def _get_etag(self, ref):
    95 
    -        etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
    96 
    -        if os.path.exists(etagfilename):
    97 
    -            with open(etagfilename, 'r') as etagfile:
    98 
    -                return etagfile.read()
    99 
    -
    100 
    -        return None
    101 
    -
    102 
    -    def _store_etag(self, ref, etag):
    103 
    -        etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
    104 
    -        with utils.save_file_atomic(etagfilename) as etagfile:
    105 
    -            etagfile.write(etag)
    101 
    +                                  .format(self._url, sha256, self._ref))
    102 
    +    ################
    103 
    +    # Public methods
    104 
    +    ################
    106 105
    107 
    -    def _ensure_mirror(self):
    106 
    +    def ensure_mirror(self):
    108 107 
             # Downloads from the url and caches it according to its sha256sum.
    109 108 
             try:
    110 109 
                 with self.tempdir() as td:
    111 
    -                default_name = os.path.basename(self.url)
    112 
    -                request = urllib.request.Request(self.url)
    110 
    +                default_name = os.path.basename(self._url)
    111 
    +                request = urllib.request.Request(self._url)
    113 112 
                     request.add_header('Accept', '*/*')
    114 113
    115 114 
                     # We do not use etag in case what we have in cache is
    116 115 
                     # not matching ref in order to be able to recover from
    117 116 
                     # corrupted download.
    118 
    -                if self.ref:
    119 
    -                    etag = self._get_etag(self.ref)
    117 
    +                if self._ref:
    118 
    +                    etag = self.__get_etag(self._ref)
    120 119
    121 120 
                         # Do not re-download the file if the ETag matches.
    122 121 
                         if etag and self.get_consistency() == Consistency.CACHED:
    ... ... @@ -134,17 +133,17 @@ class DownloadableFileSource(Source):
    134 133 
                             shutil.copyfileobj(response, dest)
    135 134
    136 135 
                     # Make sure url-specific mirror dir exists.
    137 
    -                if not os.path.isdir(self._get_mirror_dir()):
    138 
    -                    os.makedirs(self._get_mirror_dir())
    136 
    +                if not os.path.isdir(self.get_mirror_dir()):
    137 
    +                    os.makedirs(self.get_mirror_dir())
    139 138
    140 139 
                     # Store by sha256sum
    141 140 
                     sha256 = utils.sha256sum(local_file)
    142 141 
                     # Even if the file already exists, move the new file over.
    143 142 
                     # In case the old file was corrupted somehow.
    144 
    -                os.rename(local_file, self._get_mirror_file(sha256))
    143 
    +                os.rename(local_file, self.get_mirror_file(sha256))
    145 144
    146 145 
                     if etag:
    147 
    -                    self._store_etag(sha256, etag)
    146 
    +                    self.__store_etag(sha256, etag)
    148 147 
                     return sha256
    149 148
    150 149 
             except urllib.error.HTTPError as e:
    ... ... @@ -152,17 +151,40 @@ class DownloadableFileSource(Source):
    152 151 
                     # 304 Not Modified.
    153 152 
                     # Because we use etag only for matching ref, currently specified ref is what
    154 153 
                     # we would have downloaded.
    155 
    -                return self.ref
    154 
    +                return self._ref
    156 155 
                 raise SourceError("{}: Error mirroring {}: {}"
    157 
    -                              .format(self, self.url, e), temporary=True) from e
    156 
    +                              .format(self, self._url, e), temporary=True) from e
    158 157
    159 158 
             except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError) as e:
    160 159 
                 raise SourceError("{}: Error mirroring {}: {}"
    161 
    -                              .format(self, self.url, e), temporary=True) from e
    160 
    +                              .format(self, self._url, e), temporary=True) from e
    162 161
    163 
    -    def _get_mirror_dir(self):
    162 
    +    def get_mirror_dir(self):
    164 163 
             return os.path.join(self.get_mirror_directory(),
    165 
    -                            utils.url_directory_name(self.original_url))
    164 
    +                            utils.url_directory_name(self._original_url))
    165 
    +
    166 
    +    def get_mirror_file(self, sha=None):
    167 
    +        return os.path.join(self.get_mirror_dir(), sha or self._ref)
    168 
    +
    169 
    +    #######################
    170 
    +    # Local Private methods
    171 
    +    #######################
    172 
    +
    173 
    +    def __warn_deprecated_etag(self, node):
    174 
    +        etag = self.node_get_member(node, str, 'etag', None)
    175 
    +        if etag:
    176 
    +            provenance = self.node_provenance(node, member_name='etag')
    177 
    +            self.warn('{} "etag" is deprecated and ignored.'.format(provenance))
    166 178
    167 
    -    def _get_mirror_file(self, sha=None):
    168 
    -        return os.path.join(self._get_mirror_dir(), sha or self.ref)
    179 
    +    def __get_etag(self, ref):
    180 
    +        etagfilename = os.path.join(self.get_mirror_dir(), '{}.etag'.format(ref))
    181 
    +        if os.path.exists(etagfilename):
    182 
    +            with open(etagfilename, 'r') as etagfile:
    183 
    +                return etagfile.read()
    184 
    +
    185 
    +        return None
    186 
    +
    187 
    +    def __store_etag(self, ref, etag):
    188 
    +        etagfilename = os.path.join(self.get_mirror_dir(), '{}.etag'.format(ref))
    189 
    +        with utils.save_file_atomic(etagfilename) as etagfile:
    190 
    +            etagfile.write(etag)

  • buildstream/plugins/sources/deb.py
    ... ... @@ -70,7 +70,7 @@ class DebSource(TarSource):
    70 70 
         @contextmanager
    71 71 
         def _get_tar(self):
    72 72 
             with ExitStack() as context:
    73 
    -            deb_file = context.enter_context(open(self._get_mirror_file(), 'rb'))
    73 
    +            deb_file = context.enter_context(open(self.get_mirror_file(), 'rb'))
    74 74 
                 arpy_archive = arpy.Archive(fileobj=deb_file)
    75 75 
                 arpy_archive.read_all_headers()
    76 76 
                 data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0]

  • buildstream/plugins/sources/remote.py
    ... ... @@ -81,7 +81,7 @@ class RemoteSource(DownloadableFileSource):
    81 81 
             dest = os.path.join(directory, self.filename)
    82 82 
             with self.timed_activity("Staging remote file to {}".format(dest)):
    83 83
    84 
    -            utils.safe_copy(self._get_mirror_file(), dest)
    84 
    +            utils.safe_copy(self.get_mirror_file(), dest)
    85 85
    86 86 
                 # To prevent user's umask introducing variability here, explicitly set
    87 87 
                 # file modes.

  • buildstream/plugins/sources/tar.py
    ... ... @@ -88,7 +88,7 @@ class TarSource(DownloadableFileSource):
    88 88 
             assert self.host_lzip
    89 89 
             with TemporaryFile() as lzip_stdout:
    90 90 
                 with ExitStack() as context:
    91 
    -                lzip_file = context.enter_context(open(self._get_mirror_file(), 'r'))
    91 
    +                lzip_file = context.enter_context(open(self.get_mirror_file(), 'r'))
    92 92 
                     self.call([self.host_lzip, '-d'],
    93 93 
                               stdin=lzip_file,
    94 94 
                               stdout=lzip_stdout)
    ... ... @@ -103,7 +103,7 @@ class TarSource(DownloadableFileSource):
    103 103 
                     with tarfile.open(fileobj=lzip_dec, mode='r:') as tar:
    104 104 
                         yield tar
    105 105 
             else:
    106 
    -            with tarfile.open(self._get_mirror_file()) as tar:
    106 
    +            with tarfile.open(self.get_mirror_file()) as tar:
    107 107 
                     yield tar
    108 108
    109 109 
         def stage(self, directory):

  • buildstream/plugins/sources/zip.py
    ... ... @@ -84,7 +84,7 @@ class ZipSource(DownloadableFileSource):
    84 84 
             noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
    85 85
    86 86 
             try:
    87 
    -            with zipfile.ZipFile(self._get_mirror_file()) as archive:
    87 
    +            with zipfile.ZipFile(self.get_mirror_file()) as archive:
    88 88 
                     base_dir = None
    89 89 
                     if self.base_dir:
    90 90 
                         base_dir = self._find_base_dir(archive, self.base_dir)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]