[Notes] [Git][BuildStream/buildstream][tpollard/494] 8 commits: _artifactcache: Rename update_atime() to update_mtime()



Title: GitLab

Tom Pollard pushed to branch tpollard/494 at BuildStream / buildstream

Commits:

11 changed files:

Changes:

  • NEWS
    ... ... @@ -31,6 +31,17 @@ buildstream 1.3.1
    31 31
         new the `conf-root` variable to make the process easier. And there has been
    
    32 32
         a bug fix to workspaces so they can be build in workspaces too.
    
    33 33
     
    
    34
    +  o Due to the element `buildtree` being cached in the respective artifact their
    
    35
    +    size in some cases has significantly increased. In *most* cases the buildtree
    
    36
    +    is not utilised when building targets, as such by default bst 'pull' & 'build'
    
    37
    +    will not fetch buildtrees from remotes. This behaviour can be overriden with
    
    38
    +    the cli option '--pull-buildtrees', or the user configuration option
    
    39
    +    'pullbuildtrees = True'. The override will also add the buildtree to already
    
    40
    +    cached artifacts. When attempting to populate an artifactcache server with
    
    41
    +    cached artifacts, only 'complete' elements can be pushed. If the element is
    
    42
    +    expected to have a populated buildtree then it must be cached before pushing.
    
    43
    +
    
    44
    +
    
    34 45
     =================
    
    35 46
     buildstream 1.1.5
    
    36 47
     =================
    

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -228,7 +228,7 @@ class ArtifactCache():
    228 228
             self._required_elements.update(elements)
    
    229 229
     
    
    230 230
             # For the cache keys which were resolved so far, we bump
    
    231
    -        # the atime of them.
    
    231
    +        # the mtime of them.
    
    232 232
             #
    
    233 233
             # This is just in case we have concurrent instances of
    
    234 234
             # BuildStream running with the same artifact cache, it will
    
    ... ... @@ -240,7 +240,7 @@ class ArtifactCache():
    240 240
                 for key in (strong_key, weak_key):
    
    241 241
                     if key:
    
    242 242
                         try:
    
    243
    -                        self.update_atime(key)
    
    243
    +                        self.update_mtime(element, key)
    
    244 244
                         except ArtifactError:
    
    245 245
                             pass
    
    246 246
     
    
    ... ... @@ -391,15 +391,16 @@ class ArtifactCache():
    391 391
         def preflight(self):
    
    392 392
             pass
    
    393 393
     
    
    394
    -    # update_atime()
    
    394
    +    # update_mtime()
    
    395 395
         #
    
    396
    -    # Update the atime of an artifact.
    
    396
    +    # Update the mtime of an artifact.
    
    397 397
         #
    
    398 398
         # Args:
    
    399
    +    #     element (Element): The Element to update
    
    399 400
         #     key (str): The key of the artifact.
    
    400 401
         #
    
    401
    -    def update_atime(self, key):
    
    402
    -        raise ImplError("Cache '{kind}' does not implement contains()"
    
    402
    +    def update_mtime(self, element, key):
    
    403
    +        raise ImplError("Cache '{kind}' does not implement update_mtime()"
    
    403 404
                             .format(kind=type(self).__name__))
    
    404 405
     
    
    405 406
         # initialize_remotes():
    
    ... ... @@ -427,6 +428,22 @@ class ArtifactCache():
    427 428
             raise ImplError("Cache '{kind}' does not implement contains()"
    
    428 429
                             .format(kind=type(self).__name__))
    
    429 430
     
    
    431
    +    # contains_subdir_artifact():
    
    432
    +    #
    
    433
    +    # Check whether an artifact element contains a digest for a subdir
    
    434
    +    # which is populated in the cache, i.e non dangling.
    
    435
    +    #
    
    436
    +    # Args:
    
    437
    +    #     element (Element): The Element to check
    
    438
    +    #     key (str): The cache key to use
    
    439
    +    #     subdir (str): The subdir to check
    
    440
    +    #
    
    441
    +    # Returns: True if the subdir exists & is populated in the cache, False otherwise
    
    442
    +    #
    
    443
    +    def contains_subdir_artifact(self, element, key, subdir):
    
    444
    +        raise ImplError("Cache '{kind}' does not implement contains_subdir_artifact()"
    
    445
    +                        .format(kind=type(self).__name__))
    
    446
    +
    
    430 447
         # list_artifacts():
    
    431 448
         #
    
    432 449
         # List artifacts in this cache in LRU order.
    
    ... ... @@ -463,6 +480,7 @@ class ArtifactCache():
    463 480
         # Args:
    
    464 481
         #     element (Element): The Element to extract
    
    465 482
         #     key (str): The cache key to use
    
    483
    +    #     subdir (str): The optional subdir to check exists
    
    466 484
         #
    
    467 485
         # Raises:
    
    468 486
         #     ArtifactError: In cases there was an OSError, or if the artifact
    
    ... ... @@ -470,7 +488,7 @@ class ArtifactCache():
    470 488
         #
    
    471 489
         # Returns: path to extracted artifact
    
    472 490
         #
    
    473
    -    def extract(self, element, key):
    
    491
    +    def extract(self, element, key, subdir=None):
    
    474 492
             raise ImplError("Cache '{kind}' does not implement extract()"
    
    475 493
                             .format(kind=type(self).__name__))
    
    476 494
     
    
    ... ... @@ -552,11 +570,13 @@ class ArtifactCache():
    552 570
         #     element (Element): The Element whose artifact is to be fetched
    
    553 571
         #     key (str): The cache key to use
    
    554 572
         #     progress (callable): The progress callback, if any
    
    573
    +    #     subdir (str): The optional specific subdir to pull
    
    574
    +    #     excluded_subdirs (list): The optional list of subdirs to not pull
    
    555 575
         #
    
    556 576
         # Returns:
    
    557 577
         #   (bool): True if pull was successful, False if artifact was not available
    
    558 578
         #
    
    559
    -    def pull(self, element, key, *, progress=None):
    
    579
    +    def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
    
    560 580
             raise ImplError("Cache '{kind}' does not implement pull()"
    
    561 581
                             .format(kind=type(self).__name__))
    
    562 582
     
    

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -92,16 +92,36 @@ class CASCache(ArtifactCache):
    92 92
             # This assumes that the repository doesn't have any dangling pointers
    
    93 93
             return os.path.exists(refpath)
    
    94 94
     
    
    95
    -    def extract(self, element, key):
    
    95
    +    def contains_subdir_artifact(self, element, key, subdir):
    
    96
    +        tree = self.resolve_ref(self.get_artifact_fullname(element, key))
    
    97
    +
    
    98
    +        # This assumes that the subdir digest is present in the element tree
    
    99
    +        subdirdigest = self._get_subdir(tree, subdir)
    
    100
    +        objpath = self.objpath(subdirdigest)
    
    101
    +
    
    102
    +        # True if subdir content is cached or if empty as expected
    
    103
    +        return os.path.exists(objpath)
    
    104
    +
    
    105
    +    def extract(self, element, key, subdir=None):
    
    96 106
             ref = self.get_artifact_fullname(element, key)
    
    97 107
     
    
    98 108
             tree = self.resolve_ref(ref, update_mtime=True)
    
    99 109
     
    
    100
    -        dest = os.path.join(self.extractdir, element._get_project().name,
    
    101
    -                            element.normal_name, tree.hash)
    
    110
    +        dest = elementdest = os.path.join(self.extractdir, element._get_project().name,
    
    111
    +                                          element.normal_name, tree.hash)
    
    112
    +
    
    102 113
             if os.path.isdir(dest):
    
    103
    -            # artifact has already been extracted
    
    104
    -            return dest
    
    114
    +            if subdir:
    
    115
    +                # Check if we have optional subdir in the local cache and not already extracted
    
    116
    +                subdircached = self.contains_subdir_artifact(element, key, subdir)
    
    117
    +                if subdircached and not os.path.isdir(os.path.join(dest, subdir)):
    
    118
    +                    # Artifact has already been extracted without subdir content, only need to checkout the subdir
    
    119
    +                    tree = self._get_subdir(tree, subdir)
    
    120
    +                    dest = os.path.join(dest, subdir)
    
    121
    +                else:
    
    122
    +                    return dest
    
    123
    +            else:
    
    124
    +                return dest
    
    105 125
     
    
    106 126
             with tempfile.TemporaryDirectory(prefix='tmp', dir=self.extractdir) as tmpdir:
    
    107 127
                 checkoutdir = os.path.join(tmpdir, ref)
    
    ... ... @@ -120,7 +140,7 @@ class CASCache(ArtifactCache):
    120 140
                         raise ArtifactError("Failed to extract artifact for ref '{}': {}"
    
    121 141
                                             .format(ref, e)) from e
    
    122 142
     
    
    123
    -        return dest
    
    143
    +        return elementdest
    
    124 144
     
    
    125 145
         def commit(self, element, content, keys):
    
    126 146
             refs = [self.get_artifact_fullname(element, key) for key in keys]
    
    ... ... @@ -228,7 +248,7 @@ class CASCache(ArtifactCache):
    228 248
                 remotes_for_project = self._remotes[element._get_project()]
    
    229 249
                 return any(remote.spec.push for remote in remotes_for_project)
    
    230 250
     
    
    231
    -    def pull(self, element, key, *, progress=None):
    
    251
    +    def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
    
    232 252
             ref = self.get_artifact_fullname(element, key)
    
    233 253
     
    
    234 254
             project = element._get_project()
    
    ... ... @@ -247,8 +267,14 @@ class CASCache(ArtifactCache):
    247 267
                     tree.hash = response.digest.hash
    
    248 268
                     tree.size_bytes = response.digest.size_bytes
    
    249 269
     
    
    250
    -                self._fetch_directory(remote, tree)
    
    270
    +                # Check if the element artifact is present, if so just fetch subdir
    
    271
    +                if subdir and os.path.exists(self.objpath(tree)):
    
    272
    +                    self._fetch_subdir(remote, tree, subdir)
    
    273
    +                else:
    
    274
    +                    # Fetch artifact, excluded_subdirs determined in pullqueue
    
    275
    +                    self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
    
    251 276
     
    
    277
    +                # tree is the remote value, so is the same without or without dangling ref locally
    
    252 278
                     self.set_ref(ref, tree)
    
    253 279
     
    
    254 280
                     element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
    
    ... ... @@ -538,8 +564,9 @@ class CASCache(ArtifactCache):
    538 564
             except FileNotFoundError as e:
    
    539 565
                 raise ArtifactError("Attempt to access unavailable artifact: {}".format(e)) from e
    
    540 566
     
    
    541
    -    def update_atime(self, ref):
    
    567
    +    def update_mtime(self, element, key):
    
    542 568
             try:
    
    569
    +            ref = self.get_artifact_fullname(element, key)
    
    543 570
                 os.utime(self._refpath(ref))
    
    544 571
             except FileNotFoundError as e:
    
    545 572
                 raise ArtifactError("Attempt to access unavailable artifact: {}".format(e)) from e
    
    ... ... @@ -671,8 +698,10 @@ class CASCache(ArtifactCache):
    671 698
                              stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
    
    672 699
     
    
    673 700
             for dirnode in directory.directories:
    
    674
    -            fullpath = os.path.join(dest, dirnode.name)
    
    675
    -            self._checkout(fullpath, dirnode.digest)
    
    701
    +            # Don't try to checkout a dangling ref
    
    702
    +            if os.path.exists(self.objpath(dirnode.digest)):
    
    703
    +                fullpath = os.path.join(dest, dirnode.name)
    
    704
    +                self._checkout(fullpath, dirnode.digest)
    
    676 705
     
    
    677 706
             for symlinknode in directory.symlinks:
    
    678 707
                 # symlink
    
    ... ... @@ -950,11 +979,14 @@ class CASCache(ArtifactCache):
    950 979
         # Args:
    
    951 980
         #     remote (Remote): The remote to use.
    
    952 981
         #     dir_digest (Digest): Digest object for the directory to fetch.
    
    982
    +    #     excluded_subdirs (list): The optional list of subdirs to not fetch
    
    953 983
         #
    
    954
    -    def _fetch_directory(self, remote, dir_digest):
    
    984
    +    def _fetch_directory(self, remote, dir_digest, *, excluded_subdirs=None):
    
    955 985
             fetch_queue = [dir_digest]
    
    956 986
             fetch_next_queue = []
    
    957 987
             batch = _CASBatchRead(remote)
    
    988
    +        if not excluded_subdirs:
    
    989
    +            excluded_subdirs = []
    
    958 990
     
    
    959 991
             while len(fetch_queue) + len(fetch_next_queue) > 0:
    
    960 992
                 if len(fetch_queue) == 0:
    
    ... ... @@ -969,8 +1001,9 @@ class CASCache(ArtifactCache):
    969 1001
                     directory.ParseFromString(f.read())
    
    970 1002
     
    
    971 1003
                 for dirnode in directory.directories:
    
    972
    -                batch = self._fetch_directory_node(remote, dirnode.digest, batch,
    
    973
    -                                                   fetch_queue, fetch_next_queue, recursive=True)
    
    1004
    +                if dirnode.name not in excluded_subdirs:
    
    1005
    +                    batch = self._fetch_directory_node(remote, dirnode.digest, batch,
    
    1006
    +                                                       fetch_queue, fetch_next_queue, recursive=True)
    
    974 1007
     
    
    975 1008
                 for filenode in directory.files:
    
    976 1009
                     batch = self._fetch_directory_node(remote, filenode.digest, batch,
    
    ... ... @@ -979,6 +1012,10 @@ class CASCache(ArtifactCache):
    979 1012
             # Fetch final batch
    
    980 1013
             self._fetch_directory_batch(remote, batch, fetch_queue, fetch_next_queue)
    
    981 1014
     
    
    1015
    +    def _fetch_subdir(self, remote, tree, subdir):
    
    1016
    +        subdirdigest = self._get_subdir(tree, subdir)
    
    1017
    +        self._fetch_directory(remote, subdirdigest)
    
    1018
    +
    
    982 1019
         def _fetch_tree(self, remote, digest):
    
    983 1020
             # download but do not store the Tree object
    
    984 1021
             with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
    

  • buildstream/_context.py
    ... ... @@ -111,6 +111,9 @@ class Context():
    111 111
             # Make sure the XDG vars are set in the environment before loading anything
    
    112 112
             self._init_xdg()
    
    113 113
     
    
    114
    +        # Whether or not to attempt to pull buildtrees globally
    
    115
    +        self.pullbuildtrees = False
    
    116
    +
    
    114 117
             # Private variables
    
    115 118
             self._cache_key = None
    
    116 119
             self._message_handler = None
    
    ... ... @@ -161,7 +164,7 @@ class Context():
    161 164
             _yaml.node_validate(defaults, [
    
    162 165
                 'sourcedir', 'builddir', 'artifactdir', 'logdir',
    
    163 166
                 'scheduler', 'artifacts', 'logging', 'projects',
    
    164
    -            'cache'
    
    167
    +            'cache', 'pullbuildtrees'
    
    165 168
             ])
    
    166 169
     
    
    167 170
             for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
    
    ... ... @@ -186,6 +189,9 @@ class Context():
    186 189
             # Load artifact share configuration
    
    187 190
             self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
    
    188 191
     
    
    192
    +        # Load pull buildtrees configuration
    
    193
    +        self.pullbuildtrees = _yaml.node_get(defaults, bool, 'pullbuildtrees', default_value='False')
    
    194
    +
    
    189 195
             # Load logging config
    
    190 196
             logging = _yaml.node_get(defaults, Mapping, 'logging')
    
    191 197
             _yaml.node_validate(logging, [
    

  • buildstream/_frontend/cli.py
    ... ... @@ -305,10 +305,12 @@ def init(app, project_name, format_version, element_path, force):
    305 305
                   help="Allow tracking to cross junction boundaries")
    
    306 306
     @click.option('--track-save', default=False, is_flag=True,
    
    307 307
                   help="Deprecated: This is ignored")
    
    308
    +@click.option('--pull-buildtrees', default=False, is_flag=True,
    
    309
    +              help="Pull buildtrees from a remote cache server")
    
    308 310
     @click.argument('elements', nargs=-1,
    
    309 311
                     type=click.Path(readable=False))
    
    310 312
     @click.pass_obj
    
    311
    -def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
    
    313
    +def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions, pull_buildtrees):
    
    312 314
         """Build elements in a pipeline"""
    
    313 315
     
    
    314 316
         if (track_except or track_cross_junctions) and not (track_ or track_all):
    
    ... ... @@ -327,7 +329,8 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
    327 329
                              track_targets=track_,
    
    328 330
                              track_except=track_except,
    
    329 331
                              track_cross_junctions=track_cross_junctions,
    
    330
    -                         build_all=all_)
    
    332
    +                         build_all=all_,
    
    333
    +                         pull_buildtrees=pull_buildtrees)
    
    331 334
     
    
    332 335
     
    
    333 336
     ##################################################################
    
    ... ... @@ -429,10 +432,12 @@ def track(app, elements, deps, except_, cross_junctions):
    429 432
                   help='The dependency artifacts to pull (default: none)')
    
    430 433
     @click.option('--remote', '-r',
    
    431 434
                   help="The URL of the remote cache (defaults to the first configured cache)")
    
    435
    +@click.option('--pull-buildtrees', default=False, is_flag=True,
    
    436
    +              help="Pull buildtrees from a remote cache server")
    
    432 437
     @click.argument('elements', nargs=-1,
    
    433 438
                     type=click.Path(readable=False))
    
    434 439
     @click.pass_obj
    
    435
    -def pull(app, elements, deps, remote):
    
    440
    +def pull(app, elements, deps, remote, pull_buildtrees):
    
    436 441
         """Pull a built artifact from the configured remote artifact cache.
    
    437 442
     
    
    438 443
         By default the artifact will be pulled one of the configured caches
    
    ... ... @@ -446,7 +451,7 @@ def pull(app, elements, deps, remote):
    446 451
             all:   All dependencies
    
    447 452
         """
    
    448 453
         with app.initialized(session_name="Pull"):
    
    449
    -        app.stream.pull(elements, selection=deps, remote=remote)
    
    454
    +        app.stream.pull(elements, selection=deps, remote=remote, pull_buildtrees=pull_buildtrees)
    
    450 455
     
    
    451 456
     
    
    452 457
     ##################################################################
    

  • buildstream/_scheduler/queues/pullqueue.py
    ... ... @@ -32,9 +32,20 @@ class PullQueue(Queue):
    32 32
         complete_name = "Pulled"
    
    33 33
         resources = [ResourceType.DOWNLOAD, ResourceType.CACHE]
    
    34 34
     
    
    35
    +    def __init__(self, scheduler, buildtrees=False):
    
    36
    +        super().__init__(scheduler)
    
    37
    +
    
    38
    +        # Current default exclusions on pull
    
    39
    +        self._excluded_subdirs = ["buildtree"]
    
    40
    +        self._subdir = None
    
    41
    +        # If buildtrees are to be pulled, remove the value from exclusion list
    
    42
    +        if buildtrees:
    
    43
    +            self._subdir = "buildtree"
    
    44
    +            self._excluded_subdirs.remove(self._subdir)
    
    45
    +
    
    35 46
         def process(self, element):
    
    36 47
             # returns whether an artifact was downloaded or not
    
    37
    -        if not element._pull():
    
    48
    +        if not element._pull(subdir=self._subdir, excluded_subdirs=self._excluded_subdirs):
    
    38 49
                 raise SkipJob(self.action_name)
    
    39 50
     
    
    40 51
         def status(self, element):
    
    ... ... @@ -49,7 +60,7 @@ class PullQueue(Queue):
    49 60
             if not element._can_query_cache():
    
    50 61
                 return QueueStatus.WAIT
    
    51 62
     
    
    52
    -        if element._pull_pending():
    
    63
    +        if element._pull_pending(subdir=self._subdir):
    
    53 64
                 return QueueStatus.READY
    
    54 65
             else:
    
    55 66
                 return QueueStatus.SKIP
    

  • buildstream/_stream.py
    ... ... @@ -160,12 +160,14 @@ class Stream():
    160 160
         #    track_cross_junctions (bool): Whether tracking should cross junction boundaries
    
    161 161
         #    build_all (bool): Whether to build all elements, or only those
    
    162 162
         #                      which are required to build the target.
    
    163
    +    #    pull_buildtrees (bool): Whether to pull buildtrees from a remote cache server
    
    163 164
         #
    
    164 165
         def build(self, targets, *,
    
    165 166
                   track_targets=None,
    
    166 167
                   track_except=None,
    
    167 168
                   track_cross_junctions=False,
    
    168
    -              build_all=False):
    
    169
    +              build_all=False,
    
    170
    +              pull_buildtrees=False):
    
    169 171
     
    
    170 172
             if build_all:
    
    171 173
                 selection = PipelineSelection.ALL
    
    ... ... @@ -195,7 +197,10 @@ class Stream():
    195 197
                 self._add_queue(track_queue, track=True)
    
    196 198
     
    
    197 199
             if self._artifacts.has_fetch_remotes():
    
    198
    -            self._add_queue(PullQueue(self._scheduler))
    
    200
    +            # Query if pullbuildtrees has been set globally in user config
    
    201
    +            if self._context.pullbuildtrees:
    
    202
    +                pull_buildtrees = True
    
    203
    +            self._add_queue(PullQueue(self._scheduler, buildtrees=pull_buildtrees))
    
    199 204
     
    
    200 205
             self._add_queue(FetchQueue(self._scheduler, skip_cached=True))
    
    201 206
             self._add_queue(BuildQueue(self._scheduler))
    
    ... ... @@ -295,7 +300,8 @@ class Stream():
    295 300
         #
    
    296 301
         def pull(self, targets, *,
    
    297 302
                  selection=PipelineSelection.NONE,
    
    298
    -             remote=None):
    
    303
    +             remote=None,
    
    304
    +             pull_buildtrees=False):
    
    299 305
     
    
    300 306
             use_config = True
    
    301 307
             if remote:
    
    ... ... @@ -310,8 +316,12 @@ class Stream():
    310 316
             if not self._artifacts.has_fetch_remotes():
    
    311 317
                 raise StreamError("No artifact caches available for pulling artifacts")
    
    312 318
     
    
    319
    +        # Query if pullbuildtrees has been set globally in user config
    
    320
    +        if self._context.pullbuildtrees:
    
    321
    +            pull_buildtrees = True
    
    322
    +
    
    313 323
             self._pipeline.assert_consistent(elements)
    
    314
    -        self._add_queue(PullQueue(self._scheduler))
    
    324
    +        self._add_queue(PullQueue(self._scheduler, buildtrees=pull_buildtrees))
    
    315 325
             self._enqueue_plan(elements)
    
    316 326
             self._run()
    
    317 327
     
    

  • buildstream/element.py
    ... ... @@ -1693,18 +1693,26 @@ class Element(Plugin):
    1693 1693
     
    
    1694 1694
         # _pull_pending()
    
    1695 1695
         #
    
    1696
    -    # Check whether the artifact will be pulled.
    
    1696
    +    # Check whether the artifact will be pulled. If the pull operation is to
    
    1697
    +    # include a specific subdir of the element artifact (from cli or user conf)
    
    1698
    +    # then the local cache is queried for the subdirs existence.
    
    1699
    +    #
    
    1700
    +    # Args:
    
    1701
    +    #    subdir (str): Whether the pull has been invoked with a specific subdir set
    
    1697 1702
         #
    
    1698 1703
         # Returns:
    
    1699 1704
         #   (bool): Whether a pull operation is pending
    
    1700 1705
         #
    
    1701
    -    def _pull_pending(self):
    
    1706
    +    def _pull_pending(self, subdir=None):
    
    1702 1707
             if self._get_workspace():
    
    1703 1708
                 # Workspace builds are never pushed to artifact servers
    
    1704 1709
                 return False
    
    1705 1710
     
    
    1706
    -        if self.__strong_cached:
    
    1707
    -            # Artifact already in local cache
    
    1711
    +        if self.__strong_cached and subdir:
    
    1712
    +            # If we've specified a subdir, check if the subdir is cached locally
    
    1713
    +            if self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, subdir):
    
    1714
    +                return False
    
    1715
    +        elif self.__strong_cached:
    
    1708 1716
                 return False
    
    1709 1717
     
    
    1710 1718
             # Pull is pending if artifact remote server available
    
    ... ... @@ -1726,11 +1734,10 @@ class Element(Plugin):
    1726 1734
     
    
    1727 1735
             self._update_state()
    
    1728 1736
     
    
    1729
    -    def _pull_strong(self, *, progress=None):
    
    1737
    +    def _pull_strong(self, *, progress=None, subdir=None, excluded_subdirs=None):
    
    1730 1738
             weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
    
    1731
    -
    
    1732 1739
             key = self.__strict_cache_key
    
    1733
    -        if not self.__artifacts.pull(self, key, progress=progress):
    
    1740
    +        if not self.__artifacts.pull(self, key, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
    
    1734 1741
                 return False
    
    1735 1742
     
    
    1736 1743
             # update weak ref by pointing it to this newly fetched artifact
    
    ... ... @@ -1738,10 +1745,10 @@ class Element(Plugin):
    1738 1745
     
    
    1739 1746
             return True
    
    1740 1747
     
    
    1741
    -    def _pull_weak(self, *, progress=None):
    
    1748
    +    def _pull_weak(self, *, progress=None, subdir=None, excluded_subdirs=None):
    
    1742 1749
             weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
    
    1743
    -
    
    1744
    -        if not self.__artifacts.pull(self, weak_key, progress=progress):
    
    1750
    +        if not self.__artifacts.pull(self, weak_key, progress=progress, subdir=subdir,
    
    1751
    +                                     excluded_subdirs=excluded_subdirs):
    
    1745 1752
                 return False
    
    1746 1753
     
    
    1747 1754
             # extract strong cache key from this newly fetched artifact
    
    ... ... @@ -1759,17 +1766,17 @@ class Element(Plugin):
    1759 1766
         #
    
    1760 1767
         # Returns: True if the artifact has been downloaded, False otherwise
    
    1761 1768
         #
    
    1762
    -    def _pull(self):
    
    1769
    +    def _pull(self, subdir=None, excluded_subdirs=None):
    
    1763 1770
             context = self._get_context()
    
    1764 1771
     
    
    1765 1772
             def progress(percent, message):
    
    1766 1773
                 self.status(message)
    
    1767 1774
     
    
    1768 1775
             # Attempt to pull artifact without knowing whether it's available
    
    1769
    -        pulled = self._pull_strong(progress=progress)
    
    1776
    +        pulled = self._pull_strong(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
    
    1770 1777
     
    
    1771 1778
             if not pulled and not self._cached() and not context.get_strict():
    
    1772
    -            pulled = self._pull_weak(progress=progress)
    
    1779
    +            pulled = self._pull_weak(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
    
    1773 1780
     
    
    1774 1781
             if not pulled:
    
    1775 1782
                 return False
    
    ... ... @@ -1792,10 +1799,21 @@ class Element(Plugin):
    1792 1799
             if not self._cached():
    
    1793 1800
                 return True
    
    1794 1801
     
    
    1795
    -        # Do not push tained artifact
    
    1802
    +        # Do not push tainted artifact
    
    1796 1803
             if self.__get_tainted():
    
    1797 1804
                 return True
    
    1798 1805
     
    
    1806
    +        # Do not push elements that have a dangling buildtree artifact unless element type is
    
    1807
    +        # expected to have an empty buildtree directory
    
    1808
    +        if not self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, 'buildtree'):
    
    1809
    +            return True
    
    1810
    +
    
    1811
    +        # strict_cache_key can't be relied on to be available when running in non strict mode
    
    1812
    +        context = self._get_context()
    
    1813
    +        if not context.get_strict():
    
    1814
    +            if not self.__artifacts.contains_subdir_artifact(self, self.__weak_cache_key, 'buildtree'):
    
    1815
    +                return True
    
    1816
    +
    
    1799 1817
             return False
    
    1800 1818
     
    
    1801 1819
         # _push():
    
    ... ... @@ -2492,7 +2510,7 @@ class Element(Plugin):
    2492 2510
                 if not context.get_strict() and not self.__artifacts.contains(self, key):
    
    2493 2511
                     key = self._get_cache_key(strength=_KeyStrength.WEAK)
    
    2494 2512
     
    
    2495
    -        return (self.__artifacts.extract(self, key), key)
    
    2513
    +        return (self.__artifacts.extract(self, key, subdir='buildtree'), key)
    
    2496 2514
     
    
    2497 2515
         # __get_artifact_metadata_keys():
    
    2498 2516
         #
    

  • tests/completions/completions.py
    ... ... @@ -103,7 +103,7 @@ def test_commands(cli, cmd, word_idx, expected):
    103 103
         ('bst --no-colors build -', 3, ['--all ', '--track ', '--track-all ',
    
    104 104
                                         '--track-except ',
    
    105 105
                                         '--track-cross-junctions ', '-J ',
    
    106
    -                                    '--track-save ']),
    
    106
    +                                    '--track-save ', '--pull-buildtrees ']),
    
    107 107
     
    
    108 108
         # Test the behavior of completing after an option that has a
    
    109 109
         # parameter that cannot be completed, vs an option that has
    

  • tests/integration/pullbuildtrees.py
    1
    +import os
    
    2
    +import shutil
    
    3
    +import pytest
    
    4
    +
    
    5
    +from tests.testutils import cli_integration as cli, create_artifact_share
    
    6
    +from tests.testutils.integration import assert_contains
    
    7
    +
    
    8
    +
    
    9
    +DATA_DIR = os.path.join(
    
    10
    +    os.path.dirname(os.path.realpath(__file__)),
    
    11
    +    "project"
    
    12
    +)
    
    13
    +
    
    14
    +
    
    15
    +# Remove artifact cache & set cli.config value of pullbuildtrees
    
    16
    +# to false, which is the default user context. The cache has to be
    
    17
    +# cleared as just forcefully removing the refpath leaves dangling objects.
    
    18
    +def default_state(cli, tmpdir, share):
    
    19
    +    shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    20
    +    cli.configure({
    
    21
    +        'pullbuildtrees': False,
    
    22
    +        'artifacts': {'url': share.repo, 'push': False},
    
    23
    +        'artifactdir': os.path.join(str(tmpdir), 'artifacts')
    
    24
    +    })
    
    25
    +
    
    26
    +
    
    27
    +# A test to capture the integration of the pullbuildtrees
    
    28
    +# behaviour, which by default is to not include the buildtree
    
    29
    +# directory of an element.
    
    30
    +@pytest.mark.integration
    
    31
    +@pytest.mark.datafiles(DATA_DIR)
    
    32
    +def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
    
    33
    +
    
    34
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    35
    +    element_name = 'autotools/amhello.bst'
    
    36
    +
    
    37
    +    # Create artifact shares for pull & push testing
    
    38
    +    with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
    
    39
    +        create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2:
    
    40
    +        cli.configure({
    
    41
    +            'artifacts': {'url': share1.repo, 'push': True},
    
    42
    +            'artifactdir': os.path.join(str(tmpdir), 'artifacts')
    
    43
    +        })
    
    44
    +
    
    45
    +        # Build autotools element, checked pushed, delete local
    
    46
    +        result = cli.run(project=project, args=['build', element_name])
    
    47
    +        assert result.exit_code == 0
    
    48
    +        assert cli.get_element_state(project, element_name) == 'cached'
    
    49
    +        assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    
    50
    +        default_state(cli, tmpdir, share1)
    
    51
    +
    
    52
    +        # Pull artifact with default config, assert that pulling again
    
    53
    +        # doesn't create a pull job, then assert with buildtrees user
    
    54
    +        # config set creates a pull job.
    
    55
    +        result = cli.run(project=project, args=['pull', element_name])
    
    56
    +        assert element_name in result.get_pulled_elements()
    
    57
    +        result = cli.run(project=project, args=['pull', element_name])
    
    58
    +        assert element_name not in result.get_pulled_elements()
    
    59
    +        cli.configure({'pullbuildtrees': True})
    
    60
    +        result = cli.run(project=project, args=['pull', element_name])
    
    61
    +        assert element_name in result.get_pulled_elements()
    
    62
    +        default_state(cli, tmpdir, share1)
    
    63
    +
    
    64
    +        # Pull artifact with default config, then assert that pulling
    
    65
    +        # with buildtrees cli flag set creates a pull job.
    
    66
    +        result = cli.run(project=project, args=['pull', element_name])
    
    67
    +        assert element_name in result.get_pulled_elements()
    
    68
    +        result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
    
    69
    +        assert element_name in result.get_pulled_elements()
    
    70
    +        default_state(cli, tmpdir, share1)
    
    71
    +
    
    72
    +        # Pull artifact with pullbuildtrees set in user config, then assert
    
    73
    +        # that pulling with the same user config doesn't creates a pull job,
    
    74
    +        # or when buildtrees cli flag is set.
    
    75
    +        cli.configure({'pullbuildtrees': True})
    
    76
    +        result = cli.run(project=project, args=['pull', element_name])
    
    77
    +        assert element_name in result.get_pulled_elements()
    
    78
    +        result = cli.run(project=project, args=['pull', element_name])
    
    79
    +        assert element_name not in result.get_pulled_elements()
    
    80
    +        result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
    
    81
    +        assert element_name not in result.get_pulled_elements()
    
    82
    +        default_state(cli, tmpdir, share1)
    
    83
    +
    
    84
    +        # Pull artifact with default config and buildtrees cli flag set, then assert
    
    85
    +        # that pulling with pullbuildtrees set in user config doesn't create a pull
    
    86
    +        # job.
    
    87
    +        result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
    
    88
    +        assert element_name in result.get_pulled_elements()
    
    89
    +        cli.configure({'pullbuildtrees': True})
    
    90
    +        result = cli.run(project=project, args=['pull', element_name])
    
    91
    +        assert element_name not in result.get_pulled_elements()
    
    92
    +        default_state(cli, tmpdir, share1)
    
    93
    +
    
    94
    +        # Assert that a partial build element (not containing a populated buildtree dir)
    
    95
    +        # can't be pushed to an artifact share, then assert that a complete build element
    
    96
    +        # can be. This will attempt a partial pull from share1 and then a partial push
    
    97
    +        # to share2
    
    98
    +        result = cli.run(project=project, args=['pull', element_name])
    
    99
    +        assert element_name in result.get_pulled_elements()
    
    100
    +        cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
    
    101
    +        result = cli.run(project=project, args=['push', element_name])
    
    102
    +        assert element_name not in result.get_pushed_elements()
    
    103
    +        assert not share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    
    104
    +
    
    105
    +        # Assert that after pulling the missing buildtree the element artifact can be
    
    106
    +        # successfully pushed to the remote. This will attempt to pull the buildtree
    
    107
    +        # from share1 and then a 'complete' push to share2
    
    108
    +        cli.configure({'artifacts': {'url': share1.repo, 'push': False}})
    
    109
    +        result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
    
    110
    +        assert element_name in result.get_pulled_elements()
    
    111
    +        cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
    
    112
    +        result = cli.run(project=project, args=['push', element_name])
    
    113
    +        assert element_name in result.get_pushed_elements()
    
    114
    +        assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    
    115
    +        default_state(cli, tmpdir, share1)

  • tests/testutils/artifactshare.py
    ... ... @@ -122,13 +122,12 @@ class ArtifactShare():
    122 122
             #       same algo for creating an artifact reference
    
    123 123
             #
    
    124 124
     
    
    125
    -        # Chop off the .bst suffix first
    
    126
    -        assert element_name.endswith('.bst')
    
    127
    -        element_name = element_name[:-4]
    
    125
    +        # Replace path separator and chop off the .bst suffix
    
    126
    +        element_name = os.path.splitext(element_name.replace(os.sep, '-'))[0]
    
    128 127
     
    
    129 128
             valid_chars = string.digits + string.ascii_letters + '-._'
    
    130 129
             element_name = ''.join([
    
    131
    -            x if x in valid_chars else '_'
    
    130
    +            x if x in valid_chars else '-'
    
    132 131
                 for x in element_name
    
    133 132
             ])
    
    134 133
             artifact_key = '{0}/{1}/{2}'.format(project_name, element_name, cache_key)
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]