GNOME.org
 

[Notes] [Git][BuildStream/buildstream][tpollard/494] WIP: Don't pull artifact buildtrees by default

Title: GitLab

Tom Pollard pushed to branch tpollard/494 at BuildStream / buildstream

Commits:

10 changed files:

Changes:

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -426,6 +426,22 @@ class ArtifactCache():
    426 426 
             raise ImplError("Cache '{kind}' does not implement contains()"
    427 427 
                             .format(kind=type(self).__name__))
    428 428
    429 
    +    # contains_subdir_artifact():
    430 
    +    #
    431 
    +    # Check whether an artifact element contains a digest for a subdir
    432 
    +    # which is populated in the cache, i.e non dangling.
    433 
    +    #
    434 
    +    # Args:
    435 
    +    #     element (Element): The Element to check
    436 
    +    #     key (str): The cache key to use
    437 
    +    #     subdir (str): The subdir to check
    438 
    +    #
    439 
    +    # Returns: True if the subdir exists & is populated in the cache, False otherwise
    440 
    +    #
    441 
    +    def contains_subdir_artifact(self, element, key, subdir):
    442 
    +        raise ImplError("Cache '{kind}' does not implement contains_subdir_artifact()"
    443 
    +                        .format(kind=type(self).__name__))
    444 
    +
    429 445 
         # list_artifacts():
    430 446 
         #
    431 447 
         # List artifacts in this cache in LRU order.
    ... ... @@ -551,11 +567,12 @@ class ArtifactCache():
    551 567 
         #     element (Element): The Element whose artifact is to be fetched
    552 568 
         #     key (str): The cache key to use
    553 569 
         #     progress (callable): The progress callback, if any
    570 
    +    #     subdir (str): The optional specific subdir to pull
    554 571 
         #
    555 572 
         # Returns:
    556 573 
         #   (bool): True if pull was successful, False if artifact was not available
    557 574 
         #
    558 
    -    def pull(self, element, key, *, progress=None):
    575 
    +    def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
    559 576 
             raise ImplError("Cache '{kind}' does not implement pull()"
    560 577 
                             .format(kind=type(self).__name__))
    561 578

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -92,6 +92,16 @@ class CASCache(ArtifactCache):
    92 92 
             # This assumes that the repository doesn't have any dangling pointers
    93 93 
             return os.path.exists(refpath)
    94 94
    95 
    +    def contains_subdir_artifact(self, element, key, subdir):
    96 
    +        tree = self.resolve_ref(self.get_artifact_fullname(element, key))
    97 
    +
    98 
    +        # This assumes that the subdir digest is present in the element tree
    99 
    +        subdirdigest = self._get_subdir(tree, subdir)
    100 
    +        objpath = self.objpath(subdirdigest)
    101 
    +
    102 
    +        # True if subdir content is cached or if empty as expected
    103 
    +        return os.path.exists(objpath)
    104 
    +
    95 105 
         def extract(self, element, key):
    96 106 
             ref = self.get_artifact_fullname(element, key)
    97 107
    ... ... @@ -228,7 +238,7 @@ class CASCache(ArtifactCache):
    228 238 
                 remotes_for_project = self._remotes[element._get_project()]
    229 239 
                 return any(remote.spec.push for remote in remotes_for_project)
    230 240
    231 
    -    def pull(self, element, key, *, progress=None):
    241 
    +    def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
    232 242 
             ref = self.get_artifact_fullname(element, key)
    233 243
    234 244 
             project = element._get_project()
    ... ... @@ -247,8 +257,14 @@ class CASCache(ArtifactCache):
    247 257 
                     tree.hash = response.digest.hash
    248 258 
                     tree.size_bytes = response.digest.size_bytes
    249 259
    250 
    -                self._fetch_directory(remote, tree)
    260 
    +                # Check if the element artifact is present, if so just fetch subdir
    261 
    +                if subdir and os.path.exists(self.objpath(tree)):
    262 
    +                    self._fetch_subdir(remote, tree, subdir)
    263 
    +                else:
    264 
    +                    # Fetch artifact, excluded_subdirs determined in pullqueue
    265 
    +                    self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
    251 266
    267 
    +                # tree is the remote value, so is the same without or without dangling ref locally
    252 268 
                     self.set_ref(ref, tree)
    253 269
    254 270 
                     element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
    ... ... @@ -668,8 +684,10 @@ class CASCache(ArtifactCache):
    668 684 
                              stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
    669 685
    670 686 
             for dirnode in directory.directories:
    671 
    -            fullpath = os.path.join(dest, dirnode.name)
    672 
    -            self._checkout(fullpath, dirnode.digest)
    687 
    +            # Don't try to checkout a dangling ref
    688 
    +            if os.path.exists(self.objpath(dirnode.digest)):
    689 
    +                fullpath = os.path.join(dest, dirnode.name)
    690 
    +                self._checkout(fullpath, dirnode.digest)
    673 691
    674 692 
             for symlinknode in directory.symlinks:
    675 693 
                 # symlink
    ... ... @@ -948,10 +966,12 @@ class CASCache(ArtifactCache):
    948 966 
         #     remote (Remote): The remote to use.
    949 967 
         #     dir_digest (Digest): Digest object for the directory to fetch.
    950 968 
         #
    951 
    -    def _fetch_directory(self, remote, dir_digest):
    969 
    +    def _fetch_directory(self, remote, dir_digest, *, excluded_subdirs=None):
    952 970 
             fetch_queue = [dir_digest]
    953 971 
             fetch_next_queue = []
    954 972 
             batch = _CASBatchRead(remote)
    973 
    +        if not excluded_subdirs:
    974 
    +            excluded_subdirs = []
    955 975
    956 976 
             while len(fetch_queue) + len(fetch_next_queue) > 0:
    957 977 
                 if len(fetch_queue) == 0:
    ... ... @@ -966,8 +986,9 @@ class CASCache(ArtifactCache):
    966 986 
                     directory.ParseFromString(f.read())
    967 987
    968 988 
                 for dirnode in directory.directories:
    969 
    -                batch = self._fetch_directory_node(remote, dirnode.digest, batch,
    970 
    -                                                   fetch_queue, fetch_next_queue, recursive=True)
    989 
    +                if dirnode.name not in excluded_subdirs:
    990 
    +                    batch = self._fetch_directory_node(remote, dirnode.digest, batch,
    991 
    +                                                       fetch_queue, fetch_next_queue, recursive=True)
    971 992
    972 993 
                 for filenode in directory.files:
    973 994 
                     batch = self._fetch_directory_node(remote, filenode.digest, batch,
    ... ... @@ -976,6 +997,10 @@ class CASCache(ArtifactCache):
    976 997 
             # Fetch final batch
    977 998 
             self._fetch_directory_batch(remote, batch, fetch_queue, fetch_next_queue)
    978 999
    1000 
    +    def _fetch_subdir(self, remote, tree, subdir):
    1001 
    +        subdirdigest = self._get_subdir(tree, subdir)
    1002 
    +        self._fetch_directory(remote, subdirdigest)
    1003 
    +
    979 1004 
         def _fetch_tree(self, remote, digest):
    980 1005 
             # download but do not store the Tree object
    981 1006 
             with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:

  • buildstream/_context.py
    ... ... @@ -110,6 +110,9 @@ class Context():
    110 110 
             # Make sure the XDG vars are set in the environment before loading anything
    111 111 
             self._init_xdg()
    112 112
    113 
    +        # Whether or not to attempt to pull buildtrees globally
    114 
    +        self.pullbuildtrees = False
    115 
    +
    113 116 
             # Private variables
    114 117 
             self._cache_key = None
    115 118 
             self._message_handler = None
    ... ... @@ -160,7 +163,7 @@ class Context():
    160 163 
             _yaml.node_validate(defaults, [
    161 164 
                 'sourcedir', 'builddir', 'artifactdir', 'logdir',
    162 165 
                 'scheduler', 'artifacts', 'logging', 'projects',
    163 
    -            'cache'
    166 
    +            'cache', 'pullbuildtrees'
    164 167 
             ])
    165 168
    166 169 
             for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
    ... ... @@ -185,6 +188,9 @@ class Context():
    185 188 
             # Load artifact share configuration
    186 189 
             self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
    187 190
    191 
    +        # Load pull buildtrees configuration
    192 
    +        self.pullbuildtrees = _yaml.node_get(defaults, bool, 'pullbuildtrees', default_value='False')
    193 
    +
    188 194 
             # Load logging config
    189 195 
             logging = _yaml.node_get(defaults, Mapping, 'logging')
    190 196 
             _yaml.node_validate(logging, [

  • buildstream/_frontend/cli.py
    ... ... @@ -305,10 +305,12 @@ def init(app, project_name, format_version, element_path, force):
    305 305 
                   help="Allow tracking to cross junction boundaries")
    306 306 
     @click.option('--track-save', default=False, is_flag=True,
    307 307 
                   help="Deprecated: This is ignored")
    308 
    +@click.option('--pull-buildtrees', default=False, is_flag=True,
    309 
    +              help="Pull buildtrees from a remote cache server")
    308 310 
     @click.argument('elements', nargs=-1,
    309 311 
                     type=click.Path(readable=False))
    310 312 
     @click.pass_obj
    311 
    -def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
    313 
    +def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions, pull_buildtrees):
    312 314 
         """Build elements in a pipeline"""
    313 315
    314 316 
         if (track_except or track_cross_junctions) and not (track_ or track_all):
    ... ... @@ -327,7 +329,8 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
    327 329 
                              track_targets=track_,
    328 330 
                              track_except=track_except,
    329 331 
                              track_cross_junctions=track_cross_junctions,
    330 
    -                         build_all=all_)
    332 
    +                         build_all=all_,
    333 
    +                         pull_buildtrees=pull_buildtrees)
    331 334
    332 335
    333 336 
     ##################################################################
    ... ... @@ -429,10 +432,12 @@ def track(app, elements, deps, except_, cross_junctions):
    429 432 
                   help='The dependency artifacts to pull (default: none)')
    430 433 
     @click.option('--remote', '-r',
    431 434 
                   help="The URL of the remote cache (defaults to the first configured cache)")
    435 
    +@click.option('--pull-buildtrees', default=False, is_flag=True,
    436 
    +              help="Pull buildtrees from a remote cache server")
    432 437 
     @click.argument('elements', nargs=-1,
    433 438 
                     type=click.Path(readable=False))
    434 439 
     @click.pass_obj
    435 
    -def pull(app, elements, deps, remote):
    440 
    +def pull(app, elements, deps, remote, pull_buildtrees):
    436 441 
         """Pull a built artifact from the configured remote artifact cache.
    437 442
    438 443 
         By default the artifact will be pulled one of the configured caches
    ... ... @@ -446,7 +451,7 @@ def pull(app, elements, deps, remote):
    446 451 
             all:   All dependencies
    447 452 
         """
    448 453 
         with app.initialized(session_name="Pull"):
    449 
    -        app.stream.pull(elements, selection=deps, remote=remote)
    454 
    +        app.stream.pull(elements, selection=deps, remote=remote, pull_buildtrees=pull_buildtrees)
    450 455
    451 456
    452 457 
     ##################################################################

  • buildstream/_scheduler/queues/pullqueue.py
    ... ... @@ -32,9 +32,20 @@ class PullQueue(Queue):
    32 32 
         complete_name = "Pulled"
    33 33 
         resources = [ResourceType.DOWNLOAD, ResourceType.CACHE]
    34 34
    35 
    +    def __init__(self, scheduler, buildtrees=False):
    36 
    +        super().__init__(scheduler)
    37 
    +
    38 
    +        # Current default exclusions on pull
    39 
    +        self._excluded_subdirs = ["buildtree"]
    40 
    +        self._subdir = None
    41 
    +        # If buildtrees are to be pulled, remove the value from exclusion list
    42 
    +        if buildtrees:
    43 
    +            self._subdir = "buildtree"
    44 
    +            self._excluded_subdirs.remove(self._subdir)
    45 
    +
    35 46 
         def process(self, element):
    36 47 
             # returns whether an artifact was downloaded or not
    37 
    -        if not element._pull():
    48 
    +        if not element._pull(subdir=self._subdir, excluded_subdirs=self._excluded_subdirs):
    38 49 
                 raise SkipJob(self.action_name)
    39 50
    40 51 
         def status(self, element):
    ... ... @@ -49,7 +60,7 @@ class PullQueue(Queue):
    49 60 
             if not element._can_query_cache():
    50 61 
                 return QueueStatus.WAIT
    51 62
    52 
    -        if element._pull_pending():
    63 
    +        if element._pull_pending(subdir=self._subdir):
    53 64 
                 return QueueStatus.READY
    54 65 
             else:
    55 66 
                 return QueueStatus.SKIP

  • buildstream/_stream.py
    ... ... @@ -160,12 +160,14 @@ class Stream():
    160 160 
         #    track_cross_junctions (bool): Whether tracking should cross junction boundaries
    161 161 
         #    build_all (bool): Whether to build all elements, or only those
    162 162 
         #                      which are required to build the target.
    163 
    +    #    pull_buildtrees (bool): Whether to pull buildtrees from a remote cache server
    163 164 
         #
    164 165 
         def build(self, targets, *,
    165 166 
                   track_targets=None,
    166 167 
                   track_except=None,
    167 168 
                   track_cross_junctions=False,
    168 
    -              build_all=False):
    169 
    +              build_all=False,
    170 
    +              pull_buildtrees=False):
    169 171
    170 172 
             if build_all:
    171 173 
                 selection = PipelineSelection.ALL
    ... ... @@ -195,7 +197,10 @@ class Stream():
    195 197 
                 self._add_queue(track_queue, track=True)
    196 198
    197 199 
             if self._artifacts.has_fetch_remotes():
    198 
    -            self._add_queue(PullQueue(self._scheduler))
    200 
    +            # Query if pullbuildtrees has been set globally in user config
    201 
    +            if self._context.pullbuildtrees:
    202 
    +                pull_buildtrees = True
    203 
    +            self._add_queue(PullQueue(self._scheduler, buildtrees=pull_buildtrees))
    199 204
    200 205 
             self._add_queue(FetchQueue(self._scheduler, skip_cached=True))
    201 206 
             self._add_queue(BuildQueue(self._scheduler))
    ... ... @@ -295,7 +300,8 @@ class Stream():
    295 300 
         #
    296 301 
         def pull(self, targets, *,
    297 302 
                  selection=PipelineSelection.NONE,
    298 
    -             remote=None):
    303 
    +             remote=None,
    304 
    +             pull_buildtrees=False):
    299 305
    300 306 
             use_config = True
    301 307 
             if remote:
    ... ... @@ -310,8 +316,12 @@ class Stream():
    310 316 
             if not self._artifacts.has_fetch_remotes():
    311 317 
                 raise StreamError("No artifact caches available for pulling artifacts")
    312 318
    319 
    +        # Query if pullbuildtrees has been set globally in user config
    320 
    +        if self._context.pullbuildtrees:
    321 
    +            pull_buildtrees = True
    322 
    +
    313 323 
             self._pipeline.assert_consistent(elements)
    314 
    -        self._add_queue(PullQueue(self._scheduler))
    324 
    +        self._add_queue(PullQueue(self._scheduler, buildtrees=pull_buildtrees))
    315 325 
             self._enqueue_plan(elements)
    316 326 
             self._run()
    317 327

  • buildstream/element.py
    ... ... @@ -1692,18 +1692,26 @@ class Element(Plugin):
    1692 1692
    1693 1693 
         # _pull_pending()
    1694 1694 
         #
    1695 
    -    # Check whether the artifact will be pulled.
    1695 
    +    # Check whether the artifact will be pulled. If the pull operation is to
    1696 
    +    # include a specific subdir of the element artifact (from cli or user conf)
    1697 
    +    # then the local cache is queried for the subdirs existence.
    1698 
    +    #
    1699 
    +    # Args:
    1700 
    +    #    subdir (str): Whether the pull has been invoked with a specific subdir set
    1696 1701 
         #
    1697 1702 
         # Returns:
    1698 1703 
         #   (bool): Whether a pull operation is pending
    1699 1704 
         #
    1700 
    -    def _pull_pending(self):
    1705 
    +    def _pull_pending(self, subdir=None):
    1701 1706 
             if self._get_workspace():
    1702 1707 
                 # Workspace builds are never pushed to artifact servers
    1703 1708 
                 return False
    1704 1709
    1705 
    -        if self.__strong_cached:
    1706 
    -            # Artifact already in local cache
    1710 
    +        if self.__strong_cached and subdir:
    1711 
    +            # If we've specified a subdir, check if the subdir is cached locally
    1712 
    +            if self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, subdir):
    1713 
    +                return False
    1714 
    +        elif self.__strong_cached:
    1707 1715 
                 return False
    1708 1716
    1709 1717 
             # Pull is pending if artifact remote server available
    ... ... @@ -1725,11 +1733,10 @@ class Element(Plugin):
    1725 1733
    1726 1734 
             self._update_state()
    1727 1735
    1728 
    -    def _pull_strong(self, *, progress=None):
    1736 
    +    def _pull_strong(self, *, progress=None, subdir=None, excluded_subdirs=None):
    1729 1737 
             weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
    1730 
    -
    1731 1738 
             key = self.__strict_cache_key
    1732 
    -        if not self.__artifacts.pull(self, key, progress=progress):
    1739 
    +        if not self.__artifacts.pull(self, key, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
    1733 1740 
                 return False
    1734 1741
    1735 1742 
             # update weak ref by pointing it to this newly fetched artifact
    ... ... @@ -1737,10 +1744,10 @@ class Element(Plugin):
    1737 1744
    1738 1745 
             return True
    1739 1746
    1740 
    -    def _pull_weak(self, *, progress=None):
    1747 
    +    def _pull_weak(self, *, progress=None, subdir=None, excluded_subdirs=None):
    1741 1748 
             weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
    1742 
    -
    1743 
    -        if not self.__artifacts.pull(self, weak_key, progress=progress):
    1749 
    +        if not self.__artifacts.pull(self, weak_key, progress=progress, subdir=subdir,
    1750 
    +                                     excluded_subdirs=excluded_subdirs):
    1744 1751 
                 return False
    1745 1752
    1746 1753 
             # extract strong cache key from this newly fetched artifact
    ... ... @@ -1758,17 +1765,17 @@ class Element(Plugin):
    1758 1765 
         #
    1759 1766 
         # Returns: True if the artifact has been downloaded, False otherwise
    1760 1767 
         #
    1761 
    -    def _pull(self):
    1768 
    +    def _pull(self, subdir=None, excluded_subdirs=None):
    1762 1769 
             context = self._get_context()
    1763 1770
    1764 1771 
             def progress(percent, message):
    1765 1772 
                 self.status(message)
    1766 1773
    1767 1774 
             # Attempt to pull artifact without knowing whether it's available
    1768 
    -        pulled = self._pull_strong(progress=progress)
    1775 
    +        pulled = self._pull_strong(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
    1769 1776
    1770 1777 
             if not pulled and not self._cached() and not context.get_strict():
    1771 
    -            pulled = self._pull_weak(progress=progress)
    1778 
    +            pulled = self._pull_weak(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
    1772 1779
    1773 1780 
             if not pulled:
    1774 1781 
                 return False
    ... ... @@ -1791,10 +1798,21 @@ class Element(Plugin):
    1791 1798 
             if not self._cached():
    1792 1799 
                 return True
    1793 1800
    1794 
    -        # Do not push tained artifact
    1801 
    +        # Do not push tainted artifact
    1795 1802 
             if self.__get_tainted():
    1796 1803 
                 return True
    1797 1804
    1805 
    +        # Do not push elements that have a dangling buildtree artifact unless element type is
    1806 
    +        # expected to have an empty buildtree directory
    1807 
    +        if not self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, 'buildtree'):
    1808 
    +            return True
    1809 
    +
    1810 
    +        # strict_cache_key can't be relied on to be available when running in non strict mode
    1811 
    +        context = self._get_context()
    1812 
    +        if not context.get_strict():
    1813 
    +            if not self.__artifacts.contains_subdir_artifact(self, self.__weak_cache_key, 'buildtree'):
    1814 
    +                return True
    1815 
    +
    1798 1816 
             return False
    1799 1817
    1800 1818 
         # _push():

  • tests/completions/completions.py
    ... ... @@ -103,7 +103,7 @@ def test_commands(cli, cmd, word_idx, expected):
    103 103 
         ('bst --no-colors build -', 3, ['--all ', '--track ', '--track-all ',
    104 104 
                                         '--track-except ',
    105 105 
                                         '--track-cross-junctions ', '-J ',
    106 
    -                                    '--track-save ']),
    106 
    +                                    '--track-save ', '--pull-buildtrees ']),
    107 107
    108 108 
         # Test the behavior of completing after an option that has a
    109 109 
         # parameter that cannot be completed, vs an option that has

  • tests/integration/pullbuildtrees.py
    1 
    +import os
    2 
    +import shutil
    3 
    +import pytest
    4 
    +
    5 
    +from tests.testutils import cli_integration as cli, create_artifact_share
    6 
    +from tests.testutils.integration import assert_contains
    7 
    +
    8 
    +
    9 
    +DATA_DIR = os.path.join(
    10 
    +    os.path.dirname(os.path.realpath(__file__)),
    11 
    +    "project"
    12 
    +)
    13 
    +
    14 
    +
    15 
    +# Remove artifact cache & set cli.config value of pullbuildtrees
    16 
    +# to false, which is the default user context
    17 
    +def default_state(cli, integration_cache, share):
    18 
    +    shutil.rmtree(os.path.join(integration_cache, 'artifacts'))
    19 
    +    cli.configure({'pullbuildtrees': False, 'artifacts': {'url': share.repo, 'push': False}})
    20 
    +
    21 
    +
    22 
    +# A test to capture the integration of the pullbuildtrees
    23 
    +# behaviour, which by default is to not include the buildtree
    24 
    +# directory of an element
    25 
    +@pytest.mark.integration
    26 
    +@pytest.mark.datafiles(DATA_DIR)
    27 
    +def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
    28 
    +
    29 
    +    # Create a backup of the artifact cache dir as we need to perform deletions on it
    30 
    +    shutil.move(os.path.join(integration_cache, 'artifacts'), os.path.join(integration_cache, 'backup'))
    31 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    32 
    +    element_name = 'autotools/amhello.bst'
    33 
    +
    34 
    +    # Create artifact shares for pull & push testing
    35 
    +    with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
    36 
    +        create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2:
    37 
    +        cli.configure({
    38 
    +            'artifacts': {'url': share1.repo, 'push': True},
    39 
    +        })
    40 
    +
    41 
    +        # Build autotools element, checked pushed, delete local
    42 
    +        result = cli.run(project=project, args=['build', element_name])
    43 
    +        assert result.exit_code == 0
    44 
    +        assert cli.get_element_state(project, element_name) == 'cached'
    45 
    +        assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    46 
    +        default_state(cli, integration_cache, share1)
    47 
    +
    48 
    +        # Pull artifact with default config, assert that pulling again
    49 
    +        # doesn't create a pull job, then assert with buildtrees user
    50 
    +        # config set creates a pull job.
    51 
    +        result = cli.run(project=project, args=['pull', element_name])
    52 
    +        assert element_name in result.get_pulled_elements()
    53 
    +        result = cli.run(project=project, args=['pull', element_name])
    54 
    +        assert element_name not in result.get_pulled_elements()
    55 
    +        cli.configure({'pullbuildtrees': True})
    56 
    +        result = cli.run(project=project, args=['pull', element_name])
    57 
    +        assert element_name in result.get_pulled_elements()
    58 
    +        default_state(cli, integration_cache, share1)
    59 
    +
    60 
    +        # Pull artifact with default config, then assert that pulling
    61 
    +        # with buildtrees cli flag set creates a pull job.
    62 
    +        result = cli.run(project=project, args=['pull', element_name])
    63 
    +        assert element_name in result.get_pulled_elements()
    64 
    +        result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
    65 
    +        assert element_name in result.get_pulled_elements()
    66 
    +        default_state(cli, integration_cache, share1)
    67 
    +
    68 
    +        # Pull artifact with pullbuildtrees set in user config, then assert
    69 
    +        # that pulling with the same user config doesn't creates a pull job,
    70 
    +        # or when buildtrees cli flag is set.
    71 
    +        cli.configure({'pullbuildtrees': True})
    72 
    +        result = cli.run(project=project, args=['pull', element_name])
    73 
    +        assert element_name in result.get_pulled_elements()
    74 
    +        result = cli.run(project=project, args=['pull', element_name])
    75 
    +        assert element_name not in result.get_pulled_elements()
    76 
    +        result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
    77 
    +        assert element_name not in result.get_pulled_elements()
    78 
    +        default_state(cli, integration_cache, share1)
    79 
    +
    80 
    +        # Pull artifact with default config and buildtrees cli flag set, then assert
    81 
    +        # that pulling with pullbuildtrees set in user config doesn't create a pull
    82 
    +        # job.
    83 
    +        result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
    84 
    +        assert element_name in result.get_pulled_elements()
    85 
    +        cli.configure({'pullbuildtrees': True})
    86 
    +        result = cli.run(project=project, args=['pull', element_name])
    87 
    +        assert element_name not in result.get_pulled_elements()
    88 
    +        default_state(cli, integration_cache, share1)
    89 
    +
    90 
    +        # Assert that a partial build element (not containing a populated buildtree dir)
    91 
    +        # can't be pushed to an artifact share, then assert that a complete build element
    92 
    +        # can be. This will attempt a partial pull from share1 and then a partial push
    93 
    +        # to share2
    94 
    +        result = cli.run(project=project, args=['pull', element_name])
    95 
    +        assert element_name in result.get_pulled_elements()
    96 
    +        cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
    97 
    +        result = cli.run(project=project, args=['push', element_name])
    98 
    +        assert element_name not in result.get_pushed_elements()
    99 
    +        assert not share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    100 
    +
    101 
    +        # Assert that after pulling the missing buildtree the element artifact can be
    102 
    +        # successfully pushed to the remote. This will attempt to pull the buildtree
    103 
    +        # from share1 and then a 'complete' push to share2
    104 
    +        cli.configure({'artifacts': {'url': share1.repo, 'push': False}})
    105 
    +        result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
    106 
    +        assert element_name in result.get_pulled_elements()
    107 
    +        cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
    108 
    +        result = cli.run(project=project, args=['push', element_name])
    109 
    +        assert element_name in result.get_pushed_elements()
    110 
    +        assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    111 
    +
    112 
    +        # Replace artifact cache dir with original
    113 
    +        shutil.rmtree(os.path.join(integration_cache, 'artifacts'))
    114 
    +        shutil.move(os.path.join(integration_cache, 'backup'), os.path.join(integration_cache, 'artifacts'))

  • tests/testutils/artifactshare.py
    ... ... @@ -128,7 +128,7 @@ class ArtifactShare():
    128 128
    129 129 
             valid_chars = string.digits + string.ascii_letters + '-._'
    130 130 
             element_name = ''.join([
    131 
    -            x if x in valid_chars else '_'
    131 
    +            x if x in valid_chars else '-'
    132 132 
                 for x in element_name
    133 133 
             ])
    134 134 
             artifact_key = '{0}/{1}/{2}'.format(project_name, element_name, cache_key)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]