Tom Pollard pushed to branch tpollard/494 at BuildStream / buildstream
Commits:
-
4133af84
by Tom Pollard at 2018-10-23T14:30:38Z
7 changed files:
- NEWS
- buildstream/_artifactcache/artifactcache.py
- buildstream/_artifactcache/cascache.py
- buildstream/_scheduler/queues/pullqueue.py
- buildstream/_stream.py
- buildstream/element.py
- + tests/integration/pullbuildtrees.py
Changes:
... | ... | @@ -31,6 +31,17 @@ buildstream 1.3.1 |
31 | 31 |
new the `conf-root` variable to make the process easier. And there has been
|
32 | 32 |
a bug fix to workspaces so they can be build in workspaces too.
|
33 | 33 |
|
34 |
+ o Due to the element `buildtree` being cached in the respective artifact their
|
|
35 |
+ size in some cases has significantly increased. In *most* cases the buildtree
|
|
36 |
+ is not utilised when building targets, as such by default bst 'pull' & 'build'
|
|
37 |
+ will not fetch buildtrees from remotes. This behaviour can be overriden with
|
|
38 |
+ the cli option '--pull-buildtrees', or the user configuration option
|
|
39 |
+ 'pullbuildtrees = True'. The override will also add the buildtree to already
|
|
40 |
+ cached artifacts. When attempting to populate an artifactcache server with
|
|
41 |
+ cached artifacts, only 'complete' elements can be pushed. If the element is
|
|
42 |
+ expected to have a populated buildtree then it must be cached before pushing.
|
|
43 |
+ |
|
44 |
+ |
|
34 | 45 |
=================
|
35 | 46 |
buildstream 1.1.5
|
36 | 47 |
=================
|
... | ... | @@ -428,6 +428,22 @@ class ArtifactCache(): |
428 | 428 |
raise ImplError("Cache '{kind}' does not implement contains()"
|
429 | 429 |
.format(kind=type(self).__name__))
|
430 | 430 |
|
431 |
+ # contains_subdir_artifact():
|
|
432 |
+ #
|
|
433 |
+ # Check whether an artifact element contains a digest for a subdir
|
|
434 |
+ # which is populated in the cache, i.e non dangling.
|
|
435 |
+ #
|
|
436 |
+ # Args:
|
|
437 |
+ # element (Element): The Element to check
|
|
438 |
+ # key (str): The cache key to use
|
|
439 |
+ # subdir (str): The subdir to check
|
|
440 |
+ #
|
|
441 |
+ # Returns: True if the subdir exists & is populated in the cache, False otherwise
|
|
442 |
+ #
|
|
443 |
+ def contains_subdir_artifact(self, element, key, subdir):
|
|
444 |
+ raise ImplError("Cache '{kind}' does not implement contains_subdir_artifact()"
|
|
445 |
+ .format(kind=type(self).__name__))
|
|
446 |
+ |
|
431 | 447 |
# list_artifacts():
|
432 | 448 |
#
|
433 | 449 |
# List artifacts in this cache in LRU order.
|
... | ... | @@ -464,6 +480,8 @@ class ArtifactCache(): |
464 | 480 |
# Args:
|
465 | 481 |
# element (Element): The Element to extract
|
466 | 482 |
# key (str): The cache key to use
|
483 |
+ # tree (str): Optional specific tree to extract
|
|
484 |
+ # dest (str): Optional specific extract dest dir
|
|
467 | 485 |
#
|
468 | 486 |
# Raises:
|
469 | 487 |
# ArtifactError: In cases there was an OSError, or if the artifact
|
... | ... | @@ -471,7 +489,7 @@ class ArtifactCache(): |
471 | 489 |
#
|
472 | 490 |
# Returns: path to extracted artifact
|
473 | 491 |
#
|
474 |
- def extract(self, element, key):
|
|
492 |
+ def extract(self, element, key, tree=None, dest=None):
|
|
475 | 493 |
raise ImplError("Cache '{kind}' does not implement extract()"
|
476 | 494 |
.format(kind=type(self).__name__))
|
477 | 495 |
|
... | ... | @@ -553,11 +571,13 @@ class ArtifactCache(): |
553 | 571 |
# element (Element): The Element whose artifact is to be fetched
|
554 | 572 |
# key (str): The cache key to use
|
555 | 573 |
# progress (callable): The progress callback, if any
|
574 |
+ # subdir (str): The optional specific subdir to pull
|
|
575 |
+ # excluded_subdirs (list): The optional list of subdirs to not pull
|
|
556 | 576 |
#
|
557 | 577 |
# Returns:
|
558 | 578 |
# (bool): True if pull was successful, False if artifact was not available
|
559 | 579 |
#
|
560 |
- def pull(self, element, key, *, progress=None):
|
|
580 |
+ def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
|
|
561 | 581 |
raise ImplError("Cache '{kind}' does not implement pull()"
|
562 | 582 |
.format(kind=type(self).__name__))
|
563 | 583 |
|
... | ... | @@ -92,13 +92,26 @@ class CASCache(ArtifactCache): |
92 | 92 |
# This assumes that the repository doesn't have any dangling pointers
|
93 | 93 |
return os.path.exists(refpath)
|
94 | 94 |
|
95 |
- def extract(self, element, key):
|
|
95 |
+ def contains_subdir_artifact(self, element, key, subdir):
|
|
96 |
+ tree = self.resolve_ref(self.get_artifact_fullname(element, key))
|
|
97 |
+ |
|
98 |
+ # This assumes that the subdir digest is present in the element tree
|
|
99 |
+ subdirdigest = self._get_subdir(tree, subdir)
|
|
100 |
+ objpath = self.objpath(subdirdigest)
|
|
101 |
+ |
|
102 |
+ # True if subdir content is cached or if empty as expected
|
|
103 |
+ return os.path.exists(objpath)
|
|
104 |
+ |
|
105 |
+ def extract(self, element, key, tree=None, dest=None):
|
|
96 | 106 |
ref = self.get_artifact_fullname(element, key)
|
97 | 107 |
|
98 |
- tree = self.resolve_ref(ref, update_mtime=True)
|
|
108 |
+ if not tree:
|
|
109 |
+ tree = self.resolve_ref(ref, update_mtime=True)
|
|
110 |
+ |
|
111 |
+ if not dest:
|
|
112 |
+ dest = os.path.join(self.extractdir, element._get_project().name,
|
|
113 |
+ element.normal_name, tree.hash)
|
|
99 | 114 |
|
100 |
- dest = os.path.join(self.extractdir, element._get_project().name,
|
|
101 |
- element.normal_name, tree.hash)
|
|
102 | 115 |
if os.path.isdir(dest):
|
103 | 116 |
# artifact has already been extracted
|
104 | 117 |
return dest
|
... | ... | @@ -228,7 +241,7 @@ class CASCache(ArtifactCache): |
228 | 241 |
remotes_for_project = self._remotes[element._get_project()]
|
229 | 242 |
return any(remote.spec.push for remote in remotes_for_project)
|
230 | 243 |
|
231 |
- def pull(self, element, key, *, progress=None):
|
|
244 |
+ def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
|
|
232 | 245 |
ref = self.get_artifact_fullname(element, key)
|
233 | 246 |
|
234 | 247 |
project = element._get_project()
|
... | ... | @@ -247,8 +260,19 @@ class CASCache(ArtifactCache): |
247 | 260 |
tree.hash = response.digest.hash
|
248 | 261 |
tree.size_bytes = response.digest.size_bytes
|
249 | 262 |
|
250 |
- self._fetch_directory(remote, tree)
|
|
263 |
+ # Check if the element artifact is present, if so just fetch subdir
|
|
264 |
+ # and extract into existing artifact extract dir
|
|
265 |
+ if subdir and os.path.exists(self.objpath(tree)):
|
|
266 |
+ self._fetch_subdir(remote, tree, subdir)
|
|
267 |
+ subdirdigest = self._get_subdir(tree, subdir)
|
|
268 |
+ dest = os.path.join(self.extractdir, element._get_project().name,
|
|
269 |
+ element.normal_name, tree.hash, subdir)
|
|
270 |
+ self.extract(element, key, subdirdigest, dest)
|
|
271 |
+ else:
|
|
272 |
+ # Fetch artifact, excluded_subdirs determined in pullqueue
|
|
273 |
+ self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
|
|
251 | 274 |
|
275 |
+ # tree is the remote value, so is the same without or without dangling ref locally
|
|
252 | 276 |
self.set_ref(ref, tree)
|
253 | 277 |
|
254 | 278 |
element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
|
... | ... | @@ -672,8 +696,10 @@ class CASCache(ArtifactCache): |
672 | 696 |
stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
|
673 | 697 |
|
674 | 698 |
for dirnode in directory.directories:
|
675 |
- fullpath = os.path.join(dest, dirnode.name)
|
|
676 |
- self._checkout(fullpath, dirnode.digest)
|
|
699 |
+ # Don't try to checkout a dangling ref
|
|
700 |
+ if os.path.exists(self.objpath(dirnode.digest)):
|
|
701 |
+ fullpath = os.path.join(dest, dirnode.name)
|
|
702 |
+ self._checkout(fullpath, dirnode.digest)
|
|
677 | 703 |
|
678 | 704 |
for symlinknode in directory.symlinks:
|
679 | 705 |
# symlink
|
... | ... | @@ -951,11 +977,14 @@ class CASCache(ArtifactCache): |
951 | 977 |
# Args:
|
952 | 978 |
# remote (Remote): The remote to use.
|
953 | 979 |
# dir_digest (Digest): Digest object for the directory to fetch.
|
980 |
+ # excluded_subdirs (list): The optional list of subdirs to not fetch
|
|
954 | 981 |
#
|
955 |
- def _fetch_directory(self, remote, dir_digest):
|
|
982 |
+ def _fetch_directory(self, remote, dir_digest, *, excluded_subdirs=None):
|
|
956 | 983 |
fetch_queue = [dir_digest]
|
957 | 984 |
fetch_next_queue = []
|
958 | 985 |
batch = _CASBatchRead(remote)
|
986 |
+ if not excluded_subdirs:
|
|
987 |
+ excluded_subdirs = []
|
|
959 | 988 |
|
960 | 989 |
while len(fetch_queue) + len(fetch_next_queue) > 0:
|
961 | 990 |
if len(fetch_queue) == 0:
|
... | ... | @@ -970,8 +999,9 @@ class CASCache(ArtifactCache): |
970 | 999 |
directory.ParseFromString(f.read())
|
971 | 1000 |
|
972 | 1001 |
for dirnode in directory.directories:
|
973 |
- batch = self._fetch_directory_node(remote, dirnode.digest, batch,
|
|
974 |
- fetch_queue, fetch_next_queue, recursive=True)
|
|
1002 |
+ if dirnode.name not in excluded_subdirs:
|
|
1003 |
+ batch = self._fetch_directory_node(remote, dirnode.digest, batch,
|
|
1004 |
+ fetch_queue, fetch_next_queue, recursive=True)
|
|
975 | 1005 |
|
976 | 1006 |
for filenode in directory.files:
|
977 | 1007 |
batch = self._fetch_directory_node(remote, filenode.digest, batch,
|
... | ... | @@ -980,6 +1010,10 @@ class CASCache(ArtifactCache): |
980 | 1010 |
# Fetch final batch
|
981 | 1011 |
self._fetch_directory_batch(remote, batch, fetch_queue, fetch_next_queue)
|
982 | 1012 |
|
1013 |
+ def _fetch_subdir(self, remote, tree, subdir):
|
|
1014 |
+ subdirdigest = self._get_subdir(tree, subdir)
|
|
1015 |
+ self._fetch_directory(remote, subdirdigest)
|
|
1016 |
+ |
|
983 | 1017 |
def _fetch_tree(self, remote, digest):
|
984 | 1018 |
# download but do not store the Tree object
|
985 | 1019 |
with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
|
... | ... | @@ -32,9 +32,20 @@ class PullQueue(Queue): |
32 | 32 |
complete_name = "Pulled"
|
33 | 33 |
resources = [ResourceType.DOWNLOAD, ResourceType.CACHE]
|
34 | 34 |
|
35 |
+ def __init__(self, scheduler, buildtrees=False):
|
|
36 |
+ super().__init__(scheduler)
|
|
37 |
+ |
|
38 |
+ # Current default exclusions on pull
|
|
39 |
+ self._excluded_subdirs = ["buildtree"]
|
|
40 |
+ self._subdir = None
|
|
41 |
+ # If buildtrees are to be pulled, remove the value from exclusion list
|
|
42 |
+ if buildtrees:
|
|
43 |
+ self._subdir = "buildtree"
|
|
44 |
+ self._excluded_subdirs.remove(self._subdir)
|
|
45 |
+ |
|
35 | 46 |
def process(self, element):
|
36 | 47 |
# returns whether an artifact was downloaded or not
|
37 |
- if not element._pull():
|
|
48 |
+ if not element._pull(subdir=self._subdir, excluded_subdirs=self._excluded_subdirs):
|
|
38 | 49 |
raise SkipJob(self.action_name)
|
39 | 50 |
|
40 | 51 |
def status(self, element):
|
... | ... | @@ -49,7 +60,7 @@ class PullQueue(Queue): |
49 | 60 |
if not element._can_query_cache():
|
50 | 61 |
return QueueStatus.WAIT
|
51 | 62 |
|
52 |
- if element._pull_pending():
|
|
63 |
+ if element._pull_pending(subdir=self._subdir):
|
|
53 | 64 |
return QueueStatus.READY
|
54 | 65 |
else:
|
55 | 66 |
return QueueStatus.SKIP
|
... | ... | @@ -160,6 +160,7 @@ class Stream(): |
160 | 160 |
# track_cross_junctions (bool): Whether tracking should cross junction boundaries
|
161 | 161 |
# build_all (bool): Whether to build all elements, or only those
|
162 | 162 |
# which are required to build the target.
|
163 |
+ # pull_buildtrees (bool): Whether to pull buildtrees from a remote cache server
|
|
163 | 164 |
#
|
164 | 165 |
def build(self, targets, *,
|
165 | 166 |
track_targets=None,
|
... | ... | @@ -196,7 +197,10 @@ class Stream(): |
196 | 197 |
self._add_queue(track_queue, track=True)
|
197 | 198 |
|
198 | 199 |
if self._artifacts.has_fetch_remotes():
|
199 |
- self._add_queue(PullQueue(self._scheduler))
|
|
200 |
+ # Query if pullbuildtrees has been set globally in user config
|
|
201 |
+ if self._context.pullbuildtrees:
|
|
202 |
+ pull_buildtrees = True
|
|
203 |
+ self._add_queue(PullQueue(self._scheduler, buildtrees=pull_buildtrees))
|
|
200 | 204 |
|
201 | 205 |
self._add_queue(FetchQueue(self._scheduler, skip_cached=True))
|
202 | 206 |
self._add_queue(BuildQueue(self._scheduler))
|
... | ... | @@ -312,8 +316,12 @@ class Stream(): |
312 | 316 |
if not self._artifacts.has_fetch_remotes():
|
313 | 317 |
raise StreamError("No artifact caches available for pulling artifacts")
|
314 | 318 |
|
319 |
+ # Query if pullbuildtrees has been set globally in user config
|
|
320 |
+ if self._context.pullbuildtrees:
|
|
321 |
+ pull_buildtrees = True
|
|
322 |
+ |
|
315 | 323 |
self._pipeline.assert_consistent(elements)
|
316 |
- self._add_queue(PullQueue(self._scheduler))
|
|
324 |
+ self._add_queue(PullQueue(self._scheduler, buildtrees=pull_buildtrees))
|
|
317 | 325 |
self._enqueue_plan(elements)
|
318 | 326 |
self._run()
|
319 | 327 |
|
... | ... | @@ -1693,18 +1693,26 @@ class Element(Plugin): |
1693 | 1693 |
|
1694 | 1694 |
# _pull_pending()
|
1695 | 1695 |
#
|
1696 |
- # Check whether the artifact will be pulled.
|
|
1696 |
+ # Check whether the artifact will be pulled. If the pull operation is to
|
|
1697 |
+ # include a specific subdir of the element artifact (from cli or user conf)
|
|
1698 |
+ # then the local cache is queried for the subdirs existence.
|
|
1699 |
+ #
|
|
1700 |
+ # Args:
|
|
1701 |
+ # subdir (str): Whether the pull has been invoked with a specific subdir set
|
|
1697 | 1702 |
#
|
1698 | 1703 |
# Returns:
|
1699 | 1704 |
# (bool): Whether a pull operation is pending
|
1700 | 1705 |
#
|
1701 |
- def _pull_pending(self):
|
|
1706 |
+ def _pull_pending(self, subdir=None):
|
|
1702 | 1707 |
if self._get_workspace():
|
1703 | 1708 |
# Workspace builds are never pushed to artifact servers
|
1704 | 1709 |
return False
|
1705 | 1710 |
|
1706 |
- if self.__strong_cached:
|
|
1707 |
- # Artifact already in local cache
|
|
1711 |
+ if self.__strong_cached and subdir:
|
|
1712 |
+ # If we've specified a subdir, check if the subdir is cached locally
|
|
1713 |
+ if self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, subdir):
|
|
1714 |
+ return False
|
|
1715 |
+ elif self.__strong_cached:
|
|
1708 | 1716 |
return False
|
1709 | 1717 |
|
1710 | 1718 |
# Pull is pending if artifact remote server available
|
... | ... | @@ -1726,11 +1734,10 @@ class Element(Plugin): |
1726 | 1734 |
|
1727 | 1735 |
self._update_state()
|
1728 | 1736 |
|
1729 |
- def _pull_strong(self, *, progress=None):
|
|
1737 |
+ def _pull_strong(self, *, progress=None, subdir=None, excluded_subdirs=None):
|
|
1730 | 1738 |
weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
|
1731 |
- |
|
1732 | 1739 |
key = self.__strict_cache_key
|
1733 |
- if not self.__artifacts.pull(self, key, progress=progress):
|
|
1740 |
+ if not self.__artifacts.pull(self, key, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
|
|
1734 | 1741 |
return False
|
1735 | 1742 |
|
1736 | 1743 |
# update weak ref by pointing it to this newly fetched artifact
|
... | ... | @@ -1738,10 +1745,10 @@ class Element(Plugin): |
1738 | 1745 |
|
1739 | 1746 |
return True
|
1740 | 1747 |
|
1741 |
- def _pull_weak(self, *, progress=None):
|
|
1748 |
+ def _pull_weak(self, *, progress=None, subdir=None, excluded_subdirs=None):
|
|
1742 | 1749 |
weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
|
1743 |
- |
|
1744 |
- if not self.__artifacts.pull(self, weak_key, progress=progress):
|
|
1750 |
+ if not self.__artifacts.pull(self, weak_key, progress=progress, subdir=subdir,
|
|
1751 |
+ excluded_subdirs=excluded_subdirs):
|
|
1745 | 1752 |
return False
|
1746 | 1753 |
|
1747 | 1754 |
# extract strong cache key from this newly fetched artifact
|
... | ... | @@ -1759,17 +1766,17 @@ class Element(Plugin): |
1759 | 1766 |
#
|
1760 | 1767 |
# Returns: True if the artifact has been downloaded, False otherwise
|
1761 | 1768 |
#
|
1762 |
- def _pull(self):
|
|
1769 |
+ def _pull(self, subdir=None, excluded_subdirs=None):
|
|
1763 | 1770 |
context = self._get_context()
|
1764 | 1771 |
|
1765 | 1772 |
def progress(percent, message):
|
1766 | 1773 |
self.status(message)
|
1767 | 1774 |
|
1768 | 1775 |
# Attempt to pull artifact without knowing whether it's available
|
1769 |
- pulled = self._pull_strong(progress=progress)
|
|
1776 |
+ pulled = self._pull_strong(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
|
|
1770 | 1777 |
|
1771 | 1778 |
if not pulled and not self._cached() and not context.get_strict():
|
1772 |
- pulled = self._pull_weak(progress=progress)
|
|
1779 |
+ pulled = self._pull_weak(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
|
|
1773 | 1780 |
|
1774 | 1781 |
if not pulled:
|
1775 | 1782 |
return False
|
... | ... | @@ -1792,10 +1799,21 @@ class Element(Plugin): |
1792 | 1799 |
if not self._cached():
|
1793 | 1800 |
return True
|
1794 | 1801 |
|
1795 |
- # Do not push tained artifact
|
|
1802 |
+ # Do not push tainted artifact
|
|
1796 | 1803 |
if self.__get_tainted():
|
1797 | 1804 |
return True
|
1798 | 1805 |
|
1806 |
+ # Do not push elements that have a dangling buildtree artifact unless element type is
|
|
1807 |
+ # expected to have an empty buildtree directory
|
|
1808 |
+ if not self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, 'buildtree'):
|
|
1809 |
+ return True
|
|
1810 |
+ |
|
1811 |
+ # strict_cache_key can't be relied on to be available when running in non strict mode
|
|
1812 |
+ context = self._get_context()
|
|
1813 |
+ if not context.get_strict():
|
|
1814 |
+ if not self.__artifacts.contains_subdir_artifact(self, self.__weak_cache_key, 'buildtree'):
|
|
1815 |
+ return True
|
|
1816 |
+ |
|
1799 | 1817 |
return False
|
1800 | 1818 |
|
1801 | 1819 |
# _push():
|
1 |
+import os
|
|
2 |
+import shutil
|
|
3 |
+import pytest
|
|
4 |
+ |
|
5 |
+from tests.testutils import cli_integration as cli, create_artifact_share
|
|
6 |
+from tests.testutils.integration import assert_contains
|
|
7 |
+ |
|
8 |
+ |
|
9 |
+DATA_DIR = os.path.join(
|
|
10 |
+ os.path.dirname(os.path.realpath(__file__)),
|
|
11 |
+ "project"
|
|
12 |
+)
|
|
13 |
+ |
|
14 |
+ |
|
15 |
+# Remove artifact cache & set cli.config value of pullbuildtrees
|
|
16 |
+# to false, which is the default user context. The cache has to be
|
|
17 |
+# cleared as just forcefully removing the refpath leaves dangling objects.
|
|
18 |
+def default_state(cli, tmpdir, share):
|
|
19 |
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
|
|
20 |
+ cli.configure({
|
|
21 |
+ 'pullbuildtrees': False,
|
|
22 |
+ 'artifacts': {'url': share.repo, 'push': False},
|
|
23 |
+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
|
|
24 |
+ })
|
|
25 |
+ |
|
26 |
+ |
|
27 |
+# A test to capture the integration of the pullbuildtrees
|
|
28 |
+# behaviour, which by default is to not include the buildtree
|
|
29 |
+# directory of an element.
|
|
30 |
+@pytest.mark.integration
|
|
31 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
32 |
+def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
|
|
33 |
+ |
|
34 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
35 |
+ element_name = 'autotools/amhello.bst'
|
|
36 |
+ |
|
37 |
+ # Create artifact shares for pull & push testing
|
|
38 |
+ with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
|
|
39 |
+ create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2:
|
|
40 |
+ cli.configure({
|
|
41 |
+ 'artifacts': {'url': share1.repo, 'push': True},
|
|
42 |
+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
|
|
43 |
+ })
|
|
44 |
+ |
|
45 |
+ # Build autotools element, checked pushed, delete local
|
|
46 |
+ result = cli.run(project=project, args=['build', element_name])
|
|
47 |
+ assert result.exit_code == 0
|
|
48 |
+ assert cli.get_element_state(project, element_name) == 'cached'
|
|
49 |
+ assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
|
|
50 |
+ default_state(cli, tmpdir, share1)
|
|
51 |
+ |
|
52 |
+ # Pull artifact with default config, assert that pulling again
|
|
53 |
+ # doesn't create a pull job, then assert with buildtrees user
|
|
54 |
+ # config set creates a pull job.
|
|
55 |
+ result = cli.run(project=project, args=['pull', element_name])
|
|
56 |
+ assert element_name in result.get_pulled_elements()
|
|
57 |
+ result = cli.run(project=project, args=['pull', element_name])
|
|
58 |
+ assert element_name not in result.get_pulled_elements()
|
|
59 |
+ cli.configure({'pullbuildtrees': True})
|
|
60 |
+ result = cli.run(project=project, args=['pull', element_name])
|
|
61 |
+ assert element_name in result.get_pulled_elements()
|
|
62 |
+ default_state(cli, tmpdir, share1)
|
|
63 |
+ |
|
64 |
+ # Pull artifact with default config, then assert that pulling
|
|
65 |
+ # with buildtrees cli flag set creates a pull job.
|
|
66 |
+ result = cli.run(project=project, args=['pull', element_name])
|
|
67 |
+ assert element_name in result.get_pulled_elements()
|
|
68 |
+ result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
|
|
69 |
+ assert element_name in result.get_pulled_elements()
|
|
70 |
+ default_state(cli, tmpdir, share1)
|
|
71 |
+ |
|
72 |
+ # Pull artifact with pullbuildtrees set in user config, then assert
|
|
73 |
+ # that pulling with the same user config doesn't creates a pull job,
|
|
74 |
+ # or when buildtrees cli flag is set.
|
|
75 |
+ cli.configure({'pullbuildtrees': True})
|
|
76 |
+ result = cli.run(project=project, args=['pull', element_name])
|
|
77 |
+ assert element_name in result.get_pulled_elements()
|
|
78 |
+ result = cli.run(project=project, args=['pull', element_name])
|
|
79 |
+ assert element_name not in result.get_pulled_elements()
|
|
80 |
+ result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
|
|
81 |
+ assert element_name not in result.get_pulled_elements()
|
|
82 |
+ default_state(cli, tmpdir, share1)
|
|
83 |
+ |
|
84 |
+ # Pull artifact with default config and buildtrees cli flag set, then assert
|
|
85 |
+ # that pulling with pullbuildtrees set in user config doesn't create a pull
|
|
86 |
+ # job.
|
|
87 |
+ result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
|
|
88 |
+ assert element_name in result.get_pulled_elements()
|
|
89 |
+ cli.configure({'pullbuildtrees': True})
|
|
90 |
+ result = cli.run(project=project, args=['pull', element_name])
|
|
91 |
+ assert element_name not in result.get_pulled_elements()
|
|
92 |
+ default_state(cli, tmpdir, share1)
|
|
93 |
+ |
|
94 |
+ # Assert that a partial build element (not containing a populated buildtree dir)
|
|
95 |
+ # can't be pushed to an artifact share, then assert that a complete build element
|
|
96 |
+ # can be. This will attempt a partial pull from share1 and then a partial push
|
|
97 |
+ # to share2
|
|
98 |
+ result = cli.run(project=project, args=['pull', element_name])
|
|
99 |
+ assert element_name in result.get_pulled_elements()
|
|
100 |
+ cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
|
|
101 |
+ result = cli.run(project=project, args=['push', element_name])
|
|
102 |
+ assert element_name not in result.get_pushed_elements()
|
|
103 |
+ assert not share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
|
|
104 |
+ |
|
105 |
+ # Assert that after pulling the missing buildtree the element artifact can be
|
|
106 |
+ # successfully pushed to the remote. This will attempt to pull the buildtree
|
|
107 |
+ # from share1 and then a 'complete' push to share2
|
|
108 |
+ cli.configure({'artifacts': {'url': share1.repo, 'push': False}})
|
|
109 |
+ result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
|
|
110 |
+ assert element_name in result.get_pulled_elements()
|
|
111 |
+ cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
|
|
112 |
+ result = cli.run(project=project, args=['push', element_name])
|
|
113 |
+ assert element_name in result.get_pushed_elements()
|
|
114 |
+ assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
|
|
115 |
+ default_state(cli, tmpdir, share1)
|