[Notes] [Git][BuildStream/buildstream][tpollard/566] WIP: Make uploading of build trees configurable

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/566 at BuildStream / buildstream

Commits:

• 0f5028dd
  by Tom Pollard at 2019-01-07T12:41:16Z

  WIP: Make uploading of build trees configurable
  

5 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/element.py
• + tests/integration/pushbuildtrees.py
• tests/testutils/runcli.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -74,6 +74,7 @@ class ArtifactCache():
74	74
75	75	self._has_fetch_remotes = False
76	76	self._has_push_remotes = False
	77	+ self._has_partial_push_remotes = False
77	78
78	79	os.makedirs(self.extractdir, exist_ok=True)
79	80
...	...	@@ -398,6 +399,8 @@ class ArtifactCache():
398	399	self._has_fetch_remotes = True
399	400	if remote_spec.push:
400	401	self._has_push_remotes = True
	402	+ if remote_spec.partial_push:
	403	+ self._has_partial_push_remotes = True
401	404
402	405	remotes[remote_spec.url] = CASRemote(remote_spec)
403	406
...	...	@@ -596,6 +599,31 @@ class ArtifactCache():
596	599	remotes_for_project = self._remotes[element._get_project()]
597	600	return any(remote.spec.push for remote in remotes_for_project)
598	601
	602	+ # has_partial_push_remotes():
	603	+ #
	604	+ # Check whether any remote repositories are available for pushing
	605	+ # non-complete artifacts
	606	+ #
	607	+ # Args:
	608	+ # element (Element): The Element to check
	609	+ #
	610	+ # Returns:
	611	+ # (bool): True if any remote repository is configured for optional
	612	+ # partial pushes, False otherwise
	613	+ #
	614	+ def has_partial_push_remotes(self, *, element=None):
	615	+ # If there's no partial push remotes available, we can't partial push at all
	616	+ if not self._has_partial_push_remotes:
	617	+ return False
	618	+ elif element is None:
	619	+ # At least one remote is set to allow partial pushes
	620	+ return True
	621	+ else:
	622	+ # Check whether the specified element's project has push remotes configured
	623	+ # to not accept partial artifact pushes
	624	+ remotes_for_project = self._remotes[element._get_project()]
	625	+ return any(remote.spec.partial_push for remote in remotes_for_project)
	626	+
599	627	# push():
600	628	#
601	629	# Push committed artifact to remote repository.
...	...	@@ -603,6 +631,8 @@ class ArtifactCache():
603	631	# Args:
604	632	# element (Element): The Element whose artifact is to be pushed
605	633	# keys (list): The cache keys to use
	634	+ # partial(bool): If the artifact is cached in a partial state
	635	+ # subdir(string): Optional subdir to not push
606	636	#
607	637	# Returns:
608	638	# (bool): True if any remote was updated, False if no pushes were required
...	...	@@ -610,12 +640,25 @@ class ArtifactCache():
610	640	# Raises:
611	641	# (ArtifactError): if there was an error
612	642	#
613		- def push(self, element, keys):
	643	+ def push(self, element, keys, partial=False, subdir=None):
614	644	refs = [self.get_artifact_fullname(element, key) for key in list(keys)]
615	645
616	646	project = element._get_project()
617	647
618		- push_remotes = [r for r in self._remotes[project] if r.spec.push]
	648	+ push_remotes = []
	649	+ partial_remotes = []
	650	+
	651	+ # Create list of remotes to push to, given current element and partial push config
	652	+ if not partial:
	653	+ push_remotes = [r for r in self._remotes[project] if (r.spec.push and not r.spec.partial_push)]
	654	+
	655	+ if self._has_partial_push_remotes:
	656	+ # Create a specific list of the remotes expecting the artifact to be push in a partial
	657	+ # state. This list needs to be pushed in a partial state, without the optional subdir if
	658	+ # exists locally. No need to attempt pushing a partial artifact to a remote that is queued to
	659	+ # to also recieve a full artifact
	660	+ partial_remotes = [r for r in self._remotes[project] if (r.spec.partial_push and r.spec.push) and
	661	+ r not in push_remotes]
619	662
620	663	pushed = False
621	664
...	...	@@ -632,6 +675,19 @@ class ArtifactCache():
632	675	remote.spec.url, element._get_brief_display_key()
633	676	))
634	677
	678	+ for remote in partial_remotes:
	679	+ remote.init()
	680	+ display_key = element._get_brief_display_key()
	681	+ element.status("Pushing partial artifact {} -> {}".format(display_key, remote.spec.url))
	682	+
	683	+ if self.cas.push(refs, remote, subdir=subdir):
	684	+ element.info("Pushed partial artifact {} -> {}".format(display_key, remote.spec.url))
	685	+ pushed = True
	686	+ else:
	687	+ element.info("Remote ({}) already has {} partial cached".format(
	688	+ remote.spec.url, element._get_brief_display_key()
	689	+ ))
	690	+
635	691	return pushed
636	692
637	693	# pull():
...	...	@@ -659,14 +715,23 @@ class ArtifactCache():
659	715	element.status("Pulling artifact {} <- {}".format(display_key, remote.spec.url))
660	716
661	717	if self.cas.pull(ref, remote, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
662		- element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
663	718	if subdir:
664		- # Attempt to extract subdir into artifact extract dir if it already exists
665		- # without containing the subdir. If the respective artifact extract dir does not
666		- # exist a complete extraction will complete.
667		- self.extract(element, key, subdir)
668		- # no need to pull from additional remotes
669		- return True
	719	+ if not self.contains_subdir_artifact(element, key, subdir):
	720	+ # The pull was expecting the specific subdir to be present, attempt
	721	+ # to find it in other available remotes
	722	+ element.info("Pulled partial artifact {} <- {}. Attempting to retrieve {} from remotes"
	723	+ .format(display_key, remote.spec.url, subdir))
	724	+ else:
	725	+ element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
	726	+ # Attempt to extract subdir into artifact extract dir if it already exists
	727	+ # without containing the subdir. If the respective artifact extract dir does not
	728	+ # exist a complete extraction will complete.
	729	+ self.extract(element, key, subdir)
	730	+ # no need to pull from additional remotes
	731	+ return True
	732	+ else:
	733	+ element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
	734	+ return True
670	735	else:
671	736	element.info("Remote ({}) does not have {} cached".format(
672	737	remote.spec.url, element._get_brief_display_key()

buildstream/_artifactcache/cascache.py

---

...	...	@@ -45,7 +45,8 @@ from .. import _yaml
45	45	_MAX_PAYLOAD_BYTES = 1024 * 1024
46	46
47	47
48		-class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key client_cert instance_name')):
	48	+class CASRemoteSpec(namedtuple('CASRemoteSpec',
	49	+ 'url push partial_push server_cert client_key client_cert instance_name')):
49	50
50	51	# _new_from_config_node
51	52	#
...	...	@@ -53,9 +54,13 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key
53	54	#
54	55	@staticmethod
55	56	def _new_from_config_node(spec_node, basedir=None):
56		- _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert', 'instance_name'])
	57	+ _yaml.node_validate(spec_node,
	58	+ ['url', 'push', 'allow-partial-push', 'server-cert', 'client-key',
	59	+ 'client-cert', 'instance_name'])
57	60	url = _yaml.node_get(spec_node, str, 'url')
58	61	push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
	62	+ partial_push = _yaml.node_get(spec_node, bool, 'allow-partial-push', default_value=False)
	63	+
59	64	if not url:
60	65	provenance = _yaml.node_get_provenance(spec_node, 'url')
61	66	raise LoadError(LoadErrorReason.INVALID_DATA,
...	...	@@ -85,10 +90,10 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key
85	90	raise LoadError(LoadErrorReason.INVALID_DATA,
86	91	"{}: 'client-cert' was specified without 'client-key'".format(provenance))
87	92
88		- return CASRemoteSpec(url, push, server_cert, client_key, client_cert, instance_name)
	93	+ return CASRemoteSpec(url, push, partial_push, server_cert, client_key, client_cert, instance_name)
89	94
90	95
91		-CASRemoteSpec.__new__.__defaults__ = (None, None, None, None)
	96	+CASRemoteSpec.__new__.__defaults__ = (False, None, None, None, None)
92	97
93	98
94	99	class BlobNotFound(CASError):
...	...	@@ -283,34 +288,47 @@ class CASCache():
283	288	# (bool): True if pull was successful, False if ref was not available
284	289	#
285	290	def pull(self, ref, remote, *, progress=None, subdir=None, excluded_subdirs=None):
286		- try:
287		- remote.init()
288	291
289		- request = buildstream_pb2.GetReferenceRequest(instance_name=remote.spec.instance_name)
290		- request.key = ref
291		- response = remote.ref_storage.GetReference(request)
	292	+ tree_found = False
292	293
293		- tree = remote_execution_pb2.Digest()
294		- tree.hash = response.digest.hash
295		- tree.size_bytes = response.digest.size_bytes
	294	+ while True:
	295	+ try:
	296	+ if not tree_found:
	297	+ remote.init()
296	298
297		- # Check if the element artifact is present, if so just fetch the subdir.
298		- if subdir and os.path.exists(self.objpath(tree)):
299		- self._fetch_subdir(remote, tree, subdir)
300		- else:
301		- # Fetch artifact, excluded_subdirs determined in pullqueue
302		- self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
	299	+ request = buildstream_pb2.GetReferenceRequest(instance_name=remote.spec.instance_name)
	300	+ request.key = ref
	301	+ response = remote.ref_storage.GetReference(request)
303	302
304		- self.set_ref(ref, tree)
	303	+ tree = remote_execution_pb2.Digest()
	304	+ tree.hash = response.digest.hash
	305	+ tree.size_bytes = response.digest.size_bytes
305	306
306		- return True
307		- except grpc.RpcError as e:
308		- if e.code() != grpc.StatusCode.NOT_FOUND:
309		- raise CASError("Failed to pull ref {}: {}".format(ref, e)) from e
310		- else:
311		- return False
312		- except BlobNotFound as e:
313		- return False
	307	+ # Check if the element artifact is present, if so just fetch the subdir.
	308	+ if subdir and os.path.exists(self.objpath(tree)):
	309	+ self._fetch_subdir(remote, tree, subdir)
	310	+ else:
	311	+ # Fetch artifact, excluded_subdirs determined in pullqueue
	312	+ self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
	313	+
	314	+ self.set_ref(ref, tree)
	315	+
	316	+ return True
	317	+ except grpc.RpcError as e:
	318	+ if e.code() != grpc.StatusCode.NOT_FOUND:
	319	+ raise CASError("Failed to pull ref {}: {}".format(ref, e)) from e
	320	+ else:
	321	+ return False
	322	+ except BlobNotFound as e:
	323	+ if not excluded_subdirs and subdir:
	324	+ # The remote has the top level digest but could not complete a full pull,
	325	+ # attempt partial without the need to initialise and check for the artifact
	326	+ # digest. This default behaviour of dropping back to partial pulls could
	327	+ # be made a configurable warning given at artfictcache level.
	328	+ tree_found = True
	329	+ excluded_subdirs, subdir = subdir, excluded_subdirs
	330	+ else:
	331	+ return False
314	332
315	333	# pull_tree():
316	334	#
...	...	@@ -355,6 +373,7 @@ class CASCache():
355	373	# Args:
356	374	# refs (list): The refs to push
357	375	# remote (CASRemote): The remote to push to
	376	+ # subdir (string): Optional specific subdir to exempt from the push
358	377	#
359	378	# Returns:
360	379	# (bool): True if any remote was updated, False if no pushes were required
...	...	@@ -362,7 +381,7 @@ class CASCache():
362	381	# Raises:
363	382	# (CASError): if there was an error
364	383	#
365		- def push(self, refs, remote):
	384	+ def push(self, refs, remote, subdir=None):
366	385	skipped_remote = True
367	386	try:
368	387	for ref in refs:
...	...	@@ -384,7 +403,7 @@ class CASCache():
384	403	# Intentionally re-raise RpcError for outer except block.
385	404	raise
386	405
387		- self._send_directory(remote, tree)
	406	+ self._send_directory(remote, tree, excluded_dir=subdir)
388	407
389	408	request = buildstream_pb2.UpdateReferenceRequest(instance_name=remote.spec.instance_name)
390	409	request.keys.append(ref)
...	...	@@ -866,10 +885,17 @@ class CASCache():
866	885	a += 1
867	886	b += 1
868	887
869		- def _reachable_refs_dir(self, reachable, tree, update_mtime=False):
	888	+ def _reachable_refs_dir(self, reachable, tree, update_mtime=False, subdir=False):
870	889	if tree.hash in reachable:
871	890	return
872	891
	892	+ # If looping through subdir digests, skip processing if
	893	+ # ref path does not exist, allowing for partial objects
	894	+ if subdir and not os.path.exists(self.objpath(tree)):
	895	+ return
	896	+
	897	+ # Raises FileNotFound exception is path does not exist,
	898	+ # which should only be entered on the top level digest
873	899	if update_mtime:
874	900	os.utime(self.objpath(tree))
875	901
...	...	@@ -886,9 +912,9 @@ class CASCache():
886	912	reachable.add(filenode.digest.hash)
887	913
888	914	for dirnode in directory.directories:
889		- self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime)
	915	+ self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime, subdir=True)
890	916
891		- def _required_blobs(self, directory_digest):
	917	+ def _required_blobs(self, directory_digest, excluded_dir=None):
892	918	# parse directory, and recursively add blobs
893	919	d = remote_execution_pb2.Digest()
894	920	d.hash = directory_digest.hash
...	...	@@ -907,7 +933,8 @@ class CASCache():
907	933	yield d
908	934
909	935	for dirnode in directory.directories:
910		- yield from self._required_blobs(dirnode.digest)
	936	+ if dirnode.name != excluded_dir:
	937	+ yield from self._required_blobs(dirnode.digest)
911	938
912	939	def _fetch_blob(self, remote, digest, stream):
913	940	resource_name_components = ['blobs', digest.hash, str(digest.size_bytes)]
...	...	@@ -1029,6 +1056,7 @@ class CASCache():
1029	1056	objpath = self._ensure_blob(remote, dir_digest)
1030	1057
1031	1058	directory = remote_execution_pb2.Directory()
	1059	+
1032	1060	with open(objpath, 'rb') as f:
1033	1061	directory.ParseFromString(f.read())
1034	1062
...	...	@@ -1104,9 +1132,8 @@ class CASCache():
1104	1132
1105	1133	assert response.committed_size == digest.size_bytes
1106	1134
1107		- def _send_directory(self, remote, digest, u_uid=uuid.uuid4()):
1108		- required_blobs = self._required_blobs(digest)
1109		-
	1135	+ def _send_directory(self, remote, digest, u_uid=uuid.uuid4(), excluded_dir=None):
	1136	+ required_blobs = self._required_blobs(digest, excluded_dir=excluded_dir)
1110	1137	missing_blobs = dict()
1111	1138	# Limit size of FindMissingBlobs request
1112	1139	for required_blobs_group in _grouper(required_blobs, 512):

buildstream/element.py

---

...	...	@@ -1800,13 +1800,19 @@ class Element(Plugin):
1800	1800	# (bool): True if this element does not need a push job to be created
1801	1801	#
1802	1802	def _skip_push(self):
	1803	+
1803	1804	if not self.__artifacts.has_push_remotes(element=self):
1804	1805	# No push remotes for this element's project
1805	1806	return True
1806	1807
1807	1808	# Do not push elements that aren't cached, or that are cached with a dangling buildtree
1808		- # artifact unless element type is expected to have an an empty buildtree directory
1809		- if not self._cached_buildtree():
	1809	+ # artifact unless element type is expected to have an an empty buildtree directory. Check
	1810	+ # that this default behaviour is not overriden via a remote configured to allow pushing
	1811	+ # artifacts without their corresponding buildtree.
	1812	+ if not self._cached():
	1813	+ return True
	1814	+
	1815	+ if not self._cached_buildtree() and not self.__artifacts.has_partial_push_remotes(element=self):
1810	1816	return True
1811	1817
1812	1818	# Do not push tainted artifact
...	...	@@ -1817,7 +1823,8 @@ class Element(Plugin):
1817	1823
1818	1824	# _push():
1819	1825	#
1820		- # Push locally cached artifact to remote artifact repository.
	1826	+ # Push locally cached artifact to remote artifact repository. An attempt
	1827	+ # will be made to push partial artifacts given current config
1821	1828	#
1822	1829	# Returns:
1823	1830	# (bool): True if the remote was updated, False if it already existed
...	...	@@ -1830,8 +1837,19 @@ class Element(Plugin):
1830	1837	self.warn("Not pushing tainted artifact.")
1831	1838	return False
1832	1839
1833		- # Push all keys used for local commit
1834		- pushed = self.__artifacts.push(self, self.__get_cache_keys_for_commit())
	1840	+ # Push all keys used for local commit, this could be full or partial,
	1841	+ # given previous _skip_push() logic. If buildtree isn't cached, then
	1842	+ # set partial push
	1843	+
	1844	+ partial = False
	1845	+ subdir = 'buildtree'
	1846	+ if not self._cached_buildtree():
	1847	+ partial = True
	1848	+
	1849	+ pushed = self.__artifacts.push(self, self.__get_cache_keys_for_commit(), partial=partial, subdir=subdir)
	1850	+
	1851	+ # Artifact might be cached in the server partially with the top level ref existing.
	1852	+ # Check if we need to attempt a push of a locally cached buildtree given current config
1835	1853	if not pushed:
1836	1854	return False
1837	1855

tests/integration/pushbuildtrees.py

---

	1	+import os
	2	+import shutil
	3	+import pytest
	4	+
	5	+from tests.testutils import cli_integration as cli, create_artifact_share
	6	+from tests.testutils.integration import assert_contains
	7	+from tests.testutils.site import HAVE_BWRAP, IS_LINUX
	8	+from buildstream._exceptions import ErrorDomain, LoadErrorReason
	9	+
	10	+
	11	+DATA_DIR = os.path.join(
	12	+ os.path.dirname(os.path.realpath(__file__)),
	13	+ "project"
	14	+)
	15	+
	16	+
	17	+# Remove artifact cache & set cli.config value of pull-buildtrees
	18	+# to false, which is the default user context. The cache has to be
	19	+# cleared as just forcefully removing the refpath leaves dangling objects.
	20	+def default_state(cli, tmpdir, share):
	21	+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
	22	+ cli.configure({
	23	+ 'artifacts': {'url': share.repo, 'push': False},
	24	+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts'),
	25	+ 'cache': {'pull-buildtrees': False},
	26	+ })
	27	+
	28	+
	29	+# Tests to capture the integration of the optionl push of buildtrees.
	30	+# The behaviour should encompass pushing artifacts that are already cached
	31	+# without a buildtree as well as artifacts that are cached with their buildtree.
	32	+# This option is handled via 'allow-partial-push' on a per artifact remote config
	33	+# node basis. Multiple remote config nodes can point to the same url and as such can
	34	+# have different 'allow-partial-push' options, tests need to cover this using project
	35	+# confs.
	36	+@pytest.mark.integration
	37	+@pytest.mark.datafiles(DATA_DIR)
	38	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	39	+def test_pushbuildtrees(cli, tmpdir, datafiles, integration_cache):
	40	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	41	+ element_name = 'autotools/amhello.bst'
	42	+
	43	+ # Create artifact shares for pull & push testing
	44	+ with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
	45	+ create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
	46	+ create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3,\
	47	+ create_artifact_share(os.path.join(str(tmpdir), 'share4')) as share4:
	48	+
	49	+ cli.configure({
	50	+ 'artifacts': {'url': share1.repo, 'push': True},
	51	+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
	52	+ })
	53	+
	54	+ cli.configure({'artifacts': [{'url': share1.repo, 'push': True},
	55	+ {'url': share2.repo, 'push': True, 'allow-partial-push': True}]})
	56	+
	57	+ # Build autotools element, checked pushed, delete local.
	58	+ # As share 2 has push & allow-partial-push set a true, it
	59	+ # should have pushed the artifacts, without the cached buildtrees,
	60	+ # to it.
	61	+ result = cli.run(project=project, args=['build', element_name])
	62	+ assert result.exit_code == 0
	63	+ assert cli.get_element_state(project, element_name) == 'cached'
	64	+ elementdigest = share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	65	+ buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
	66	+ elementdigest.hash, 'buildtree')
	67	+ assert os.path.isdir(buildtreedir)
	68	+ assert element_name in result.get_partial_pushed_elements()
	69	+ assert element_name in result.get_pushed_elements()
	70	+ assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	71	+ assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	72	+ default_state(cli, tmpdir, share1)
	73	+
	74	+ # Check that after explictly pulling an artifact without it's buildtree,
	75	+ # we can push it to another remote that is configured to accept the partial
	76	+ # artifact
	77	+ result = cli.run(project=project, args=['pull', element_name])
	78	+ assert element_name in result.get_pulled_elements()
	79	+ cli.configure({'artifacts': {'url': share3.repo, 'push': True, 'allow-partial-push': True}})
	80	+ assert cli.get_element_state(project, element_name) == 'cached'
	81	+ assert not os.path.isdir(buildtreedir)
	82	+ result = cli.run(project=project, args=['push', element_name])
	83	+ assert result.exit_code == 0
	84	+ assert element_name in result.get_partial_pushed_elements()
	85	+ assert element_name not in result.get_pushed_elements()
	86	+ assert share3.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	87	+ default_state(cli, tmpdir, share3)
	88	+
	89	+ # Delete the local cache and pull the partial artifact from share 3,
	90	+ # this should not include the buildtree when extracted locally, even when
	91	+ # pull-buildtrees is given as a cli parameter as no available remotes will
	92	+ # contain the buildtree
	93	+ assert not os.path.isdir(buildtreedir)
	94	+ assert cli.get_element_state(project, element_name) != 'cached'
	95	+ result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name])
	96	+ assert element_name in result.get_partial_pulled_elements()
	97	+ assert not os.path.isdir(buildtreedir)
	98	+ default_state(cli, tmpdir, share3)
	99	+
	100	+ # Delete the local cache and attempt to pull a 'full' artifact, including its
	101	+ # buildtree. As with before share3 being the first listed remote will not have
	102	+ # the buildtree available and should spawn a partial pull. Having share1 as the
	103	+ # second available remote should allow the buildtree to be pulled thus 'completing'
	104	+ # the artifact
	105	+ cli.configure({'artifacts': [{'url': share3.repo, 'push': True, 'allow-partial-push': True},
	106	+ {'url': share1.repo, 'push': True}]})
	107	+ assert cli.get_element_state(project, element_name) != 'cached'
	108	+ result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name])
	109	+ assert element_name in result.get_partial_pulled_elements()
	110	+ assert element_name in result.get_pulled_elements()
	111	+ assert "Attempting to retrieve buildtree from remotes" in result.stderr
	112	+ assert os.path.isdir(buildtreedir)
	113	+ assert cli.get_element_state(project, element_name) == 'cached'

tests/testutils/runcli.py

---

...	...	@@ -191,6 +191,13 @@ class Result():
191	191
192	192	return list(pushed)
193	193
	194	+ def get_partial_pushed_elements(self):
	195	+ pushed = re.findall(r'\[\s*push:(\S+)\s*\]\s*INFO\s*Pushed partial artifact', self.stderr)
	196	+ if pushed is None:
	197	+ return []
	198	+
	199	+ return list(pushed)
	200	+
194	201	def get_pulled_elements(self):
195	202	pulled = re.findall(r'\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled artifact', self.stderr)
196	203	if pulled is None:
...	...	@@ -198,6 +205,13 @@ class Result():
198	205
199	206	return list(pulled)
200	207
	208	+ def get_partial_pulled_elements(self):
	209	+ pulled = re.findall(r'\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled partial artifact', self.stderr)
	210	+ if pulled is None:
	211	+ return []
	212	+
	213	+ return list(pulled)
	214	+
201	215
202	216	class Cli():
203	217

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.