[Notes] [Git][BuildStream/buildstream][tpollard/494] WIP: Don't pull artifact buildtrees by default

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/494 at BuildStream / buildstream

Commits:

• 7762bb03
  by Tom Pollard at 2018-10-10T14:15:38Z

  WIP: Don't pull artifact buildtrees by default
  
  - Set default pull to not include buildtree artifact dir
  - PullQueue configurable subdir attribute
  - Add 'pullbuildtrees' global option to for user conf
  - Add --pull-buildtrees flag to bst build cli
  - Add --pull-buildtrees flag to bst pull cli
  - Add helper function _fetch_subdir to cascache, to fetch buildtree
    or any other subdir digest
  - Make element._pull_pending not assume no need to process pull if
    artifact is cached if buildtrees are set to be pulled
  - Ensure cascache.py doesn't try to checkout/extract a dangling ref
  - Required test and changes
  

10 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_scheduler/queues/pullqueue.py
• buildstream/_stream.py
• buildstream/element.py
• tests/completions/completions.py
• + tests/integration/pullbuildtrees.py
• tests/testutils/artifactshare.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -426,6 +426,22 @@ class ArtifactCache():
426	426	raise ImplError("Cache '{kind}' does not implement contains()"
427	427	.format(kind=type(self).__name__))
428	428
	429	+ # contains_subdir_artifact():
	430	+ #
	431	+ # Check whether an artifact element contains a digest for a subdir
	432	+ # which is populated in the cache, i.e non dangling.
	433	+ #
	434	+ # Args:
	435	+ # element (Element): The Element to check
	436	+ # key (str): The cache key to use
	437	+ # subdir (str): The subdir to check
	438	+ #
	439	+ # Returns: True if the subdir exists & is populated in the cache, False otherwise
	440	+ #
	441	+ def contains_subdir_artifact(self, element, key, subdir):
	442	+ raise ImplError("Cache '{kind}' does not implement contains_subdir_artifact()"
	443	+ .format(kind=type(self).__name__))
	444	+
429	445	# list_artifacts():
430	446	#
431	447	# List artifacts in this cache in LRU order.
...	...	@@ -551,11 +567,12 @@ class ArtifactCache():
551	567	# element (Element): The Element whose artifact is to be fetched
552	568	# key (str): The cache key to use
553	569	# progress (callable): The progress callback, if any
	570	+ # subdir (str): The optional specific subdir to pull
554	571	#
555	572	# Returns:
556	573	# (bool): True if pull was successful, False if artifact was not available
557	574	#
558		- def pull(self, element, key, *, progress=None):
	575	+ def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
559	576	raise ImplError("Cache '{kind}' does not implement pull()"
560	577	.format(kind=type(self).__name__))
561	578

buildstream/_artifactcache/cascache.py

---

...	...	@@ -92,6 +92,16 @@ class CASCache(ArtifactCache):
92	92	# This assumes that the repository doesn't have any dangling pointers
93	93	return os.path.exists(refpath)
94	94
	95	+ def contains_subdir_artifact(self, element, key, subdir):
	96	+ tree = self.resolve_ref(self.get_artifact_fullname(element, key))
	97	+
	98	+ # This assumes that the subdir digest is present in the element tree
	99	+ subdirdigest = self._get_subdir(tree, subdir)
	100	+ objpath = self.objpath(subdirdigest)
	101	+
	102	+ # True if subdir content is cached or if empty as expected
	103	+ return os.path.exists(objpath)
	104	+
95	105	def extract(self, element, key):
96	106	ref = self.get_artifact_fullname(element, key)
97	107
...	...	@@ -228,7 +238,7 @@ class CASCache(ArtifactCache):
228	238	remotes_for_project = self._remotes[element._get_project()]
229	239	return any(remote.spec.push for remote in remotes_for_project)
230	240
231		- def pull(self, element, key, *, progress=None):
	241	+ def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
232	242	ref = self.get_artifact_fullname(element, key)
233	243
234	244	project = element._get_project()
...	...	@@ -247,8 +257,14 @@ class CASCache(ArtifactCache):
247	257	tree.hash = response.digest.hash
248	258	tree.size_bytes = response.digest.size_bytes
249	259
250		- self._fetch_directory(remote, tree)
	260	+ # Check if the element artifact is present, if so just fetch subdir
	261	+ if subdir and os.path.exists(self.objpath(tree)):
	262	+ self._fetch_subdir(remote, tree, subdir)
	263	+ else:
	264	+ # Fetch artifact, excluded_subdirs determined in pullqueue
	265	+ self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
251	266
	267	+ # tree is the remote value, so is the same without or without dangling ref locally
252	268	self.set_ref(ref, tree)
253	269
254	270	element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
...	...	@@ -668,8 +684,10 @@ class CASCache(ArtifactCache):
668	684	stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
669	685
670	686	for dirnode in directory.directories:
671		- fullpath = os.path.join(dest, dirnode.name)
672		- self._checkout(fullpath, dirnode.digest)
	687	+ # Don't try to checkout a dangling ref
	688	+ if os.path.exists(self.objpath(dirnode.digest)):
	689	+ fullpath = os.path.join(dest, dirnode.name)
	690	+ self._checkout(fullpath, dirnode.digest)
673	691
674	692	for symlinknode in directory.symlinks:
675	693	# symlink
...	...	@@ -948,10 +966,12 @@ class CASCache(ArtifactCache):
948	966	# remote (Remote): The remote to use.
949	967	# dir_digest (Digest): Digest object for the directory to fetch.
950	968	#
951		- def _fetch_directory(self, remote, dir_digest):
	969	+ def _fetch_directory(self, remote, dir_digest, *, excluded_subdirs=None):
952	970	fetch_queue = [dir_digest]
953	971	fetch_next_queue = []
954	972	batch = _CASBatchRead(remote)
	973	+ if not excluded_subdirs:
	974	+ excluded_subdirs = []
955	975
956	976	while len(fetch_queue) + len(fetch_next_queue) > 0:
957	977	if len(fetch_queue) == 0:
...	...	@@ -966,8 +986,9 @@ class CASCache(ArtifactCache):
966	986	directory.ParseFromString(f.read())
967	987
968	988	for dirnode in directory.directories:
969		- batch = self._fetch_directory_node(remote, dirnode.digest, batch,
970		- fetch_queue, fetch_next_queue, recursive=True)
	989	+ if dirnode.name not in excluded_subdirs:
	990	+ batch = self._fetch_directory_node(remote, dirnode.digest, batch,
	991	+ fetch_queue, fetch_next_queue, recursive=True)
971	992
972	993	for filenode in directory.files:
973	994	batch = self._fetch_directory_node(remote, filenode.digest, batch,
...	...	@@ -976,6 +997,10 @@ class CASCache(ArtifactCache):
976	997	# Fetch final batch
977	998	self._fetch_directory_batch(remote, batch, fetch_queue, fetch_next_queue)
978	999
	1000	+ def _fetch_subdir(self, remote, tree, subdir):
	1001	+ subdirdigest = self._get_subdir(tree, subdir)
	1002	+ self._fetch_directory(remote, subdirdigest)
	1003	+
979	1004	def _fetch_tree(self, remote, digest):
980	1005	# download but do not store the Tree object
981	1006	with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:

buildstream/_context.py

---

...	...	@@ -110,6 +110,9 @@ class Context():
110	110	# Make sure the XDG vars are set in the environment before loading anything
111	111	self._init_xdg()
112	112
	113	+ # Whether or not to attempt to pull buildtrees globally
	114	+ self.pullbuildtrees = False
	115	+
113	116	# Private variables
114	117	self._cache_key = None
115	118	self._message_handler = None
...	...	@@ -160,7 +163,7 @@ class Context():
160	163	_yaml.node_validate(defaults, [
161	164	'sourcedir', 'builddir', 'artifactdir', 'logdir',
162	165	'scheduler', 'artifacts', 'logging', 'projects',
163		- 'cache'
	166	+ 'cache', 'pullbuildtrees'
164	167	])
165	168
166	169	for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
...	...	@@ -185,6 +188,9 @@ class Context():
185	188	# Load artifact share configuration
186	189	self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
187	190
	191	+ # Load pull buildtrees configuration
	192	+ self.pullbuildtrees = _yaml.node_get(defaults, bool, 'pullbuildtrees', default_value='False')
	193	+
188	194	# Load logging config
189	195	logging = _yaml.node_get(defaults, Mapping, 'logging')
190	196	_yaml.node_validate(logging, [

buildstream/_frontend/cli.py

---

...	...	@@ -305,10 +305,12 @@ def init(app, project_name, format_version, element_path, force):
305	305	help="Allow tracking to cross junction boundaries")
306	306	@click.option('--track-save', default=False, is_flag=True,
307	307	help="Deprecated: This is ignored")
	308	+@click.option('--pull-buildtrees', default=False, is_flag=True,
	309	+ help="Pull buildtrees from a remote cache server")
308	310	@click.argument('elements', nargs=-1,
309	311	type=click.Path(readable=False))
310	312	@click.pass_obj
311		-def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
	313	+def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions, pull_buildtrees):
312	314	"""Build elements in a pipeline"""
313	315
314	316	if (track_except or track_cross_junctions) and not (track_ or track_all):
...	...	@@ -327,7 +329,8 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
327	329	track_targets=track_,
328	330	track_except=track_except,
329	331	track_cross_junctions=track_cross_junctions,
330		- build_all=all_)
	332	+ build_all=all_,
	333	+ pull_buildtrees=pull_buildtrees)
331	334
332	335
333	336	##################################################################
...	...	@@ -429,10 +432,12 @@ def track(app, elements, deps, except_, cross_junctions):
429	432	help='The dependency artifacts to pull (default: none)')
430	433	@click.option('--remote', '-r',
431	434	help="The URL of the remote cache (defaults to the first configured cache)")
	435	+@click.option('--pull-buildtrees', default=False, is_flag=True,
	436	+ help="Pull buildtrees from a remote cache server")
432	437	@click.argument('elements', nargs=-1,
433	438	type=click.Path(readable=False))
434	439	@click.pass_obj
435		-def pull(app, elements, deps, remote):
	440	+def pull(app, elements, deps, remote, pull_buildtrees):
436	441	"""Pull a built artifact from the configured remote artifact cache.
437	442
438	443	By default the artifact will be pulled one of the configured caches
...	...	@@ -446,7 +451,7 @@ def pull(app, elements, deps, remote):
446	451	all: All dependencies
447	452	"""
448	453	with app.initialized(session_name="Pull"):
449		- app.stream.pull(elements, selection=deps, remote=remote)
	454	+ app.stream.pull(elements, selection=deps, remote=remote, pull_buildtrees=pull_buildtrees)
450	455
451	456
452	457	##################################################################

buildstream/_scheduler/queues/pullqueue.py

---

...	...	@@ -32,9 +32,20 @@ class PullQueue(Queue):
32	32	complete_name = "Pulled"
33	33	resources = [ResourceType.DOWNLOAD, ResourceType.CACHE]
34	34
	35	+ def __init__(self, scheduler, buildtrees=False):
	36	+ super().__init__(scheduler)
	37	+
	38	+ # Current default exclusions on pull
	39	+ self._excluded_subdirs = ["buildtree"]
	40	+ self._subdir = None
	41	+ # If buildtrees are to be pulled, remove the value from exclusion list
	42	+ if buildtrees:
	43	+ self._subdir = "buildtree"
	44	+ self._excluded_subdirs.remove(self._subdir)
	45	+
35	46	def process(self, element):
36	47	# returns whether an artifact was downloaded or not
37		- if not element._pull():
	48	+ if not element._pull(subdir=self._subdir, excluded_subdirs=self._excluded_subdirs):
38	49	raise SkipJob(self.action_name)
39	50
40	51	def status(self, element):
...	...	@@ -49,7 +60,7 @@ class PullQueue(Queue):
49	60	if not element._can_query_cache():
50	61	return QueueStatus.WAIT
51	62
52		- if element._pull_pending():
	63	+ if element._pull_pending(subdir=self._subdir):
53	64	return QueueStatus.READY
54	65	else:
55	66	return QueueStatus.SKIP

buildstream/_stream.py

---

...	...	@@ -160,12 +160,14 @@ class Stream():
160	160	# track_cross_junctions (bool): Whether tracking should cross junction boundaries
161	161	# build_all (bool): Whether to build all elements, or only those
162	162	# which are required to build the target.
	163	+ # pull_buildtrees (bool): Whether to pull buildtrees from a remote cache server
163	164	#
164	165	def build(self, targets, *,
165	166	track_targets=None,
166	167	track_except=None,
167	168	track_cross_junctions=False,
168		- build_all=False):
	169	+ build_all=False,
	170	+ pull_buildtrees=False):
169	171
170	172	if build_all:
171	173	selection = PipelineSelection.ALL
...	...	@@ -195,7 +197,10 @@ class Stream():
195	197	self._add_queue(track_queue, track=True)
196	198
197	199	if self._artifacts.has_fetch_remotes():
198		- self._add_queue(PullQueue(self._scheduler))
	200	+ # Query if pullbuildtrees has been set globally in user config
	201	+ if self._context.pullbuildtrees:
	202	+ pull_buildtrees = True
	203	+ self._add_queue(PullQueue(self._scheduler, buildtrees=pull_buildtrees))
199	204
200	205	self._add_queue(FetchQueue(self._scheduler, skip_cached=True))
201	206	self._add_queue(BuildQueue(self._scheduler))
...	...	@@ -295,7 +300,8 @@ class Stream():
295	300	#
296	301	def pull(self, targets, *,
297	302	selection=PipelineSelection.NONE,
298		- remote=None):
	303	+ remote=None,
	304	+ pull_buildtrees=False):
299	305
300	306	use_config = True
301	307	if remote:
...	...	@@ -310,8 +316,12 @@ class Stream():
310	316	if not self._artifacts.has_fetch_remotes():
311	317	raise StreamError("No artifact caches available for pulling artifacts")
312	318
	319	+ # Query if pullbuildtrees has been set globally in user config
	320	+ if self._context.pullbuildtrees:
	321	+ pull_buildtrees = True
	322	+
313	323	self._pipeline.assert_consistent(elements)
314		- self._add_queue(PullQueue(self._scheduler))
	324	+ self._add_queue(PullQueue(self._scheduler, buildtrees=pull_buildtrees))
315	325	self._enqueue_plan(elements)
316	326	self._run()
317	327

buildstream/element.py

---

...	...	@@ -1692,18 +1692,26 @@ class Element(Plugin):
1692	1692
1693	1693	# _pull_pending()
1694	1694	#
1695		- # Check whether the artifact will be pulled.
	1695	+ # Check whether the artifact will be pulled. If the pull operation is to
	1696	+ # include a specific subdir of the element artifact (from cli or user conf)
	1697	+ # then the local cache is queried for the subdirs existence.
	1698	+ #
	1699	+ # Args:
	1700	+ # subdir (str): Whether the pull has been invoked with a specific subdir set
1696	1701	#
1697	1702	# Returns:
1698	1703	# (bool): Whether a pull operation is pending
1699	1704	#
1700		- def _pull_pending(self):
	1705	+ def _pull_pending(self, subdir=None):
1701	1706	if self._get_workspace():
1702	1707	# Workspace builds are never pushed to artifact servers
1703	1708	return False
1704	1709
1705		- if self.__strong_cached:
1706		- # Artifact already in local cache
	1710	+ if self.__strong_cached and subdir:
	1711	+ # If we've specified a subdir, check if the subdir is cached locally
	1712	+ if self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, subdir):
	1713	+ return False
	1714	+ elif self.__strong_cached:
1707	1715	return False
1708	1716
1709	1717	# Pull is pending if artifact remote server available
...	...	@@ -1725,11 +1733,10 @@ class Element(Plugin):
1725	1733
1726	1734	self._update_state()
1727	1735
1728		- def _pull_strong(self, *, progress=None):
	1736	+ def _pull_strong(self, *, progress=None, subdir=None, excluded_subdirs=None):
1729	1737	weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
1730		-
1731	1738	key = self.__strict_cache_key
1732		- if not self.__artifacts.pull(self, key, progress=progress):
	1739	+ if not self.__artifacts.pull(self, key, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
1733	1740	return False
1734	1741
1735	1742	# update weak ref by pointing it to this newly fetched artifact
...	...	@@ -1737,10 +1744,10 @@ class Element(Plugin):
1737	1744
1738	1745	return True
1739	1746
1740		- def _pull_weak(self, *, progress=None):
	1747	+ def _pull_weak(self, *, progress=None, subdir=None, excluded_subdirs=None):
1741	1748	weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
1742		-
1743		- if not self.__artifacts.pull(self, weak_key, progress=progress):
	1749	+ if not self.__artifacts.pull(self, weak_key, progress=progress, subdir=subdir,
	1750	+ excluded_subdirs=excluded_subdirs):
1744	1751	return False
1745	1752
1746	1753	# extract strong cache key from this newly fetched artifact
...	...	@@ -1758,17 +1765,17 @@ class Element(Plugin):
1758	1765	#
1759	1766	# Returns: True if the artifact has been downloaded, False otherwise
1760	1767	#
1761		- def _pull(self):
	1768	+ def _pull(self, subdir=None, excluded_subdirs=None):
1762	1769	context = self._get_context()
1763	1770
1764	1771	def progress(percent, message):
1765	1772	self.status(message)
1766	1773
1767	1774	# Attempt to pull artifact without knowing whether it's available
1768		- pulled = self._pull_strong(progress=progress)
	1775	+ pulled = self._pull_strong(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
1769	1776
1770	1777	if not pulled and not self._cached() and not context.get_strict():
1771		- pulled = self._pull_weak(progress=progress)
	1778	+ pulled = self._pull_weak(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
1772	1779
1773	1780	if not pulled:
1774	1781	return False
...	...	@@ -1791,10 +1798,21 @@ class Element(Plugin):
1791	1798	if not self._cached():
1792	1799	return True
1793	1800
1794		- # Do not push tained artifact
	1801	+ # Do not push tainted artifact
1795	1802	if self.__get_tainted():
1796	1803	return True
1797	1804
	1805	+ # Do not push elements that have a dangling buildtree artifact unless element type is
	1806	+ # expected to have an empty buildtree directory
	1807	+ if not self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, 'buildtree'):
	1808	+ return True
	1809	+
	1810	+ # strict_cache_key can't be relied on to be available when running in non strict mode
	1811	+ context = self._get_context()
	1812	+ if not context.get_strict():
	1813	+ if not self.__artifacts.contains_subdir_artifact(self, self.__weak_cache_key, 'buildtree'):
	1814	+ return True
	1815	+
1798	1816	return False
1799	1817
1800	1818	# _push():

tests/completions/completions.py

---

...	...	@@ -103,7 +103,7 @@ def test_commands(cli, cmd, word_idx, expected):
103	103	('bst --no-colors build -', 3, ['--all ', '--track ', '--track-all ',
104	104	'--track-except ',
105	105	'--track-cross-junctions ', '-J ',
106		- '--track-save ']),
	106	+ '--track-save ', '--pull-buildtrees ']),
107	107
108	108	# Test the behavior of completing after an option that has a
109	109	# parameter that cannot be completed, vs an option that has

tests/integration/pullbuildtrees.py

---

	1	+import os
	2	+import shutil
	3	+import pytest
	4	+
	5	+from tests.testutils import cli_integration as cli, create_artifact_share
	6	+from tests.testutils.integration import assert_contains
	7	+
	8	+
	9	+DATA_DIR = os.path.join(
	10	+ os.path.dirname(os.path.realpath(__file__)),
	11	+ "project"
	12	+)
	13	+
	14	+
	15	+# Remove artifact cache & set cli.config value of pullbuildtrees
	16	+# to false, which is the default user context
	17	+def default_state(cli, integration_cache, share):
	18	+ # cache_dir = os.path.join(integration_cache, 'artifacts')
	19	+ # Is this leaving behind the buildtree dir object?
	20	+ # Might need to nuke the whole /artifacts dir instead
	21	+ # cli.remove_artifact_from_cache(project, element_name, cache_dir=cache_dir)
	22	+ shutil.rmtree(os.path.join(integration_cache, 'artifacts'))
	23	+ cli.configure({'pullbuildtrees': False, 'artifacts': {'url': share.repo, 'push': False}})
	24	+
	25	+
	26	+# A test to capture the integration of the pullbuildtrees
	27	+# behaviour, which by default is to not include the buildtree
	28	+# directory of an element
	29	+@pytest.mark.integration
	30	+@pytest.mark.datafiles(DATA_DIR)
	31	+def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
	32	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	33	+ element_name = 'autotools/amhello.bst'
	34	+
	35	+ # Create artifact shares for pull & push testing
	36	+ with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
	37	+ create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2:
	38	+ cli.configure({
	39	+ 'artifacts': {'url': share1.repo, 'push': True},
	40	+ })
	41	+
	42	+ # Build autotools element, checked pushed, delete local
	43	+ result = cli.run(project=project, args=['build', element_name])
	44	+ assert result.exit_code == 0
	45	+ assert cli.get_element_state(project, element_name) == 'cached'
	46	+ assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	47	+ default_state(cli, integration_cache, share1)
	48	+
	49	+ # Pull artifact with default config, assert that pulling again
	50	+ # doesn't create a pull job, then assert with buildtrees user
	51	+ # config set creates a pull job.
	52	+ result = cli.run(project=project, args=['pull', element_name])
	53	+ assert element_name in result.get_pulled_elements()
	54	+ result = cli.run(project=project, args=['pull', element_name])
	55	+ assert element_name not in result.get_pulled_elements()
	56	+ cli.configure({'pullbuildtrees': True})
	57	+ result = cli.run(project=project, args=['pull', element_name])
	58	+ assert element_name in result.get_pulled_elements()
	59	+ default_state(cli, integration_cache, share1)
	60	+
	61	+ # Pull artifact with default config, then assert that pulling
	62	+ # with buildtrees cli flag set creates a pull job.
	63	+ result = cli.run(project=project, args=['pull', element_name])
	64	+ assert element_name in result.get_pulled_elements()
	65	+ result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
	66	+ assert element_name in result.get_pulled_elements()
	67	+ default_state(cli, integration_cache, share1)
	68	+
	69	+ # Pull artifact with pullbuildtrees set in user config, then assert
	70	+ # that pulling with the same user config doesn't creates a pull job,
	71	+ # or when buildtrees cli flag is set.
	72	+ cli.configure({'pullbuildtrees': True})
	73	+ result = cli.run(project=project, args=['pull', element_name])
	74	+ assert element_name in result.get_pulled_elements()
	75	+ result = cli.run(project=project, args=['pull', element_name])
	76	+ assert element_name not in result.get_pulled_elements()
	77	+ result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
	78	+ assert element_name not in result.get_pulled_elements()
	79	+ default_state(cli, integration_cache, share1)
	80	+
	81	+ # Pull artifact with default config and buildtrees cli flag set, then assert
	82	+ # that pulling with pullbuildtrees set in user config doesn't create a pull
	83	+ # job.
	84	+ result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
	85	+ assert element_name in result.get_pulled_elements()
	86	+ cli.configure({'pullbuildtrees': True})
	87	+ result = cli.run(project=project, args=['pull', element_name])
	88	+ assert element_name not in result.get_pulled_elements()
	89	+ default_state(cli, integration_cache, share1)
	90	+
	91	+ # Assert that a partial build element (not containing a populated buildtree dir)
	92	+ # can't be pushed to an artifact share, then assert that a complete build element
	93	+ # can be. This will attempt a partial pull from share1 and then a partial push
	94	+ # to share2
	95	+ result = cli.run(project=project, args=['pull', element_name])
	96	+ assert element_name in result.get_pulled_elements()
	97	+ cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
	98	+ result = cli.run(project=project, args=['push', element_name])
	99	+ assert element_name not in result.get_pushed_elements()
	100	+ assert not share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	101	+
	102	+ # Assert that after pulling the missing buildtree the element artifact can be
	103	+ # successfully pushed to the remote. This will attempt to pull the buildtree
	104	+ # from share1 and then a 'complete' push to share2
	105	+ cli.configure({'artifacts': {'url': share1.repo, 'push': False}})
	106	+ result = cli.run(project=project, args=['pull', '--pull-buildtrees', element_name])
	107	+ assert element_name in result.get_pulled_elements()
	108	+ cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
	109	+ result = cli.run(project=project, args=['push', element_name])
	110	+ assert element_name in result.get_pushed_elements()
	111	+ assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	112	+ default_state(cli, integration_cache, share1)

tests/testutils/artifactshare.py

---

...	...	@@ -128,7 +128,7 @@ class ArtifactShare():
128	128
129	129	valid_chars = string.digits + string.ascii_letters + '-._'
130	130	element_name = ''.join([
131		- x if x in valid_chars else '_'
	131	+ x if x in valid_chars else '-'
132	132	for x in element_name
133	133	])
134	134	artifact_key = '{0}/{1}/{2}'.format(project_name, element_name, cache_key)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.