[Notes] [Git][BuildStream/buildstream][tpollard/workspacebuildtree] WIP: Opening a workspace with a cached build

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/workspacebuildtree at BuildStream / buildstream

Commits:

• aa6acaab
  by Tom Pollard at 2018-11-07T12:21:26Z

  WIP: Opening a workspace with a cached build
  

9 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/element.py
• doc/source/developing/workspaces.rst
• tests/frontend/workspace.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -835,6 +835,20 @@ class ArtifactCache():
835	835
836	836	self.cas.link_ref(oldref, newref)
837	837
	838	+ # checkout_artifact_subdir()
	839	+ #
	840	+ # Checkout given artifact subdir into provided directory
	841	+ #
	842	+ # Args:
	843	+ # element (Element): The Element
	844	+ # key (str): The cache key to use
	845	+ # subdir (str): The subdir to checkout
	846	+ # tmpdir (str): The dir to place the subdir content
	847	+ #
	848	+ def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
	849	+ ref = self.get_artifact_fullname(element, key)
	850	+ return self.cas.checkout_artifact_subdir(ref, subdir, tmpdir)
	851	+
838	852	################################################
839	853	# Local Private Methods #
840	854	################################################

buildstream/_artifactcache/cascache.py

---

...	...	@@ -409,6 +409,21 @@ class CASCache():
409	409
410	410	return True
411	411
	412	+ # checkout_artifact_subdir():
	413	+ #
	414	+ # Checkout given artifact subdir into provided directory
	415	+ #
	416	+ # Args:
	417	+ # ref (str): The ref to check
	418	+ # subdir (str): The subdir to checkout
	419	+ # tmpdir (str): The dir to place the subdir content
	420	+ #
	421	+ def checkout_artifact_subdir(self, ref, subdir, tmpdir):
	422	+ tree = self.resolve_ref(ref)
	423	+ # This assumes that the subdir digest is present in the element tree
	424	+ subdirdigest = self._get_subdir(tree, subdir)
	425	+ self._checkout(tmpdir, subdirdigest)
	426	+
412	427	# objpath():
413	428	#
414	429	# Return the path of an object based on its digest.

buildstream/_context.py

---

...	...	@@ -107,6 +107,9 @@ class Context():
107	107	# Whether or not to attempt to pull build trees globally
108	108	self.pull_build_trees = False
109	109
	110	+ # Whether to not include artifact buildtrees in workspaces if available
	111	+ self.workspace_build_trees = True
	112	+
110	113	# Whether elements must be rebuilt when their dependencies have changed
111	114	self._strict_build_plan = None
112	115
...	...	@@ -163,7 +166,7 @@ class Context():
163	166	_yaml.node_validate(defaults, [
164	167	'sourcedir', 'builddir', 'artifactdir', 'logdir',
165	168	'scheduler', 'artifacts', 'logging', 'projects',
166		- 'cache', 'pullbuildtrees'
	169	+ 'cache', 'pullbuildtrees', 'workspacebuildtrees'
167	170	])
168	171
169	172	for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
...	...	@@ -191,6 +194,9 @@ class Context():
191	194	# Load pull build trees configuration
192	195	self.pull_build_trees = _yaml.node_get(defaults, bool, 'pullbuildtrees', default_value='False')
193	196
	197	+ # Load workspace buildtrees configuration
	198	+ self.workspace_build_trees = _yaml.node_get(defaults, bool, 'workspacebuildtrees', default_value='True')
	199	+
194	200	# Load logging config
195	201	logging = _yaml.node_get(defaults, Mapping, 'logging')
196	202	_yaml.node_validate(logging, [

buildstream/_frontend/cli.py

---

...	...	@@ -678,17 +678,21 @@ def workspace():
678	678	##################################################################
679	679	@workspace.command(name='open', short_help="Open a new workspace")
680	680	@click.option('--no-checkout', default=False, is_flag=True,
681		- help="Do not checkout the source, only link to the given directory")
	681	+ help="Do not checkout the source or cached buildtree, only link to the given directory")
682	682	@click.option('--force', '-f', default=False, is_flag=True,
683	683	help="Overwrite files existing in checkout directory")
684	684	@click.option('--track', 'track_', default=False, is_flag=True,
685	685	help="Track and fetch new source references before checking out the workspace")
	686	+@click.option('--no-cache', default=False, is_flag=True,
	687	+ help="Do not checkout the cached buildtree")
686	688	@click.argument('element',
687	689	type=click.Path(readable=False))
688	690	@click.argument('directory', type=click.Path(file_okay=False))
689	691	@click.pass_obj
690		-def workspace_open(app, no_checkout, force, track_, element, directory):
691		- """Open a workspace for manual source modification"""
	692	+def workspace_open(app, no_checkout, force, track_, no_cache, element, directory):
	693	+ """Open a workspace for manual source modification, the elements buildtree
	694	+ will be provided if available in the local artifact cache.
	695	+ """
692	696
693	697	if os.path.exists(directory):
694	698
...	...	@@ -700,11 +704,15 @@ def workspace_open(app, no_checkout, force, track_, element, directory):
700	704	click.echo("Checkout directory is not empty: {}".format(directory), err=True)
701	705	sys.exit(-1)
702	706
	707	+ if not no_cache and not no_checkout:
	708	+ click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
	709	+
703	710	with app.initialized():
704	711	app.stream.workspace_open(element, directory,
705	712	no_checkout=no_checkout,
706	713	track_first=track_,
707		- force=force)
	714	+ force=force,
	715	+ no_cache=no_cache)
708	716
709	717
710	718	##################################################################

buildstream/_stream.py

---

...	...	@@ -453,11 +453,17 @@ class Stream():
453	453	# no_checkout (bool): Whether to skip checking out the source
454	454	# track_first (bool): Whether to track and fetch first
455	455	# force (bool): Whether to ignore contents in an existing directory
	456	+ # no_cache (bool): Whether to not include the cached buildtree
456	457	#
457	458	def workspace_open(self, target, directory, *,
458	459	no_checkout,
459	460	track_first,
460		- force):
	461	+ force,
	462	+ no_cache):
	463	+
	464	+ # Override no_cache if the global user conf workspacebuildtrees is false
	465	+ if not self._context.workspace_build_trees:
	466	+ no_cache = True
461	467
462	468	if track_first:
463	469	track_targets = (target,)
...	...	@@ -470,6 +476,20 @@ class Stream():
470	476	target = elements[0]
471	477	directory = os.path.abspath(directory)
472	478
	479	+ # Check if given target has a buildtree artifact cached locally
	480	+ buildtree = None
	481	+ if target._cached():
	482	+ buildtree = self._artifacts.contains_subdir_artifact(target, target._get_cache_key(), 'buildtree')
	483	+
	484	+ # If we're running in the default state, make the user aware of buildtree usage
	485	+ if not no_cache and not no_checkout:
	486	+ if buildtree:
	487	+ self._message(MessageType.INFO, "{} buildtree artifact is available,"
	488	+ " workspace will be opened with it".format(target.name))
	489	+ else:
	490	+ self._message(MessageType.WARN, "{} buildtree artifact not available,"
	491	+ " workspace will be opened with source checkout".format(target.name))
	492	+
473	493	if not list(target.sources()):
474	494	build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
475	495	if not build_depends:
...	...	@@ -501,6 +521,7 @@ class Stream():
501	521	"fetch the latest version of the " +
502	522	"source.")
503	523
	524	+ # Presume workspace to be forced if previous StreamError not raised
504	525	if workspace:
505	526	workspaces.delete_workspace(target._get_full_name())
506	527	workspaces.save_config()
...	...	@@ -510,11 +531,16 @@ class Stream():
510	531	except OSError as e:
511	532	raise StreamError("Failed to create workspace directory: {}".format(e)) from e
512	533
513		- workspaces.create_workspace(target._get_full_name(), directory)
514		-
515		- if not no_checkout:
516		- with target.timed_activity("Staging sources to {}".format(directory)):
517		- target._open_workspace()
	534	+ # Handle opening workspace with buildtree included
	535	+ if (buildtree and not no_cache) and not no_checkout:
	536	+ workspaces.create_workspace(target._get_full_name(), directory, cached_build=buildtree)
	537	+ with target.timed_activity("Staging buildtree to {}".format(directory)):
	538	+ target._open_workspace(buildtree=buildtree)
	539	+ else:
	540	+ workspaces.create_workspace(target._get_full_name(), directory)
	541	+ if (not buildtree or no_cache) and not no_checkout:
	542	+ with target.timed_activity("Staging sources to {}".format(directory)):
	543	+ target._open_workspace()
518	544
519	545	workspaces.save_config()
520	546	self._message(MessageType.INFO, "Saved workspace configuration")
...	...	@@ -598,10 +624,24 @@ class Stream():
598	624	.format(workspace_path, e)) from e
599	625
600	626	workspaces.delete_workspace(element._get_full_name())
601		- workspaces.create_workspace(element._get_full_name(), workspace_path)
602	627
603		- with element.timed_activity("Staging sources to {}".format(workspace_path)):
604		- element._open_workspace()
	628	+ # Create the workspace, ensuring the original optional cached build state is preserved if
	629	+ # possible.
	630	+ buildtree = False
	631	+ if workspace.cached_build and element._cached():
	632	+ if self._artifacts.contains_subdir_artifact(element, element._get_cache_key(), 'buildtree'):
	633	+ buildtree = True
	634	+
	635	+ # Warn the user if the workspace cannot be opened with the original cached build state
	636	+ if workspace.cached_build and not buildtree:
	637	+ self._message(MessageType.WARN, "{} original buildtree artifact not available,"
	638	+ " workspace will be opened with source checkout".format(element.name))
	639	+
	640	+ workspaces.create_workspace(element._get_full_name(), workspace_path,
	641	+ cached_build=buildtree)
	642	+
	643	+ with element.timed_activity("Staging to {}".format(workspace_path)):
	644	+ element._open_workspace(buildtree=buildtree)
605	645
606	646	self._message(MessageType.INFO,
607	647	"Reset workspace for {} at: {}".format(element.name,

buildstream/_workspaces.py

---

...	...	@@ -24,7 +24,7 @@ from . import _yaml
24	24	from ._exceptions import LoadError, LoadErrorReason
25	25
26	26
27		-BST_WORKSPACE_FORMAT_VERSION = 3
	27	+BST_WORKSPACE_FORMAT_VERSION = 4
28	28
29	29
30	30	# Workspace()
...	...	@@ -43,9 +43,11 @@ BST_WORKSPACE_FORMAT_VERSION = 3
43	43	# running_files (dict): A dict mapping dependency elements to files
44	44	# changed between failed builds. Should be
45	45	# made obsolete with failed build artifacts.
	46	+# cached_build (bool): If the workspace is staging the cached build artifact
46	47	#
47	48	class Workspace():
48		- def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
	49	+ def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False,
	50	+ running_files=None, cached_build=False):
49	51	self.prepared = prepared
50	52	self.last_successful = last_successful
51	53	self._path = path
...	...	@@ -53,6 +55,7 @@ class Workspace():
53	55
54	56	self._toplevel_project = toplevel_project
55	57	self._key = None
	58	+ self.cached_build = cached_build
56	59
57	60	# to_dict()
58	61	#
...	...	@@ -65,7 +68,8 @@ class Workspace():
65	68	ret = {
66	69	'prepared': self.prepared,
67	70	'path': self._path,
68		- 'running_files': self.running_files
	71	+ 'running_files': self.running_files,
	72	+ 'cached_build': self.cached_build
69	73	}
70	74	if self.last_successful is not None:
71	75	ret["last_successful"] = self.last_successful
...	...	@@ -224,12 +228,13 @@ class Workspaces():
224	228	# Args:
225	229	# element_name (str) - The element name to create a workspace for
226	230	# path (str) - The path in which the workspace should be kept
	231	+ # cached_build (bool) - If the workspace is staging the cached build artifact
227	232	#
228		- def create_workspace(self, element_name, path):
	233	+ def create_workspace(self, element_name, path, cached_build=False):
229	234	if path.startswith(self._toplevel_project.directory):
230	235	path = os.path.relpath(path, self._toplevel_project.directory)
231	236
232		- self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
	237	+ self._workspaces[element_name] = Workspace(self._toplevel_project, path=path, cached_build=cached_build)
233	238
234	239	return self._workspaces[element_name]
235	240
...	...	@@ -396,6 +401,7 @@ class Workspaces():
396	401	'path': _yaml.node_get(node, str, 'path'),
397	402	'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
398	403	'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None),
	404	+ 'cached_build': _yaml.node_get(node, bool, 'cached_build', default_value=False)
399	405	}
400	406	return Workspace.from_dict(self._toplevel_project, dictionary)
401	407

buildstream/element.py

---

...	...	@@ -1903,7 +1903,10 @@ class Element(Plugin):
1903	1903	# This requires that a workspace already be created in
1904	1904	# the workspaces metadata first.
1905	1905	#
1906		- def _open_workspace(self):
	1906	+ # Args:
	1907	+ # buildtree (bool): Whether to open workspace with artifact buildtree
	1908	+ #
	1909	+ def _open_workspace(self, buildtree=False):
1907	1910	context = self._get_context()
1908	1911	workspace = self._get_workspace()
1909	1912	assert workspace is not None
...	...	@@ -1916,12 +1919,22 @@ class Element(Plugin):
1916	1919	# files in the target directory actually works without any
1917	1920	# additional support from Source implementations.
1918	1921	#
	1922	+
1919	1923	os.makedirs(context.builddir, exist_ok=True)
1920		- with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
1921		- .format(self.normal_name)) as temp:
	1924	+ with utils._tempdir(dir=context.builddir, prefix='workspace-source-{}'
	1925	+ .format(self.normal_name)) as temp,\
	1926	+ utils._tempdir(dir=context.builddir, prefix='workspace-buildtree-{}'
	1927	+ .format(self.normal_name)) as buildtreetemp:
	1928	+
1922	1929	for source in self.sources():
1923	1930	source._init_workspace(temp)
1924	1931
	1932	+ # Overwrite the source checkout with the cached buildtree
	1933	+ if buildtree:
	1934	+ self.__artifacts.checkout_artifact_subdir(self, self._get_cache_key(), 'buildtree', buildtreetemp)
	1935	+ if utils._call([utils.get_host_tool('cp'), '-pfr', "".join((buildtreetemp, '/.')), temp])[0] != 0:
	1936	+ raise ElementError("Failed to copy buildtree into workspace checkout at {}".format(buildtreetemp))
	1937	+
1925	1938	# Now hardlink the files into the workspace target.
1926	1939	utils.link_files(temp, workspace.get_absolute_path())
1927	1940

doc/source/developing/workspaces.rst

---

...	...	@@ -96,3 +96,17 @@ To discard the workspace completely we can do:
96	96	This will close the workspace and completely remove the workspace_hello
97	97	directory.
98	98
	99	+
	100	+Utilising cached buildtrees
	101	+---------------------------
	102	+When a BuildStream build element artifact is created and cached, a snapshot of the
	103	+build directory after the build commands have completed is included in the artifact.
	104	+This `build tree` can be considered an intermediary state of element, where the source
	105	+is present along with any output created during the build execution.
	106	+
	107	+By default when opening a workspace, bst will attempt to stage the build tree into
	108	+the workspace if it's available in the local cache. If the respective build tree is
	109	+not present in the cache (element not cached, partially cached or is a non build element)
	110	+then the source will be staged as is. The default behaviour to attempt to use the build tree
	111	+can be overriden with specific bst workspace open option of `--no-cache`, or via setting
	112	+user configuration option `workspacebuildtrees: False`
		\ No newline at end of file

tests/frontend/workspace.py

---

...	...	@@ -44,7 +44,7 @@ DATA_DIR = os.path.join(
44	44
45	45
46	46	def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
47		- project_path=None, element_attrs=None):
	47	+ project_path=None, element_attrs=None, no_cache=False):
48	48	if not workspace_dir:
49	49	workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
50	50	if not project_path:
...	...	@@ -88,6 +88,8 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir
88	88	args = ['workspace', 'open']
89	89	if track:
90	90	args.append('--track')
	91	+ if no_cache:
	92	+ args.append('--no-cache')
91	93	args.extend([element_name, workspace_dir])
92	94	result = cli.run(project=project_path, args=args)
93	95
...	...	@@ -101,7 +103,7 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir
101	103	filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
102	104	assert os.path.exists(filename)
103	105
104		- return (element_name, project_path, workspace_dir)
	106	+ return (element_name, project_path, workspace_dir, result)
105	107
106	108
107	109	@pytest.mark.datafiles(DATA_DIR)
...	...	@@ -112,7 +114,7 @@ def test_open(cli, tmpdir, datafiles, kind):
112	114
113	115	@pytest.mark.datafiles(DATA_DIR)
114	116	def test_open_bzr_customize(cli, tmpdir, datafiles):
115		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "bzr", False)
	117	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, "bzr", False)
116	118
117	119	# Check that the .bzr dir exists
118	120	bzrdir = os.path.join(workspace, ".bzr")
...	...	@@ -137,7 +139,7 @@ def test_open_track(cli, tmpdir, datafiles, kind):
137	139	@pytest.mark.datafiles(DATA_DIR)
138	140	@pytest.mark.parametrize("kind", repo_kinds)
139	141	def test_open_force(cli, tmpdir, datafiles, kind):
140		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
	142	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
141	143
142	144	# Close the workspace
143	145	result = cli.run(project=project, args=[
...	...	@@ -158,7 +160,7 @@ def test_open_force(cli, tmpdir, datafiles, kind):
158	160	@pytest.mark.datafiles(DATA_DIR)
159	161	@pytest.mark.parametrize("kind", repo_kinds)
160	162	def test_open_force_open(cli, tmpdir, datafiles, kind):
161		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
	163	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
162	164
163	165	# Assert the workspace dir exists
164	166	assert os.path.exists(workspace)
...	...	@@ -173,7 +175,7 @@ def test_open_force_open(cli, tmpdir, datafiles, kind):
173	175	@pytest.mark.datafiles(DATA_DIR)
174	176	@pytest.mark.parametrize("kind", repo_kinds)
175	177	def test_open_force_different_workspace(cli, tmpdir, datafiles, kind):
176		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False, "-alpha")
	178	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False, "-alpha")
177	179
178	180	# Assert the workspace dir exists
179	181	assert os.path.exists(workspace)
...	...	@@ -183,7 +185,7 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles, kind):
183	185
184	186	tmpdir = os.path.join(str(tmpdir), "-beta")
185	187	shutil.move(hello_path, hello1_path)
186		- element_name2, project2, workspace2 = open_workspace(cli, tmpdir, datafiles, kind, False, "-beta")
	188	+ element_name2, project2, workspace2, _ = open_workspace(cli, tmpdir, datafiles, kind, False, "-beta")
187	189
188	190	# Assert the workspace dir exists
189	191	assert os.path.exists(workspace2)
...	...	@@ -210,7 +212,7 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles, kind):
210	212	@pytest.mark.datafiles(DATA_DIR)
211	213	@pytest.mark.parametrize("kind", repo_kinds)
212	214	def test_close(cli, tmpdir, datafiles, kind):
213		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
	215	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
214	216
215	217	# Close the workspace
216	218	result = cli.run(project=project, args=[
...	...	@@ -226,7 +228,7 @@ def test_close(cli, tmpdir, datafiles, kind):
226	228	def test_close_external_after_move_project(cli, tmpdir, datafiles):
227	229	workspace_dir = os.path.join(str(tmpdir), "workspace")
228	230	project_path = os.path.join(str(tmpdir), 'initial_project')
229		- element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir, project_path)
	231	+ element_name, _, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir, project_path)
230	232	assert os.path.exists(workspace_dir)
231	233	moved_dir = os.path.join(str(tmpdir), 'external_project')
232	234	shutil.move(project_path, moved_dir)
...	...	@@ -246,8 +248,8 @@ def test_close_external_after_move_project(cli, tmpdir, datafiles):
246	248	def test_close_internal_after_move_project(cli, tmpdir, datafiles):
247	249	initial_dir = os.path.join(str(tmpdir), 'initial_project')
248	250	initial_workspace = os.path.join(initial_dir, 'workspace')
249		- element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False,
250		- workspace_dir=initial_workspace, project_path=initial_dir)
	251	+ element_name, _, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False,
	252	+ workspace_dir=initial_workspace, project_path=initial_dir)
251	253	moved_dir = os.path.join(str(tmpdir), 'internal_project')
252	254	shutil.move(initial_dir, moved_dir)
253	255	assert os.path.exists(moved_dir)
...	...	@@ -265,7 +267,7 @@ def test_close_internal_after_move_project(cli, tmpdir, datafiles):
265	267
266	268	@pytest.mark.datafiles(DATA_DIR)
267	269	def test_close_removed(cli, tmpdir, datafiles):
268		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
	270	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
269	271
270	272	# Remove it first, closing the workspace should work
271	273	shutil.rmtree(workspace)
...	...	@@ -282,7 +284,7 @@ def test_close_removed(cli, tmpdir, datafiles):
282	284
283	285	@pytest.mark.datafiles(DATA_DIR)
284	286	def test_close_nonexistant_element(cli, tmpdir, datafiles):
285		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
	287	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
286	288	element_path = os.path.join(datafiles.dirname, datafiles.basename, 'elements', element_name)
287	289
288	290	# First brutally remove the element.bst file, ensuring that
...	...	@@ -304,9 +306,9 @@ def test_close_nonexistant_element(cli, tmpdir, datafiles):
304	306	def test_close_multiple(cli, tmpdir, datafiles):
305	307	tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
306	308	tmpdir_beta = os.path.join(str(tmpdir), 'beta')
307		- alpha, project, workspace_alpha = open_workspace(
	309	+ alpha, project, workspace_alpha, _ = open_workspace(
308	310	cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha')
309		- beta, project, workspace_beta = open_workspace(
	311	+ beta, project, workspace_beta, _ = open_workspace(
310	312	cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta')
311	313
312	314	# Close the workspaces
...	...	@@ -324,9 +326,9 @@ def test_close_multiple(cli, tmpdir, datafiles):
324	326	def test_close_all(cli, tmpdir, datafiles):
325	327	tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
326	328	tmpdir_beta = os.path.join(str(tmpdir), 'beta')
327		- alpha, project, workspace_alpha = open_workspace(
	329	+ alpha, project, workspace_alpha, _ = open_workspace(
328	330	cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha')
329		- beta, project, workspace_beta = open_workspace(
	331	+ beta, project, workspace_beta, _ = open_workspace(
330	332	cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta')
331	333
332	334	# Close the workspaces
...	...	@@ -343,7 +345,7 @@ def test_close_all(cli, tmpdir, datafiles):
343	345	@pytest.mark.datafiles(DATA_DIR)
344	346	def test_reset(cli, tmpdir, datafiles):
345	347	# Open the workspace
346		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
	348	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
347	349
348	350	# Modify workspace
349	351	shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
...	...	@@ -366,9 +368,9 @@ def test_reset_multiple(cli, tmpdir, datafiles):
366	368	# Open the workspaces
367	369	tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
368	370	tmpdir_beta = os.path.join(str(tmpdir), 'beta')
369		- alpha, project, workspace_alpha = open_workspace(
	371	+ alpha, project, workspace_alpha, _ = open_workspace(
370	372	cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha')
371		- beta, project, workspace_beta = open_workspace(
	373	+ beta, project, workspace_beta, _ = open_workspace(
372	374	cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta')
373	375
374	376	# Modify workspaces
...	...	@@ -392,9 +394,9 @@ def test_reset_all(cli, tmpdir, datafiles):
392	394	# Open the workspaces
393	395	tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
394	396	tmpdir_beta = os.path.join(str(tmpdir), 'beta')
395		- alpha, project, workspace_alpha = open_workspace(
	397	+ alpha, project, workspace_alpha, _ = open_workspace(
396	398	cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha')
397		- beta, project, workspace_beta = open_workspace(
	399	+ beta, project, workspace_beta, _ = open_workspace(
398	400	cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta')
399	401
400	402	# Modify workspaces
...	...	@@ -415,7 +417,7 @@ def test_reset_all(cli, tmpdir, datafiles):
415	417
416	418	@pytest.mark.datafiles(DATA_DIR)
417	419	def test_list(cli, tmpdir, datafiles):
418		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
	420	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
419	421
420	422	# Now list the workspaces
421	423	result = cli.run(project=project, args=[
...	...	@@ -437,7 +439,7 @@ def test_list(cli, tmpdir, datafiles):
437	439	@pytest.mark.parametrize("kind", repo_kinds)
438	440	@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
439	441	def test_build(cli, tmpdir, datafiles, kind, strict):
440		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
	442	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
441	443	checkout = os.path.join(str(tmpdir), 'checkout')
442	444
443	445	# Modify workspace
...	...	@@ -516,7 +518,7 @@ def test_buildable_no_ref(cli, tmpdir, datafiles):
516	518	@pytest.mark.parametrize("modification", [("addfile"), ("removefile"), ("modifyfile")])
517	519	@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
518	520	def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
519		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
	521	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
520	522	checkout = os.path.join(str(tmpdir), 'checkout')
521	523
522	524	# Configure strict mode
...	...	@@ -637,7 +639,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
637	639	"alpha.bst": {
638	640	"prepared": False,
639	641	"path": "/workspaces/bravo",
640		- "running_files": {}
	642	+ "running_files": {},
	643	+ "cached_build": False
641	644	}
642	645	}
643	646	}),
...	...	@@ -652,7 +655,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
652	655	"alpha.bst": {
653	656	"prepared": False,
654	657	"path": "/workspaces/bravo",
655		- "running_files": {}
	658	+ "running_files": {},
	659	+ "cached_build": False
656	660	}
657	661	}
658	662	}),
...	...	@@ -670,7 +674,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
670	674	"alpha.bst": {
671	675	"prepared": False,
672	676	"path": "/workspaces/bravo",
673		- "running_files": {}
	677	+ "running_files": {},
	678	+ "cached_build": False
674	679	}
675	680	}
676	681	}),
...	...	@@ -695,7 +700,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
695	700	"last_successful": "some_key",
696	701	"running_files": {
697	702	"beta.bst": ["some_file"]
698		- }
	703	+ },
	704	+ "cached_build": False
699	705	}
700	706	}
701	707	}),
...	...	@@ -715,7 +721,30 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
715	721	"alpha.bst": {
716	722	"prepared": True,
717	723	"path": "/workspaces/bravo",
718		- "running_files": {}
	724	+ "running_files": {},
	725	+ "cached_build": False
	726	+ }
	727	+ }
	728	+ }),
	729	+ # Test loading version 4
	730	+ ({
	731	+ "format-version": 4,
	732	+ "workspaces": {
	733	+ "alpha.bst": {
	734	+ "prepared": False,
	735	+ "path": "/workspaces/bravo",
	736	+ "running_files": {},
	737	+ "cached_build": True
	738	+ }
	739	+ }
	740	+ }, {
	741	+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
	742	+ "workspaces": {
	743	+ "alpha.bst": {
	744	+ "prepared": False,
	745	+ "path": "/workspaces/bravo",
	746	+ "running_files": {},
	747	+ "cached_build": True
719	748	}
720	749	}
721	750	})
...	...	@@ -779,7 +808,7 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte
779	808	@pytest.mark.datafiles(DATA_DIR)
780	809	@pytest.mark.parametrize("kind", repo_kinds)
781	810	def test_inconsitent_pipeline_message(cli, tmpdir, datafiles, kind):
782		- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
	811	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
783	812
784	813	shutil.rmtree(workspace)
785	814
...	...	@@ -793,8 +822,8 @@ def test_inconsitent_pipeline_message(cli, tmpdir, datafiles, kind):
793	822	@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
794	823	def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict):
795	824	checkout = os.path.join(str(tmpdir), 'checkout')
796		- element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'),
797		- datafiles, 'git', False)
	825	+ element_name, project, workspace, _ = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'),
	826	+ datafiles, 'git', False)
798	827
799	828	element_path = os.path.join(project, 'elements')
800	829	back_dep_element_name = 'workspace-test-back-dep.bst'
...	...	@@ -869,10 +898,75 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
869	898	]
870	899	}
871	900	}
872		- element_name, project, _ = open_workspace(cli, tmpdir, datafiles,
873		- "git", False, element_attrs=element_config)
	901	+ element_name, project, _, _ = open_workspace(cli, tmpdir, datafiles,
	902	+ "git", False, element_attrs=element_config)
874	903
875	904	for _ in range(2):
876	905	result = cli.run(project=project, args=["build", element_name])
877	906	assert "BUG" not in result.stderr
878	907	assert cli.get_element_state(project, element_name) != "cached"
	908	+
	909	+
	910	+@pytest.mark.datafiles(DATA_DIR)
	911	+def test_nocache_open_messages(cli, tmpdir, datafiles):
	912	+ # cli default WARN for source dropback possibility when no-cache flag is not passed
	913	+ element_name, project, workspace, result = open_workspace(cli, tmpdir, datafiles, 'git', False, suffix='1')
	914	+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" in result.output
	915	+
	916	+ # cli WARN for source dropback happening when no-cache flag not given, but buildtree not available
	917	+ assert "workspace will be opened with source checkout" in result.stderr
	918	+
	919	+ # cli default WARN for source dropback possibilty not given when no-cache flag is passed
	920	+ tmpdir = os.path.join(str(tmpdir), "2")
	921	+ element_name, project, workspace, result = open_workspace(cli, tmpdir, datafiles, 'git', False, suffix='2',
	922	+ no_cache=True)
	923	+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" not in result.output
	924	+
	925	+
	926	+@pytest.mark.datafiles(DATA_DIR)
	927	+def test_nocache_reset_messages(cli, tmpdir, datafiles):
	928	+ element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, suffix='1')
	929	+
	930	+ # Modify workspace, without building so the artifact is not cached
	931	+ shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
	932	+ os.makedirs(os.path.join(workspace, 'etc'))
	933	+ with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
	934	+ f.write("PONY='pink'")
	935	+
	936	+ # Now reset the open workspace, this should have the
	937	+ # effect of reverting our changes to the original source, as it
	938	+ # was not originally opened with a cached buildtree and as such
	939	+ # should not notify the user
	940	+ result = cli.run(project=project, args=[
	941	+ 'workspace', 'reset', element_name
	942	+ ])
	943	+ result.assert_success()
	944	+ assert "original buildtree artifact not available" not in result.output
	945	+ assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello'))
	946	+ assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
	947	+
	948	+ # Close the workspace
	949	+ result = cli.run(project=project, args=[
	950	+ 'workspace', 'close', '--remove-dir', element_name
	951	+ ])
	952	+ result.assert_success()
	953	+
	954	+ # Build the workspace so we have a cached buildtree artifact for the element
	955	+ assert cli.get_element_state(project, element_name) == 'buildable'
	956	+ result = cli.run(project=project, args=['build', element_name])
	957	+ result.assert_success()
	958	+
	959	+ # Opening the workspace after a build should lead to the cached buildtree being
	960	+ # staged by default
	961	+ result = cli.run(project=project, args=[
	962	+ 'workspace', 'open', element_name, workspace
	963	+ ])
	964	+ result.assert_success()
	965	+
	966	+ # Now reset the workspace and ensure that a warning is not given about the artifact
	967	+ # buildtree not being available
	968	+ result = cli.run(project=project, args=[
	969	+ 'workspace', 'reset', element_name
	970	+ ])
	971	+ result.assert_success()
	972	+ assert "original buildtree artifact not available" not in result.output

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.