[Notes] [Git][BuildStream/buildstream][tpollard/workspacebuildtree] WIP: Opening a workspace with a cached build

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/workspacebuildtree at BuildStream / buildstream

Commits:

• 9a702ef3
  by Tom Pollard at 2018-11-23T18:49:42Z

  WIP: Opening a workspace with a cached build
  

12 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/element.py
• buildstream/plugins/sources/git.py
• buildstream/source.py
• doc/source/developing/workspaces.rst
• tests/frontend/workspace.py
• tests/integration/workspace.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -835,6 +835,20 @@ class ArtifactCache():
835	835
836	836	self.cas.link_ref(oldref, newref)
837	837
	838	+ # checkout_artifact_subdir()
	839	+ #
	840	+ # Checkout given artifact subdir into provided directory
	841	+ #
	842	+ # Args:
	843	+ # element (Element): The Element
	844	+ # key (str): The cache key to use
	845	+ # subdir (str): The subdir to checkout
	846	+ # tmpdir (str): The dir to place the subdir content
	847	+ #
	848	+ def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
	849	+ ref = self.get_artifact_fullname(element, key)
	850	+ return self.cas.checkout_artifact_subdir(ref, subdir, tmpdir)
	851	+
838	852	################################################
839	853	# Local Private Methods #
840	854	################################################

buildstream/_artifactcache/cascache.py

---

...	...	@@ -404,6 +404,21 @@ class CASCache():
404	404
405	405	return True
406	406
	407	+ # checkout_artifact_subdir():
	408	+ #
	409	+ # Checkout given artifact subdir into provided directory
	410	+ #
	411	+ # Args:
	412	+ # ref (str): The ref to check
	413	+ # subdir (str): The subdir to checkout
	414	+ # tmpdir (str): The dir to place the subdir content
	415	+ #
	416	+ def checkout_artifact_subdir(self, ref, subdir, tmpdir):
	417	+ tree = self.resolve_ref(ref)
	418	+ # This assumes that the subdir digest is present in the element tree
	419	+ subdirdigest = self._get_subdir(tree, subdir)
	420	+ self._checkout(tmpdir, subdirdigest)
	421	+
407	422	# objpath():
408	423	#
409	424	# Return the path of an object based on its digest.

buildstream/_context.py

---

...	...	@@ -125,6 +125,9 @@ class Context():
125	125	# a hard reset of a workspace, potentially losing changes.
126	126	self.prompt_workspace_reset_hard = None
127	127
	128	+ # Whether to not include artifact buildtrees in workspaces if available
	129	+ self.workspace_buildtrees = True
	130	+
128	131	# Whether elements must be rebuilt when their dependencies have changed
129	132	self._strict_build_plan = None
130	133
...	...	@@ -180,7 +183,7 @@ class Context():
180	183	_yaml.node_validate(defaults, [
181	184	'sourcedir', 'builddir', 'artifactdir', 'logdir',
182	185	'scheduler', 'artifacts', 'logging', 'projects',
183		- 'cache', 'prompt', 'workspacedir',
	186	+ 'cache', 'prompt', 'workspacedir', 'workspace-buildtrees'
184	187	])
185	188
186	189	for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
...	...	@@ -208,6 +211,9 @@ class Context():
208	211	# Load pull build trees configuration
209	212	self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
210	213
	214	+ # Load workspace buildtrees configuration
	215	+ self.workspace_buildtrees = _yaml.node_get(defaults, bool, 'workspace-buildtrees', default_value='True')
	216	+
211	217	# Load logging config
212	218	logging = _yaml.node_get(defaults, Mapping, 'logging')
213	219	_yaml.node_validate(logging, [

buildstream/_frontend/cli.py

---

...	...	@@ -705,7 +705,7 @@ def workspace():
705	705	##################################################################
706	706	@workspace.command(name='open', short_help="Open a new workspace")
707	707	@click.option('--no-checkout', default=False, is_flag=True,
708		- help="Do not checkout the source, only link to the given directory")
	708	+ help="Do not checkout the source or cached buildtree, only link to the given directory")
709	709	@click.option('--force', '-f', default=False, is_flag=True,
710	710	help="The workspace will be created even if the directory in which it will be created is not empty " +
711	711	"or if a workspace for that element already exists")
...	...	@@ -714,16 +714,25 @@ def workspace():
714	714	@click.option('--directory', type=click.Path(file_okay=False), default=None,
715	715	help="Only for use when a single Element is given: Set the directory to use to create the workspace")
716	716	@click.argument('elements', nargs=-1, type=click.Path(readable=False), required=True)
	717	+@click.option('--no-cache', default=False, is_flag=True,
	718	+ help="Do not checkout the cached buildtree")
717	719	@click.pass_obj
718		-def workspace_open(app, no_checkout, force, track_, directory, elements):
719		- """Open a workspace for manual source modification"""
	720	+def workspace_open(app, no_checkout, force, track_, directory, elements, no_cache):
	721	+
	722	+ """Open a workspace for manual source modification, the elements buildtree
	723	+ will be provided if available in the local artifact cache.
	724	+ """
	725	+
	726	+ if not no_cache and not no_checkout:
	727	+ click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
720	728
721	729	with app.initialized():
722	730	app.stream.workspace_open(elements,
723	731	no_checkout=no_checkout,
724	732	track_first=track_,
725	733	force=force,
726		- custom_dir=directory)
	734	+ custom_dir=directory,
	735	+ no_cache=no_cache)
727	736
728	737
729	738	##################################################################

buildstream/_stream.py

---

...	...	@@ -469,14 +469,21 @@ class Stream():
469	469	# track_first (bool): Whether to track and fetch first
470	470	# force (bool): Whether to ignore contents in an existing directory
471	471	# custom_dir (str): Custom location to create a workspace or false to use default location.
	472	+ # no_cache (bool): Whether to not include the cached buildtree
472	473	#
473	474	def workspace_open(self, targets, *,
474	475	no_checkout,
475	476	track_first,
476	477	force,
477		- custom_dir):
	478	+ custom_dir,
	479	+ no_cache):
	480	+
478	481	# This function is a little funny but it is trying to be as atomic as possible.
479	482
	483	+ # Set no_cache if the global user conf workspacebuildtrees is false
	484	+ if not self._context.workspace_buildtrees:
	485	+ no_cache = True
	486	+
480	487	if track_first:
481	488	track_targets = targets
482	489	else:
...	...	@@ -533,7 +540,7 @@ class Stream():
533	540	reason='directory-with-multiple-elements')
534	541	expanded_directories = [custom_dir, ]
535	542	else:
536		- # If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the
	543	+ # If this fails it is a bug in whatever calls this, usually cli.py and so can not be tested for via the
537	544	# run bst test mechanism.
538	545	assert len(elements) == len(expanded_directories)
539	546
...	...	@@ -548,12 +555,26 @@ class Stream():
548	555	.format(target.name, directory), reason='bad-directory')
549	556
550	557	# So far this function has tried to catch as many issues as possible with out making any changes
551		- # Now it dose the bits that can not be made atomic.
	558	+ # Now it does the bits that can not be made atomic.
552	559	targetGenerator = zip(elements, expanded_directories)
553	560	for target, directory in targetGenerator:
554	561	self._message(MessageType.INFO, "Creating workspace for element {}"
555	562	.format(target.name))
556	563
	564	+ # Check if given target has a buildtree artifact cached locally
	565	+ buildtree = None
	566	+ if target._cached():
	567	+ buildtree = self._artifacts.contains_subdir_artifact(target, target._get_cache_key(), 'buildtree')
	568	+
	569	+ # If we're running in the default state, make the user aware of buildtree usage
	570	+ if not no_cache and not no_checkout:
	571	+ if buildtree:
	572	+ self._message(MessageType.INFO, "{} buildtree artifact is available,"
	573	+ " workspace will be opened with it".format(target.name))
	574	+ else:
	575	+ self._message(MessageType.WARN, "{} buildtree artifact not available,"
	576	+ " workspace will be opened with source checkout".format(target.name))
	577	+
557	578	workspace = workspaces.get_workspace(target._get_full_name())
558	579	if workspace:
559	580	workspaces.delete_workspace(target._get_full_name())
...	...	@@ -568,11 +589,16 @@ class Stream():
568	589	todo_elements = "\nDid not try to create workspaces for " + todo_elements
569	590	raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
570	591
571		- workspaces.create_workspace(target._get_full_name(), directory)
572		-
573		- if not no_checkout:
574		- with target.timed_activity("Staging sources to {}".format(directory)):
575		- target._open_workspace()
	592	+ # Handle opening workspace with buildtree included
	593	+ if (buildtree and not no_cache) and not no_checkout:
	594	+ workspaces.create_workspace(target._get_full_name(), directory, cached_build=buildtree)
	595	+ with target.timed_activity("Staging buildtree to {}".format(directory)):
	596	+ target._open_workspace(buildtree=buildtree)
	597	+ else:
	598	+ workspaces.create_workspace(target._get_full_name(), directory)
	599	+ if (not buildtree or no_cache) and not no_checkout:
	600	+ with target.timed_activity("Staging sources to {}".format(directory)):
	601	+ target._open_workspace()
576	602
577	603	# Saving the workspace once it is set up means that if the next workspace fails to be created before
578	604	# the configuration gets saved. The successfully created workspace still gets saved.
...	...	@@ -659,10 +685,24 @@ class Stream():
659	685	.format(workspace_path, e)) from e
660	686
661	687	workspaces.delete_workspace(element._get_full_name())
662		- workspaces.create_workspace(element._get_full_name(), workspace_path)
663	688
664		- with element.timed_activity("Staging sources to {}".format(workspace_path)):
665		- element._open_workspace()
	689	+ # Create the workspace, ensuring the original optional cached build state is preserved if
	690	+ # possible.
	691	+ buildtree = False
	692	+ if workspace.cached_build and element._cached():
	693	+ if self._artifacts.contains_subdir_artifact(element, element._get_cache_key(), 'buildtree'):
	694	+ buildtree = True
	695	+
	696	+ # Warn the user if the workspace cannot be opened with the original cached build state
	697	+ if workspace.cached_build and not buildtree:
	698	+ self._message(MessageType.WARN, "{} original buildtree artifact not available,"
	699	+ " workspace will be opened with source checkout".format(element.name))
	700	+
	701	+ workspaces.create_workspace(element._get_full_name(), workspace_path,
	702	+ cached_build=buildtree)
	703	+
	704	+ with element.timed_activity("Staging to {}".format(workspace_path)):
	705	+ element._open_workspace(buildtree=buildtree)
666	706
667	707	self._message(MessageType.INFO,
668	708	"Reset workspace for {} at: {}".format(element.name,

buildstream/_workspaces.py

---

...	...	@@ -24,7 +24,7 @@ from . import _yaml
24	24	from ._exceptions import LoadError, LoadErrorReason
25	25
26	26
27		-BST_WORKSPACE_FORMAT_VERSION = 3
	27	+BST_WORKSPACE_FORMAT_VERSION = 4
28	28
29	29
30	30	# Workspace()
...	...	@@ -43,9 +43,11 @@ BST_WORKSPACE_FORMAT_VERSION = 3
43	43	# running_files (dict): A dict mapping dependency elements to files
44	44	# changed between failed builds. Should be
45	45	# made obsolete with failed build artifacts.
	46	+# cached_build (bool): If the workspace is staging the cached build artifact
46	47	#
47	48	class Workspace():
48		- def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
	49	+ def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False,
	50	+ running_files=None, cached_build=False):
49	51	self.prepared = prepared
50	52	self.last_successful = last_successful
51	53	self._path = path
...	...	@@ -53,6 +55,7 @@ class Workspace():
53	55
54	56	self._toplevel_project = toplevel_project
55	57	self._key = None
	58	+ self.cached_build = cached_build
56	59
57	60	# to_dict()
58	61	#
...	...	@@ -65,7 +68,8 @@ class Workspace():
65	68	ret = {
66	69	'prepared': self.prepared,
67	70	'path': self._path,
68		- 'running_files': self.running_files
	71	+ 'running_files': self.running_files,
	72	+ 'cached_build': self.cached_build
69	73	}
70	74	if self.last_successful is not None:
71	75	ret["last_successful"] = self.last_successful
...	...	@@ -224,12 +228,13 @@ class Workspaces():
224	228	# Args:
225	229	# element_name (str) - The element name to create a workspace for
226	230	# path (str) - The path in which the workspace should be kept
	231	+ # cached_build (bool) - If the workspace is staging the cached build artifact
227	232	#
228		- def create_workspace(self, element_name, path):
	233	+ def create_workspace(self, element_name, path, cached_build=False):
229	234	if path.startswith(self._toplevel_project.directory):
230	235	path = os.path.relpath(path, self._toplevel_project.directory)
231	236
232		- self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
	237	+ self._workspaces[element_name] = Workspace(self._toplevel_project, path=path, cached_build=cached_build)
233	238
234	239	return self._workspaces[element_name]
235	240
...	...	@@ -396,6 +401,7 @@ class Workspaces():
396	401	'path': _yaml.node_get(node, str, 'path'),
397	402	'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
398	403	'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None),
	404	+ 'cached_build': _yaml.node_get(node, bool, 'cached_build', default_value=False)
399	405	}
400	406	return Workspace.from_dict(self._toplevel_project, dictionary)
401	407

buildstream/element.py

---

...	...	@@ -1880,7 +1880,10 @@ class Element(Plugin):
1880	1880	# This requires that a workspace already be created in
1881	1881	# the workspaces metadata first.
1882	1882	#
1883		- def _open_workspace(self):
	1883	+ # Args:
	1884	+ # buildtree (bool): Whether to open workspace with artifact buildtree
	1885	+ #
	1886	+ def _open_workspace(self, buildtree=False):
1884	1887	context = self._get_context()
1885	1888	workspace = self._get_workspace()
1886	1889	assert workspace is not None
...	...	@@ -1893,11 +1896,19 @@ class Element(Plugin):
1893	1896	# files in the target directory actually works without any
1894	1897	# additional support from Source implementations.
1895	1898	#
	1899	+
1896	1900	os.makedirs(context.builddir, exist_ok=True)
1897	1901	with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
1898	1902	.format(self.normal_name)) as temp:
1899		- for source in self.sources():
1900		- source._init_workspace(temp)
	1903	+
	1904	+ # Checkout cached buildtree, augment with source plugin if applicable
	1905	+ if buildtree:
	1906	+ self.__artifacts.checkout_artifact_subdir(self, self._get_cache_key(), 'buildtree', temp)
	1907	+ for source in self.sources():
	1908	+ source._init_cached_build_workspace(temp)
	1909	+ else:
	1910	+ for source in self.sources():
	1911	+ source._init_workspace(temp)
1901	1912
1902	1913	# Now hardlink the files into the workspace target.
1903	1914	utils.link_files(temp, workspace.get_absolute_path())

buildstream/plugins/sources/git.py

---

...	...	@@ -268,6 +268,35 @@ class GitMirror(SourceFetcher):
268	268	if track:
269	269	self.assert_ref_in_track(fullpath, track)
270	270
	271	+ def init_cached_build_workspace(self, directory, track=None):
	272	+ fullpath = os.path.join(directory, self.path)
	273	+ url = self.source.translate_url(self.url)
	274	+
	275	+ self.source.call([self.source.host_git, 'init', fullpath],
	276	+ fail="Failed to init git in directory: {}".format(fullpath),
	277	+ fail_temporarily=True,
	278	+ cwd=fullpath)
	279	+
	280	+ self.source.call([self.source.host_git, 'fetch', self.mirror],
	281	+ fail='Failed to fetch from local mirror "{}"'.format(self.mirror),
	282	+ cwd=fullpath)
	283	+
	284	+ self.source.call([self.source.host_git, 'remote', 'add', 'origin', url],
	285	+ fail='Failed to add remote origin "{}"'.format(url),
	286	+ cwd=fullpath)
	287	+
	288	+ self.source.call([self.source.host_git, 'update-ref', '--no-deref', 'HEAD', self.ref],
	289	+ fail='Failed update HEAD to ref "{}"'.format(self.ref),
	290	+ cwd=fullpath)
	291	+
	292	+ self.source.call([self.source.host_git, 'read-tree', 'HEAD'],
	293	+ fail='Failed to read HEAD into index',
	294	+ cwd=fullpath)
	295	+
	296	+ # Check that the user specified ref exists in the track if provided & not already tracked
	297	+ if track:
	298	+ self.assert_ref_in_track(fullpath, track)
	299	+
271	300	# List the submodules (path/url tuples) present at the given ref of this repo
272	301	def submodule_list(self):
273	302	modules = "{}:{}".format(self.ref, GIT_MODULES)
...	...	@@ -480,6 +509,14 @@ class GitSource(Source):
480	509	for mirror in self.submodules:
481	510	mirror.init_workspace(directory)
482	511
	512	+ def init_cached_build_workspace(self, directory):
	513	+ self.refresh_submodules()
	514	+
	515	+ with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
	516	+ self.mirror.init_cached_build_workspace(directory, track=(self.tracking if not self.tracked else None))
	517	+ for mirror in self.submodules:
	518	+ mirror.init_cached_build_workspace(directory)
	519	+
483	520	def stage(self, directory):
484	521
485	522	# Need to refresh submodule list here again, because

buildstream/source.py

---

...	...	@@ -459,6 +459,24 @@ class Source(Plugin):
459	459	"""
460	460	self.stage(directory)
461	461
	462	+ def init_cached_build_workspace(self, directory):
	463	+ """Initialises a new cached build workspace
	464	+
	465	+ Args:
	466	+ directory (str): Path of the workspace to init
	467	+
	468	+ Raises:
	469	+ :class:`.SourceError`
	470	+
	471	+ Implementors overriding this method should assume that *directory*
	472	+ already exists.
	473	+
	474	+ Implementors should raise :class:`.SourceError` when encountering
	475	+ some system error.
	476	+ """
	477	+ # Allow a non implementation
	478	+ return None
	479	+
462	480	def get_source_fetchers(self):
463	481	"""Get the objects that are used for fetching
464	482
...	...	@@ -677,6 +695,12 @@ class Source(Plugin):
677	695
678	696	self.init_workspace(directory)
679	697
	698	+ # Wrapper for init_cached_build_workspace()
	699	+ def _init_cached_build_workspace(self, directory):
	700	+ directory = self.__ensure_directory(directory)
	701	+
	702	+ self.init_cached_build_workspace(directory)
	703	+
680	704	# _get_unique_key():
681	705	#
682	706	# Wrapper for get_unique_key() api

doc/source/developing/workspaces.rst

---

...	...	@@ -24,9 +24,32 @@ Suppose we now want to alter the functionality of the *hello* command. We can
24	24	make changes to the source code of Buildstream elements by making use of
25	25	BuildStream's workspace command.
26	26
	27	+Utilising cached buildtrees
	28	+---------------------------
	29	+ When a BuildStream build element artifact is created and cached, a snapshot of
	30	+ the build directory after the build commands have completed is included in the
	31	+ artifact. This `build tree` can be considered an intermediary state of element,
	32	+ where the source is present along with any output created during the build
	33	+ execution.
	34	+
	35	+ By default when opening a workspace, bst will attempt to stage the build tree
	36	+ into the workspace if it's available in the local cache. If the respective
	37	+ build tree is not present in the cache (element not cached, partially cached or
	38	+ is a non build element) then the source will be staged as is. The default
	39	+ behaviour to attempt to use the build tree can be overriden with specific bst
	40	+ workspace open option of `--no-cache`, or via setting user configuration option
	41	+ `workspacebuildtrees: False`
	42	+
27	43
28	44	Opening a workspace
29	45	-------------------
	46	+.. note::
	47	+
	48	+ This example presumes you built the hello.bst during
	49	+ :ref:`running commands <tutorial_running_commands>`
	50	+ if not, please start by building it.
	51	+
	52	+
30	53	First we need to open a workspace, we can do this by running
31	54
32	55	.. raw:: html
...	...	@@ -88,6 +111,15 @@ Alternatively, if we wish to discard the changes we can use
88	111
89	112	This resets the workspace to its original state.
90	113
	114	+.. note::
	115	+
	116	+ bst reset will attempt to open the workspace in
	117	+ the condition in which it was originally staged,
	118	+ i.e with or without consuming the element build tree.
	119	+ If it was originally staged with a cached build tree
	120	+ and there's no longer one available, the source will
	121	+ be staged as is.
	122	+
91	123	To discard the workspace completely we can do:
92	124
93	125	.. raw:: html

tests/frontend/workspace.py

---

...	...	@@ -91,18 +91,18 @@ class WorkspaceCreater():
91	91	element_name))
92	92	return element_name, element_path, workspace_dir
93	93
94		- def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None,
	94	+ def create_workspace_elements(self, kinds, track, suffixes=None, workspace_dir_usr=None,
95	95	element_attrs=None):
96	96
97	97	element_tuples = []
98	98
99		- if suffixs is None:
100		- suffixs = ['', ] * len(kinds)
	99	+ if suffixes is None:
	100	+ suffixes = ['', ] * len(kinds)
101	101	else:
102		- if len(suffixs) != len(kinds):
	102	+ if len(suffixes) != len(kinds):
103	103	raise "terable error"
104	104
105		- for suffix, kind in zip(suffixs, kinds):
	105	+ for suffix, kind in zip(suffixes, kinds):
106	106	element_name, element_path, workspace_dir = \
107	107	self.create_workspace_element(kind, track, suffix, workspace_dir_usr,
108	108	element_attrs)
...	...	@@ -117,10 +117,10 @@ class WorkspaceCreater():
117	117
118	118	return element_tuples
119	119
120		- def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None,
121		- element_attrs=None):
	120	+ def open_workspaces(self, kinds, track, suffixes=None, workspace_dir=None,
	121	+ element_attrs=None, no_cache=False):
122	122
123		- element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir,
	123	+ element_tuples = self.create_workspace_elements(kinds, track, suffixes, workspace_dir,
124	124	element_attrs)
125	125	os.makedirs(self.workspace_cmd, exist_ok=True)
126	126
...	...	@@ -129,12 +129,15 @@ class WorkspaceCreater():
129	129	args = ['workspace', 'open']
130	130	if track:
131	131	args.append('--track')
	132	+ if no_cache:
	133	+ args.append('--no-cache')
132	134	if workspace_dir is not None:
133	135	assert len(element_tuples) == 1, "test logic error"
134	136	_, workspace_dir = element_tuples[0]
135	137	args.extend(['--directory', workspace_dir])
136		-
	138	+ print("element_tuples", element_tuples)
137	139	args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
	140	+ print("args", args)
138	141	result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
139	142
140	143	result.assert_success()
...	...	@@ -148,14 +151,14 @@ class WorkspaceCreater():
148	151	filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
149	152	assert os.path.exists(filename)
150	153
151		- return element_tuples
	154	+ return element_tuples, result
152	155
153	156
154	157	def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
155		- project_path=None, element_attrs=None):
	158	+ project_path=None, element_attrs=None, no_cache=False):
156	159	workspace_object = WorkspaceCreater(cli, tmpdir, datafiles, project_path)
157		- workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
158		- element_attrs)
	160	+ workspaces, _ = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
	161	+ element_attrs, no_cache)
159	162	assert len(workspaces) == 1
160	163	element_name, workspace = workspaces[0]
161	164	return element_name, workspace_object.project_path, workspace
...	...	@@ -189,7 +192,7 @@ def test_open_bzr_customize(cli, tmpdir, datafiles):
189	192	def test_open_multi(cli, tmpdir, datafiles):
190	193
191	194	workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
192		- workspaces = workspace_object.open_workspaces(repo_kinds, False)
	195	+ workspaces, _ = workspace_object.open_workspaces(repo_kinds, False)
193	196
194	197	for (elname, workspace), kind in zip(workspaces, repo_kinds):
195	198	assert kind in elname
...	...	@@ -816,7 +819,9 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
816	819	"alpha.bst": {
817	820	"prepared": False,
818	821	"path": "/workspaces/bravo",
819		- "running_files": {}
	822	+ "running_files": {},
	823	+ "cached_build": False
	824	+
820	825	}
821	826	}
822	827	}),
...	...	@@ -831,7 +836,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
831	836	"alpha.bst": {
832	837	"prepared": False,
833	838	"path": "/workspaces/bravo",
834		- "running_files": {}
	839	+ "running_files": {},
	840	+ "cached_build": False
835	841	}
836	842	}
837	843	}),
...	...	@@ -849,7 +855,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
849	855	"alpha.bst": {
850	856	"prepared": False,
851	857	"path": "/workspaces/bravo",
852		- "running_files": {}
	858	+ "running_files": {},
	859	+ "cached_build": False
853	860	}
854	861	}
855	862	}),
...	...	@@ -874,7 +881,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
874	881	"last_successful": "some_key",
875	882	"running_files": {
876	883	"beta.bst": ["some_file"]
877		- }
	884	+ },
	885	+ "cached_build": False
878	886	}
879	887	}
880	888	}),
...	...	@@ -894,7 +902,30 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
894	902	"alpha.bst": {
895	903	"prepared": True,
896	904	"path": "/workspaces/bravo",
897		- "running_files": {}
	905	+ "running_files": {},
	906	+ "cached_build": False
	907	+ }
	908	+ }
	909	+ }),
	910	+ # Test loading version 4
	911	+ ({
	912	+ "format-version": 4,
	913	+ "workspaces": {
	914	+ "alpha.bst": {
	915	+ "prepared": False,
	916	+ "path": "/workspaces/bravo",
	917	+ "running_files": {},
	918	+ "cached_build": True
	919	+ }
	920	+ }
	921	+ }, {
	922	+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
	923	+ "workspaces": {
	924	+ "alpha.bst": {
	925	+ "prepared": False,
	926	+ "path": "/workspaces/bravo",
	927	+ "running_files": {},
	928	+ "cached_build": True
898	929	}
899	930	}
900	931	})
...	...	@@ -1055,3 +1086,80 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
1055	1086	result = cli.run(project=project, args=["build", element_name])
1056	1087	assert "BUG" not in result.stderr
1057	1088	assert cli.get_element_state(project, element_name) != "cached"
	1089	+
	1090	+
	1091	+@pytest.mark.datafiles(DATA_DIR)
	1092	+def test_nocache_open_messages(cli, tmpdir, datafiles):
	1093	+
	1094	+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
	1095	+ _, result = workspace_object.open_workspaces(('git', ), False)
	1096	+
	1097	+ # cli default WARN for source dropback possibility when no-cache flag is not passed
	1098	+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" in result.output
	1099	+
	1100	+ # cli WARN for source dropback happening when no-cache flag not given, but buildtree not available
	1101	+ assert "workspace will be opened with source checkout" in result.stderr
	1102	+
	1103	+ # cli default WARN for source dropback possibilty not given when no-cache flag is passed
	1104	+ tmpdir = os.path.join(str(tmpdir), "2")
	1105	+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
	1106	+ _, result = workspace_object.open_workspaces(('git', ), False, suffixes='1', no_cache=True)
	1107	+
	1108	+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" not in result.output
	1109	+
	1110	+
	1111	+@pytest.mark.datafiles(DATA_DIR)
	1112	+def test_nocache_reset_messages(cli, tmpdir, datafiles):
	1113	+
	1114	+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
	1115	+ workspaces, result = workspace_object.open_workspaces(('git', ), False)
	1116	+ element_name, workspace = workspaces[0]
	1117	+ project = workspace_object.project_path
	1118	+
	1119	+ # Modify workspace, without building so the artifact is not cached
	1120	+ shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
	1121	+ os.makedirs(os.path.join(workspace, 'etc'))
	1122	+ with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
	1123	+ f.write("PONY='pink'")
	1124	+
	1125	+ # Now reset the open workspace, this should have the
	1126	+ # effect of reverting our changes to the original source, as it
	1127	+ # was not originally opened with a cached buildtree and as such
	1128	+ # should not notify the user
	1129	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1130	+ 'workspace', 'reset', element_name
	1131	+ ])
	1132	+ result.assert_success()
	1133	+ assert "original buildtree artifact not available" not in result.output
	1134	+ assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello'))
	1135	+ assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
	1136	+
	1137	+ # Close the workspace
	1138	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1139	+ 'workspace', 'close', '--remove-dir', element_name
	1140	+ ])
	1141	+ result.assert_success()
	1142	+
	1143	+ # Build the workspace so we have a cached buildtree artifact for the element
	1144	+ assert cli.get_element_state(project, element_name) == 'buildable'
	1145	+ result = cli.run(project=project, args=['build', element_name])
	1146	+ result.assert_success()
	1147	+
	1148	+ # Opening the workspace after a build should lead to the cached buildtree being
	1149	+ # staged by default
	1150	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1151	+ 'workspace', 'open', element_name
	1152	+ ])
	1153	+ result.assert_success()
	1154	+
	1155	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1156	+ 'workspace', 'list'
	1157	+ ])
	1158	+ result.assert_success()
	1159	+ # Now reset the workspace and ensure that a warning is not given about the artifact
	1160	+ # buildtree not being available
	1161	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1162	+ 'workspace', 'reset', element_name
	1163	+ ])
	1164	+ result.assert_success()
	1165	+ assert "original buildtree artifact not available" not in result.output

tests/integration/workspace.py

---

...	...	@@ -278,3 +278,39 @@ def test_incremental_configure_commands_run_only_once(cli, tmpdir, datafiles):
278	278	res = cli.run(project=project, args=['build', element_name])
279	279	res.assert_success()
280	280	assert not os.path.exists(os.path.join(workspace, 'prepared-again'))
	281	+
	282	+
	283	+@pytest.mark.integration
	284	+@pytest.mark.datafiles(DATA_DIR)
	285	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	286	+def test_workspace_contains_buildtree(cli, tmpdir, datafiles):
	287	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	288	+ workspace = os.path.join(cli.directory, 'workspace')
	289	+ element_name = 'autotools/amhello.bst'
	290	+
	291	+ # Ensure we're not using the shared artifact cache
	292	+ cli.configure({
	293	+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
	294	+ })
	295	+
	296	+ # First open the workspace
	297	+ res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
	298	+ res.assert_success()
	299	+
	300	+ # Check that by default the buildtree wasn't staged as not yet available in the cache
	301	+ assert not os.path.exists(os.path.join(workspace, 'src', 'hello'))
	302	+
	303	+ # Close the workspace, removing the dir
	304	+ res = cli.run(project=project, args=['workspace', 'close', '--remove-dir', element_name])
	305	+ res.assert_success()
	306	+
	307	+ # Build the element, so we have it cached along with the buildtreee
	308	+ res = cli.run(project=project, args=['build', element_name])
	309	+ res.assert_success()
	310	+
	311	+ # Open up the workspace, as the buildtree is cached by default it should open with the buildtree
	312	+ res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
	313	+ res.assert_success()
	314	+
	315	+ # Check that the buildtree was staged, by asserting output of the build exists in the dir
	316	+ assert os.path.exists(os.path.join(workspace, 'src', 'hello'))

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.