[Notes] [Git][BuildStream/buildstream][tpollard/workspacebuildtree] 3 commits: filter.py: don't recurse when staging dependencies

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/workspacebuildtree at BuildStream / buildstream

Commits:

• 03111d39
  by Dor Askayo at 2019-01-30T10:35:06Z

  filter.py: don't recurse when staging dependencies
  
  Also bump the element's version so cached artifacts would be
  invalidated.
  
  Fixes #883
  

• 7256bb0c
  by James Ennis at 2019-01-30T11:34:04Z

  Merge branch 'doraskayo/filter-indirect-deps-fix' into 'master'
  
  filter.py: don't recurse when staging dependencies
  
  Closes #883
  
  See merge request BuildStream/buildstream!1110

• 4413ea95
  by Tom Pollard at 2019-01-31T10:08:24Z

  WIP: Opening a workspace with a cached build
  

16 changed files:

• buildstream/_artifactcache.py
• buildstream/_cas/cascache.py
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_gitsourcebase.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/element.py
• buildstream/plugins/elements/filter.py
• buildstream/source.py
• doc/source/developing/workspaces.rst
• tests/elements/filter.py
• + tests/elements/filter/basic/elements/input-with-deps.bst
• + tests/elements/filter/basic/elements/output-include-with-indirect-deps.bst
• tests/frontend/workspace.py
• tests/integration/workspace.py

Changes:

buildstream/_artifactcache.py

---

...	...	@@ -855,6 +855,20 @@ class ArtifactCache():
855	855
856	856	self.cas.link_ref(oldref, newref)
857	857
	858	+ # checkout_artifact_subdir()
	859	+ #
	860	+ # Checkout given artifact subdir into provided directory
	861	+ #
	862	+ # Args:
	863	+ # element (Element): The Element
	864	+ # key (str): The cache key to use
	865	+ # subdir (str): The subdir to checkout
	866	+ # tmpdir (str): The dir to place the subdir content
	867	+ #
	868	+ def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
	869	+ ref = self.get_artifact_fullname(element, key)
	870	+ return self.cas.checkout_artifact_subdir(ref, subdir, tmpdir)
	871	+
858	872	################################################
859	873	# Local Private Methods #
860	874	################################################

buildstream/_cas/cascache.py

---

...	...	@@ -587,6 +587,21 @@ class CASCache():
587	587	reachable = set()
588	588	self._reachable_refs_dir(reachable, tree, update_mtime=True)
589	589
	590	+ # checkout_artifact_subdir():
	591	+ #
	592	+ # Checkout given artifact subdir into provided directory
	593	+ #
	594	+ # Args:
	595	+ # ref (str): The ref to check
	596	+ # subdir (str): The subdir to checkout
	597	+ # tmpdir (str): The dir to place the subdir content
	598	+ #
	599	+ def checkout_artifact_subdir(self, ref, subdir, tmpdir):
	600	+ tree = self.resolve_ref(ref)
	601	+ # This assumes that the subdir digest is present in the element tree
	602	+ subdirdigest = self._get_subdir(tree, subdir)
	603	+ self._checkout(tmpdir, subdirdigest)
	604	+
590	605	################################################
591	606	# Local Private Methods #
592	607	################################################

buildstream/_context.py

---

...	...	@@ -125,6 +125,9 @@ class Context():
125	125	# close the workspace when they're using it to access the project.
126	126	self.prompt_workspace_close_project_inaccessible = None
127	127
	128	+ # Whether to include artifact buildtrees in workspaces if available
	129	+ self.workspace_buildtrees = True
	130	+
128	131	# Whether elements must be rebuilt when their dependencies have changed
129	132	self._strict_build_plan = None
130	133
...	...	@@ -183,7 +186,8 @@ class Context():
183	186	_yaml.node_validate(defaults, [
184	187	'sourcedir', 'builddir', 'artifactdir', 'logdir',
185	188	'scheduler', 'artifacts', 'logging', 'projects',
186		- 'cache', 'prompt', 'workspacedir', 'remote-execution'
	189	+ 'cache', 'prompt', 'workspacedir', 'remote-execution',
	190	+ 'workspace-buildtrees'
187	191	])
188	192
189	193	for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
...	...	@@ -213,6 +217,9 @@ class Context():
213	217	# Load pull build trees configuration
214	218	self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
215	219
	220	+ # Load workspace buildtrees configuration
	221	+ self.workspace_buildtrees = _yaml.node_get(defaults, bool, 'workspace-buildtrees', default_value='True')
	222	+
216	223	# Load logging config
217	224	logging = _yaml.node_get(defaults, Mapping, 'logging')
218	225	_yaml.node_validate(logging, [

buildstream/_frontend/cli.py

---

...	...	@@ -770,7 +770,7 @@ def workspace():
770	770	##################################################################
771	771	@workspace.command(name='open', short_help="Open a new workspace")
772	772	@click.option('--no-checkout', default=False, is_flag=True,
773		- help="Do not checkout the source, only link to the given directory")
	773	+ help="Do not checkout the source or cached buildtree, only link to the given directory")
774	774	@click.option('--force', '-f', default=False, is_flag=True,
775	775	help="The workspace will be created even if the directory in which it will be created is not empty " +
776	776	"or if a workspace for that element already exists")
...	...	@@ -779,16 +779,25 @@ def workspace():
779	779	@click.option('--directory', type=click.Path(file_okay=False), default=None,
780	780	help="Only for use when a single Element is given: Set the directory to use to create the workspace")
781	781	@click.argument('elements', nargs=-1, type=click.Path(readable=False), required=True)
	782	+@click.option('--no-cache', default=False, is_flag=True,
	783	+ help="Do not checkout the cached buildtree")
782	784	@click.pass_obj
783		-def workspace_open(app, no_checkout, force, track_, directory, elements):
784		- """Open a workspace for manual source modification"""
	785	+def workspace_open(app, no_checkout, force, track_, directory, elements, no_cache):
	786	+
	787	+ """Open a workspace for manual source modification, the elements buildtree
	788	+ will be provided if available in the local artifact cache.
	789	+ """
	790	+
	791	+ if not no_cache and not no_checkout:
	792	+ click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
785	793
786	794	with app.initialized():
787	795	app.stream.workspace_open(elements,
788	796	no_checkout=no_checkout,
789	797	track_first=track_,
790	798	force=force,
791		- custom_dir=directory)
	799	+ custom_dir=directory,
	800	+ no_cache=no_cache)
792	801
793	802
794	803	##################################################################

buildstream/_gitsourcebase.py

---

...	...	@@ -223,6 +223,31 @@ class GitMirror(SourceFetcher):
223	223	fail="Failed to checkout git ref {}".format(self.ref),
224	224	cwd=fullpath)
225	225
	226	+ def init_cached_build_workspace(self, directory):
	227	+ fullpath = os.path.join(directory, self.path)
	228	+ url = self.source.translate_url(self.url)
	229	+
	230	+ self.source.call([self.source.host_git, 'init', fullpath],
	231	+ fail="Failed to init git in directory: {}".format(fullpath),
	232	+ fail_temporarily=True,
	233	+ cwd=fullpath)
	234	+
	235	+ self.source.call([self.source.host_git, 'fetch', self.mirror],
	236	+ fail='Failed to fetch from local mirror "{}"'.format(self.mirror),
	237	+ cwd=fullpath)
	238	+
	239	+ self.source.call([self.source.host_git, 'remote', 'add', 'origin', url],
	240	+ fail='Failed to add remote origin "{}"'.format(url),
	241	+ cwd=fullpath)
	242	+
	243	+ self.source.call([self.source.host_git, 'update-ref', '--no-deref', 'HEAD', self.ref],
	244	+ fail='Failed update HEAD to ref "{}"'.format(self.ref),
	245	+ cwd=fullpath)
	246	+
	247	+ self.source.call([self.source.host_git, 'read-tree', 'HEAD'],
	248	+ fail='Failed to read HEAD into index',
	249	+ cwd=fullpath)
	250	+
226	251	# List the submodules (path/url tuples) present at the given ref of this repo
227	252	def submodule_list(self):
228	253	modules = "{}:{}".format(self.ref, GIT_MODULES)
...	...	@@ -522,6 +547,14 @@ class _GitSourceBase(Source):
522	547	for mirror in self.submodules:
523	548	mirror.init_workspace(directory)
524	549
	550	+ def init_cached_build_workspace(self, directory):
	551	+ self._refresh_submodules()
	552	+
	553	+ with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
	554	+ self.mirror.init_cached_build_workspace(directory)
	555	+ for mirror in self.submodules:
	556	+ mirror.init_cached_build_workspace(directory)
	557	+
525	558	def stage(self, directory):
526	559
527	560	# Need to refresh submodule list here again, because

buildstream/_stream.py

---

...	...	@@ -531,14 +531,21 @@ class Stream():
531	531	# track_first (bool): Whether to track and fetch first
532	532	# force (bool): Whether to ignore contents in an existing directory
533	533	# custom_dir (str): Custom location to create a workspace or false to use default location.
	534	+ # no_cache (bool): Whether to not include the cached buildtree
534	535	#
535	536	def workspace_open(self, targets, *,
536	537	no_checkout,
537	538	track_first,
538	539	force,
539		- custom_dir):
	540	+ custom_dir,
	541	+ no_cache):
	542	+
540	543	# This function is a little funny but it is trying to be as atomic as possible.
541	544
	545	+ # Set no_cache if the global user conf workspacebuildtrees is false
	546	+ if not self._context.workspace_buildtrees:
	547	+ no_cache = True
	548	+
542	549	if track_first:
543	550	track_targets = targets
544	551	else:
...	...	@@ -596,7 +603,7 @@ class Stream():
596	603	directory = os.path.abspath(custom_dir)
597	604	expanded_directories = [directory, ]
598	605	else:
599		- # If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the
	606	+ # If this fails it is a bug in whatever calls this, usually cli.py and so can not be tested for via the
600	607	# run bst test mechanism.
601	608	assert len(elements) == len(expanded_directories)
602	609
...	...	@@ -611,12 +618,26 @@ class Stream():
611	618	.format(target.name, directory), reason='bad-directory')
612	619
613	620	# So far this function has tried to catch as many issues as possible with out making any changes
614		- # Now it dose the bits that can not be made atomic.
	621	+ # Now it does the bits that can not be made atomic.
615	622	targetGenerator = zip(elements, expanded_directories)
616	623	for target, directory in targetGenerator:
617	624	self._message(MessageType.INFO, "Creating workspace for element {}"
618	625	.format(target.name))
619	626
	627	+ # Check if given target has a buildtree artifact cached locally
	628	+ buildtree = None
	629	+ if target._cached():
	630	+ buildtree = target._cached_buildtree()
	631	+
	632	+ # If we're running in the default state, make the user aware of buildtree usage
	633	+ if not no_cache and not no_checkout:
	634	+ if buildtree:
	635	+ self._message(MessageType.INFO, "{} buildtree artifact is available,"
	636	+ " workspace will be opened with it".format(target.name))
	637	+ else:
	638	+ self._message(MessageType.WARN, "{} buildtree artifact not available,"
	639	+ " workspace will be opened with source checkout".format(target.name))
	640	+
620	641	workspace = workspaces.get_workspace(target._get_full_name())
621	642	if workspace:
622	643	workspaces.delete_workspace(target._get_full_name())
...	...	@@ -631,7 +652,20 @@ class Stream():
631	652	todo_elements = "\nDid not try to create workspaces for " + todo_elements
632	653	raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
633	654
634		- workspaces.create_workspace(target, directory, checkout=not no_checkout)
	655	+ # Handle opening workspace with buildtree included
	656	+ if (buildtree and not no_cache) and not no_checkout:
	657	+ workspaces.create_workspace(target, directory, checkout=not no_checkout, cached_build=buildtree)
	658	+ with target.timed_activity("Staging buildtree to {}".format(directory)):
	659	+ target._open_workspace(buildtree=buildtree)
	660	+ else:
	661	+ workspaces.create_workspace(target, directory, checkout=not no_checkout)
	662	+ if (not buildtree or no_cache) and not no_checkout:
	663	+ with target.timed_activity("Staging sources to {}".format(directory)):
	664	+ target._open_workspace()
	665	+
	666	+ # Saving the workspace once it is set up means that if the next workspace fails to be created before
	667	+ # the configuration gets saved. The successfully created workspace still gets saved.
	668	+ workspaces.save_config()
635	669	self._message(MessageType.INFO, "Created a workspace for element: {}"
636	670	.format(target._get_full_name()))
637	671
...	...	@@ -714,7 +748,25 @@ class Stream():
714	748	.format(workspace_path, e)) from e
715	749
716	750	workspaces.delete_workspace(element._get_full_name())
717		- workspaces.create_workspace(element, workspace_path, checkout=True)
	751	+
	752	+ # Create the workspace, ensuring the original optional cached build state is preserved if
	753	+ # possible.
	754	+ buildtree = False
	755	+ if workspace.cached_build and element._cached():
	756	+ if self._artifacts.contains_subdir_artifact(element, element._get_cache_key(), 'buildtree'):
	757	+ buildtree = True
	758	+
	759	+ # Warn the user if the workspace cannot be opened with the original cached build state
	760	+ if workspace.cached_build and not buildtree:
	761	+ self._message(MessageType.WARN, "{} original buildtree artifact not available,"
	762	+ " workspace will be opened with source checkout".format(element.name))
	763	+
	764	+ # If opening the cached build, set checkout to false
	765	+ workspaces.create_workspace(element, workspace_path,
	766	+ checkout=not buildtree, cached_build=buildtree)
	767	+
	768	+ with element.timed_activity("Staging to {}".format(workspace_path)):
	769	+ element._open_workspace(buildtree=buildtree)
718	770
719	771	self._message(MessageType.INFO,
720	772	"Reset workspace for {} at: {}".format(element.name,

buildstream/_workspaces.py

---

...	...	@@ -24,7 +24,7 @@ from . import _yaml
24	24	from ._exceptions import LoadError, LoadErrorReason
25	25
26	26
27		-BST_WORKSPACE_FORMAT_VERSION = 3
	27	+BST_WORKSPACE_FORMAT_VERSION = 4
28	28	BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1
29	29	WORKSPACE_PROJECT_FILE = ".bstproject.yaml"
30	30
...	...	@@ -239,9 +239,11 @@ class WorkspaceProjectCache():
239	239	# running_files (dict): A dict mapping dependency elements to files
240	240	# changed between failed builds. Should be
241	241	# made obsolete with failed build artifacts.
	242	+# cached_build (bool): If the workspace is staging the cached build artifact
242	243	#
243	244	class Workspace():
244		- def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
	245	+ def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False,
	246	+ running_files=None, cached_build=False):
245	247	self.prepared = prepared
246	248	self.last_successful = last_successful
247	249	self._path = path
...	...	@@ -249,6 +251,7 @@ class Workspace():
249	251
250	252	self._toplevel_project = toplevel_project
251	253	self._key = None
	254	+ self.cached_build = cached_build
252	255
253	256	# to_dict()
254	257	#
...	...	@@ -261,7 +264,8 @@ class Workspace():
261	264	ret = {
262	265	'prepared': self.prepared,
263	266	'path': self._path,
264		- 'running_files': self.running_files
	267	+ 'running_files': self.running_files,
	268	+ 'cached_build': self.cached_build
265	269	}
266	270	if self.last_successful is not None:
267	271	ret["last_successful"] = self.last_successful
...	...	@@ -429,8 +433,9 @@ class Workspaces():
429	433	# target (Element) - The element to create a workspace for
430	434	# path (str) - The path in which the workspace should be kept
431	435	# checkout (bool): Whether to check-out the element's sources into the directory
	436	+ # cached_build (bool) - If the workspace is staging the cached build artifact
432	437	#
433		- def create_workspace(self, target, path, *, checkout):
	438	+ def create_workspace(self, target, path, *, checkout, cached_build=False):
434	439	element_name = target._get_full_name()
435	440	project_dir = self._toplevel_project.directory
436	441	if path.startswith(project_dir):
...	...	@@ -438,7 +443,8 @@ class Workspaces():
438	443	else:
439	444	workspace_path = path
440	445
441		- self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path)
	446	+ self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path,
	447	+ cached_build=cached_build)
442	448
443	449	if checkout:
444	450	with target.timed_activity("Staging sources to {}".format(path)):
...	...	@@ -627,6 +633,7 @@ class Workspaces():
627	633	'path': _yaml.node_get(node, str, 'path'),
628	634	'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
629	635	'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None),
	636	+ 'cached_build': _yaml.node_get(node, bool, 'cached_build', default_value=False)
630	637	}
631	638	return Workspace.from_dict(self._toplevel_project, dictionary)
632	639

buildstream/element.py

---

...	...	@@ -1909,7 +1909,10 @@ class Element(Plugin):
1909	1909	# This requires that a workspace already be created in
1910	1910	# the workspaces metadata first.
1911	1911	#
1912		- def _open_workspace(self):
	1912	+ # Args:
	1913	+ # buildtree (bool): Whether to open workspace with artifact buildtree
	1914	+ #
	1915	+ def _open_workspace(self, buildtree=False):
1913	1916	context = self._get_context()
1914	1917	workspace = self._get_workspace()
1915	1918	assert workspace is not None
...	...	@@ -1922,11 +1925,19 @@ class Element(Plugin):
1922	1925	# files in the target directory actually works without any
1923	1926	# additional support from Source implementations.
1924	1927	#
	1928	+
1925	1929	os.makedirs(context.builddir, exist_ok=True)
1926	1930	with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
1927	1931	.format(self.normal_name)) as temp:
1928		- for source in self.sources():
1929		- source._init_workspace(temp)
	1932	+
	1933	+ # Checkout cached buildtree, augment with source plugin if applicable
	1934	+ if buildtree:
	1935	+ self.__artifacts.checkout_artifact_subdir(self, self._get_cache_key(), 'buildtree', temp)
	1936	+ for source in self.sources():
	1937	+ source._init_cached_build_workspace(temp)
	1938	+ else:
	1939	+ for source in self.sources():
	1940	+ source._init_workspace(temp)
1930	1941
1931	1942	# Now hardlink the files into the workspace target.
1932	1943	utils.link_files(temp, workspace.get_absolute_path())

buildstream/plugins/elements/filter.py

---

...	...	@@ -47,6 +47,8 @@ from buildstream import Element, ElementError, Scope
47	47	class FilterElement(Element):
48	48	# pylint: disable=attribute-defined-outside-init
49	49
	50	+ BST_ARTIFACT_VERSION = 1
	51	+
50	52	# The filter element's output is its dependencies, so
51	53	# we must rebuild if the dependencies change even when
52	54	# not in strict build plans.
...	...	@@ -102,7 +104,7 @@ class FilterElement(Element):
102	104
103	105	def assemble(self, sandbox):
104	106	with self.timed_activity("Staging artifact", silent_nested=True):
105		- for dep in self.dependencies(Scope.BUILD):
	107	+ for dep in self.dependencies(Scope.BUILD, recurse=False):
106	108	dep.stage_artifact(sandbox, include=self.include,
107	109	exclude=self.exclude, orphans=self.include_orphans)
108	110	return ""

buildstream/source.py

---

...	...	@@ -465,6 +465,24 @@ class Source(Plugin):
465	465	"""
466	466	self.stage(directory)
467	467
	468	+ def init_cached_build_workspace(self, directory):
	469	+ """Initialises a new cached build workspace
	470	+
	471	+ Args:
	472	+ directory (str): Path of the workspace to init
	473	+
	474	+ Raises:
	475	+ :class:`.SourceError`
	476	+
	477	+ Implementors overriding this method should assume that *directory*
	478	+ already exists.
	479	+
	480	+ Implementors should raise :class:`.SourceError` when encountering
	481	+ some system error.
	482	+ """
	483	+ # Allow a non implementation
	484	+ return None
	485	+
468	486	def get_source_fetchers(self):
469	487	"""Get the objects that are used for fetching
470	488
...	...	@@ -717,6 +735,12 @@ class Source(Plugin):
717	735
718	736	self.init_workspace(directory)
719	737
	738	+ # Wrapper for init_cached_build_workspace()
	739	+ def _init_cached_build_workspace(self, directory):
	740	+ directory = self.__ensure_directory(directory)
	741	+
	742	+ self.init_cached_build_workspace(directory)
	743	+
720	744	# _get_unique_key():
721	745	#
722	746	# Wrapper for get_unique_key() api

doc/source/developing/workspaces.rst

---

...	...	@@ -24,9 +24,32 @@ Suppose we now want to alter the functionality of the *hello* command. We can
24	24	make changes to the source code of Buildstream elements by making use of
25	25	BuildStream's workspace command.
26	26
	27	+Utilising cached buildtrees
	28	+---------------------------
	29	+ When a BuildStream build element artifact is created and cached, a snapshot of
	30	+ the build directory after the build commands have completed is included in the
	31	+ artifact. This `build tree` can be considered an intermediary state of element,
	32	+ where the source is present along with any output created during the build
	33	+ execution.
	34	+
	35	+ By default when opening a workspace, bst will attempt to stage the build tree
	36	+ into the workspace if it's available in the local cache. If the respective
	37	+ build tree is not present in the cache (element not cached, partially cached or
	38	+ is a non build element) then the source will be staged as is. The default
	39	+ behaviour to attempt to use the build tree can be overriden with specific bst
	40	+ workspace open option of `--no-cache`, or via setting user configuration option
	41	+ `workspacebuildtrees: False`
	42	+
27	43
28	44	Opening a workspace
29	45	-------------------
	46	+.. note::
	47	+
	48	+ This example presumes you built the hello.bst during
	49	+ :ref:`running commands <tutorial_running_commands>`
	50	+ if not, please start by building it.
	51	+
	52	+
30	53	First we need to open a workspace, we can do this by running
31	54
32	55	.. raw:: html
...	...	@@ -93,6 +116,15 @@ Alternatively, if we wish to discard the changes we can use
93	116
94	117	This resets the workspace to its original state.
95	118
	119	+.. note::
	120	+
	121	+ bst reset will attempt to open the workspace in
	122	+ the condition in which it was originally staged,
	123	+ i.e with or without consuming the element build tree.
	124	+ If it was originally staged with a cached build tree
	125	+ and there's no longer one available, the source will
	126	+ be staged as is.
	127	+
96	128	To discard the workspace completely we can do:
97	129
98	130	.. raw:: html

tests/elements/filter.py

---

...	...	@@ -464,3 +464,23 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir):
464	464	assert "ref" not in new_input["sources"][0]
465	465	new_input2 = _yaml.load(input2_file)
466	466	assert new_input2["sources"][0]["ref"] == ref
	467	+
	468	+
	469	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
	470	+def test_filter_include_with_indirect_deps(datafiles, cli, tmpdir):
	471	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	472	+ result = cli.run(project=project, args=[
	473	+ 'build', 'output-include-with-indirect-deps.bst'])
	474	+ result.assert_success()
	475	+
	476	+ checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout')
	477	+ result = cli.run(project=project, args=[
	478	+ 'artifact', 'checkout', 'output-include-with-indirect-deps.bst', '--directory', checkout])
	479	+ result.assert_success()
	480	+
	481	+ # direct dependencies should be staged and filtered
	482	+ assert os.path.exists(os.path.join(checkout, "baz"))
	483	+
	484	+ # indirect dependencies shouldn't be staged and filtered
	485	+ assert not os.path.exists(os.path.join(checkout, "foo"))
	486	+ assert not os.path.exists(os.path.join(checkout, "bar"))

tests/elements/filter/basic/elements/input-with-deps.bst

---

	1	+kind: import
	2	+
	3	+depends:
	4	+- filename: input.bst
	5	+
	6	+sources:
	7	+- kind: local
	8	+ path: files
	9	+
	10	+public:
	11	+ bst:
	12	+ split-rules:
	13	+ baz:
	14	+ - /baz

tests/elements/filter/basic/elements/output-include-with-indirect-deps.bst

---

	1	+kind: filter
	2	+
	3	+depends:
	4	+- filename: input-with-deps.bst
	5	+ type: build

tests/frontend/workspace.py

---

...	...	@@ -92,18 +92,18 @@ class WorkspaceCreater():
92	92	element_name))
93	93	return element_name, element_path, workspace_dir
94	94
95		- def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None,
	95	+ def create_workspace_elements(self, kinds, track, suffixes=None, workspace_dir_usr=None,
96	96	element_attrs=None):
97	97
98	98	element_tuples = []
99	99
100		- if suffixs is None:
101		- suffixs = ['', ] * len(kinds)
	100	+ if suffixes is None:
	101	+ suffixes = ['', ] * len(kinds)
102	102	else:
103		- if len(suffixs) != len(kinds):
	103	+ if len(suffixes) != len(kinds):
104	104	raise "terable error"
105	105
106		- for suffix, kind in zip(suffixs, kinds):
	106	+ for suffix, kind in zip(suffixes, kinds):
107	107	element_name, element_path, workspace_dir = \
108	108	self.create_workspace_element(kind, track, suffix, workspace_dir_usr,
109	109	element_attrs)
...	...	@@ -120,10 +120,10 @@ class WorkspaceCreater():
120	120
121	121	return element_tuples
122	122
123		- def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None,
124		- element_attrs=None, no_checkout=False):
	123	+ def open_workspaces(self, kinds, track, suffixes=None, workspace_dir=None,
	124	+ element_attrs=None, no_checkout=False, no_cache=False):
125	125
126		- element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir,
	126	+ element_tuples = self.create_workspace_elements(kinds, track, suffixes, workspace_dir,
127	127	element_attrs)
128	128	os.makedirs(self.workspace_cmd, exist_ok=True)
129	129
...	...	@@ -134,12 +134,15 @@ class WorkspaceCreater():
134	134	args.append('--track')
135	135	if no_checkout:
136	136	args.append('--no-checkout')
	137	+ if no_cache:
	138	+ args.append('--no-cache')
137	139	if workspace_dir is not None:
138	140	assert len(element_tuples) == 1, "test logic error"
139	141	_, workspace_dir = element_tuples[0]
140	142	args.extend(['--directory', workspace_dir])
141		-
	143	+ print("element_tuples", element_tuples)
142	144	args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
	145	+ print("args", args)
143	146	result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
144	147
145	148	result.assert_success()
...	...	@@ -156,14 +159,14 @@ class WorkspaceCreater():
156	159	filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
157	160	assert os.path.exists(filename)
158	161
159		- return element_tuples
	162	+ return element_tuples, result
160	163
161	164
162	165	def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
163		- project_path=None, element_attrs=None, no_checkout=False):
	166	+ project_path=None, element_attrs=None, no_checkout=False, no_cache=False):
164	167	workspace_object = WorkspaceCreater(cli, tmpdir, datafiles, project_path)
165		- workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
166		- element_attrs, no_checkout)
	168	+ workspaces, _ = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
	169	+ element_attrs, no_checkout, no_cache)
167	170	assert len(workspaces) == 1
168	171	element_name, workspace = workspaces[0]
169	172	return element_name, workspace_object.project_path, workspace
...	...	@@ -197,7 +200,7 @@ def test_open_bzr_customize(cli, tmpdir, datafiles):
197	200	def test_open_multi(cli, tmpdir, datafiles):
198	201
199	202	workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
200		- workspaces = workspace_object.open_workspaces(repo_kinds, False)
	203	+ workspaces, _ = workspace_object.open_workspaces(repo_kinds, False)
201	204
202	205	for (elname, workspace), kind in zip(workspaces, repo_kinds):
203	206	assert kind in elname
...	...	@@ -831,7 +834,9 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
831	834	"alpha.bst": {
832	835	"prepared": False,
833	836	"path": "/workspaces/bravo",
834		- "running_files": {}
	837	+ "running_files": {},
	838	+ "cached_build": False
	839	+
835	840	}
836	841	}
837	842	}),
...	...	@@ -846,7 +851,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
846	851	"alpha.bst": {
847	852	"prepared": False,
848	853	"path": "/workspaces/bravo",
849		- "running_files": {}
	854	+ "running_files": {},
	855	+ "cached_build": False
850	856	}
851	857	}
852	858	}),
...	...	@@ -864,7 +870,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
864	870	"alpha.bst": {
865	871	"prepared": False,
866	872	"path": "/workspaces/bravo",
867		- "running_files": {}
	873	+ "running_files": {},
	874	+ "cached_build": False
868	875	}
869	876	}
870	877	}),
...	...	@@ -889,7 +896,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
889	896	"last_successful": "some_key",
890	897	"running_files": {
891	898	"beta.bst": ["some_file"]
892		- }
	899	+ },
	900	+ "cached_build": False
893	901	}
894	902	}
895	903	}),
...	...	@@ -909,7 +917,30 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
909	917	"alpha.bst": {
910	918	"prepared": True,
911	919	"path": "/workspaces/bravo",
912		- "running_files": {}
	920	+ "running_files": {},
	921	+ "cached_build": False
	922	+ }
	923	+ }
	924	+ }),
	925	+ # Test loading version 4
	926	+ ({
	927	+ "format-version": 4,
	928	+ "workspaces": {
	929	+ "alpha.bst": {
	930	+ "prepared": False,
	931	+ "path": "/workspaces/bravo",
	932	+ "running_files": {},
	933	+ "cached_build": True
	934	+ }
	935	+ }
	936	+ }, {
	937	+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
	938	+ "workspaces": {
	939	+ "alpha.bst": {
	940	+ "prepared": False,
	941	+ "path": "/workspaces/bravo",
	942	+ "running_files": {},
	943	+ "cached_build": True
913	944	}
914	945	}
915	946	})
...	...	@@ -1235,3 +1266,80 @@ def test_external_list(cli, datafiles, tmpdir_factory):
1235	1266
1236	1267	result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
1237	1268	result.assert_success()
	1269	+
	1270	+
	1271	+@pytest.mark.datafiles(DATA_DIR)
	1272	+def test_nocache_open_messages(cli, tmpdir, datafiles):
	1273	+
	1274	+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
	1275	+ _, result = workspace_object.open_workspaces(('git', ), False)
	1276	+
	1277	+ # cli default WARN for source dropback possibility when no-cache flag is not passed
	1278	+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" in result.output
	1279	+
	1280	+ # cli WARN for source dropback happening when no-cache flag not given, but buildtree not available
	1281	+ assert "workspace will be opened with source checkout" in result.stderr
	1282	+
	1283	+ # cli default WARN for source dropback possibilty not given when no-cache flag is passed
	1284	+ tmpdir = os.path.join(str(tmpdir), "2")
	1285	+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
	1286	+ _, result = workspace_object.open_workspaces(('git', ), False, suffixes='1', no_cache=True)
	1287	+
	1288	+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" not in result.output
	1289	+
	1290	+
	1291	+@pytest.mark.datafiles(DATA_DIR)
	1292	+def test_nocache_reset_messages(cli, tmpdir, datafiles):
	1293	+
	1294	+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
	1295	+ workspaces, result = workspace_object.open_workspaces(('git', ), False)
	1296	+ element_name, workspace = workspaces[0]
	1297	+ project = workspace_object.project_path
	1298	+
	1299	+ # Modify workspace, without building so the artifact is not cached
	1300	+ shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
	1301	+ os.makedirs(os.path.join(workspace, 'etc'))
	1302	+ with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
	1303	+ f.write("PONY='pink'")
	1304	+
	1305	+ # Now reset the open workspace, this should have the
	1306	+ # effect of reverting our changes to the original source, as it
	1307	+ # was not originally opened with a cached buildtree and as such
	1308	+ # should not notify the user
	1309	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1310	+ 'workspace', 'reset', element_name
	1311	+ ])
	1312	+ result.assert_success()
	1313	+ assert "original buildtree artifact not available" not in result.output
	1314	+ assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello'))
	1315	+ assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
	1316	+
	1317	+ # Close the workspace
	1318	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1319	+ 'workspace', 'close', '--remove-dir', element_name
	1320	+ ])
	1321	+ result.assert_success()
	1322	+
	1323	+ # Build the workspace so we have a cached buildtree artifact for the element
	1324	+ assert cli.get_element_state(project, element_name) == 'buildable'
	1325	+ result = cli.run(project=project, args=['build', element_name])
	1326	+ result.assert_success()
	1327	+
	1328	+ # Opening the workspace after a build should lead to the cached buildtree being
	1329	+ # staged by default
	1330	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1331	+ 'workspace', 'open', element_name
	1332	+ ])
	1333	+ result.assert_success()
	1334	+
	1335	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1336	+ 'workspace', 'list'
	1337	+ ])
	1338	+ result.assert_success()
	1339	+ # Now reset the workspace and ensure that a warning is not given about the artifact
	1340	+ # buildtree not being available
	1341	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1342	+ 'workspace', 'reset', element_name
	1343	+ ])
	1344	+ result.assert_success()
	1345	+ assert "original buildtree artifact not available" not in result.output

tests/integration/workspace.py

---

...	...	@@ -278,3 +278,39 @@ def test_incremental_configure_commands_run_only_once(cli, tmpdir, datafiles):
278	278	res = cli.run(project=project, args=['build', element_name])
279	279	res.assert_success()
280	280	assert not os.path.exists(os.path.join(workspace, 'prepared-again'))
	281	+
	282	+
	283	+@pytest.mark.integration
	284	+@pytest.mark.datafiles(DATA_DIR)
	285	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	286	+def test_workspace_contains_buildtree(cli, tmpdir, datafiles):
	287	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	288	+ workspace = os.path.join(cli.directory, 'workspace')
	289	+ element_name = 'autotools/amhello.bst'
	290	+
	291	+ # Ensure we're not using the shared artifact cache
	292	+ cli.configure({
	293	+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
	294	+ })
	295	+
	296	+ # First open the workspace
	297	+ res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
	298	+ res.assert_success()
	299	+
	300	+ # Check that by default the buildtree wasn't staged as not yet available in the cache
	301	+ assert not os.path.exists(os.path.join(workspace, 'src', 'hello'))
	302	+
	303	+ # Close the workspace, removing the dir
	304	+ res = cli.run(project=project, args=['workspace', 'close', '--remove-dir', element_name])
	305	+ res.assert_success()
	306	+
	307	+ # Build the element, so we have it cached along with the buildtreee
	308	+ res = cli.run(project=project, args=['build', element_name])
	309	+ res.assert_success()
	310	+
	311	+ # Open up the workspace, as the buildtree is cached by default it should open with the buildtree
	312	+ res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
	313	+ res.assert_success()
	314	+
	315	+ # Check that the buildtree was staged, by asserting output of the build exists in the dir
	316	+ assert os.path.exists(os.path.join(workspace, 'src', 'hello'))

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.