[Notes] [Git][BuildStream/buildstream][willsalmon/shellBuildTrees] Basic options for shell --build to use buildtrees

[Will Salmon <gitlab mg gitlab com>]

Title: GitLab

Will Salmon pushed to branch willsalmon/shellBuildTrees at BuildStream / buildstream

Commits:

• 2b33ea5e
  by William Salmon at 2018-12-11T15:06:38Z

  Basic options for shell --build to use buildtrees
  
  Fixes issue #740
  

5 changed files:

• buildstream/_frontend/cli.py
• buildstream/_stream.py
• buildstream/element.py
• tests/integration/build-tree.py
• tests/testutils/runcli.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -582,11 +582,14 @@ def show(app, elements, deps, except_, order, format_):
582	582	help="Mount a file or directory into the sandbox")
583	583	@click.option('--isolate', is_flag=True, default=False,
584	584	help='Create an isolated build sandbox')
	585	+@click.option('--use-buildtree', '-t', 'cli_buildtree', type=click.Choice(['ask', 'if_available', 'always', 'never']),
	586	+ default='ask',
	587	+ help='Defaults to ask but if set to always the function will fail if a build tree is not available')
585	588	@click.argument('element',
586	589	type=click.Path(readable=False))
587	590	@click.argument('command', type=click.STRING, nargs=-1)
588	591	@click.pass_obj
589		-def shell(app, element, sysroot, mount, isolate, build_, command):
	592	+def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command):
590	593	"""Run a command in the target element's sandbox environment
591	594
592	595	This will stage a temporary sysroot for running the target
...	...	@@ -612,6 +615,9 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
612	615	else:
613	616	scope = Scope.RUN
614	617
	618	+ cli_buildtree = cli_buildtree.lower().strip()
	619	+ use_buildtree = False
	620	+
615	621	with app.initialized():
616	622	dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
617	623	element = dependencies[0]
...	...	@@ -620,12 +626,30 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
620	626	HostMount(path, host_path)
621	627	for host_path, path in mount
622	628	]
	629	+
	630	+ cached = element._cached_buildtree()
	631	+ if cli_buildtree == "always":
	632	+ if cached:
	633	+ use_buildtree = True
	634	+ else:
	635	+ raise AppError("No buildtree is cached but the use buildtree option was specified")
	636	+ elif cli_buildtree == "never":
	637	+ pass
	638	+ elif cli_buildtree == "if_available":
	639	+ use_buildtree = cached
	640	+ else:
	641	+ if app.interactive and cached:
	642	+ use_buildtree = bool(click.confirm('Do you want to use the cached buildtree?'))
	643	+ if use_buildtree and not element._cached_success():
	644	+ click.echo("Warning: using a buildtree from a failed build.")
	645	+
623	646	try:
624	647	exitcode = app.stream.shell(element, scope, prompt,
625	648	directory=sysroot,
626	649	mounts=mounts,
627	650	isolate=isolate,
628		- command=command)
	651	+ command=command,
	652	+ usebuildtree=use_buildtree)
629	653	except BstError as e:
630	654	raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e
631	655

buildstream/_stream.py

---

...	...	@@ -132,7 +132,8 @@ class Stream():
132	132	directory=None,
133	133	mounts=None,
134	134	isolate=False,
135		- command=None):
	135	+ command=None,
	136	+ usebuildtree=None):
136	137
137	138	# Assert we have everything we need built, unless the directory is specified
138	139	# in which case we just blindly trust the directory, using the element
...	...	@@ -147,7 +148,8 @@ class Stream():
147	148	raise StreamError("Elements need to be built or downloaded before staging a shell environment",
148	149	detail="\n".join(missing_deps))
149	150
150		- return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command)
	151	+ return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command,
	152	+ usebuildtree=usebuildtree)
151	153
152	154	# build()
153	155	#

buildstream/element.py

---

...	...	@@ -1338,11 +1338,12 @@ class Element(Plugin):
1338	1338	# is used to stage things by the `bst checkout` codepath
1339	1339	#
1340	1340	@contextmanager
1341		- def _prepare_sandbox(self, scope, directory, shell=False, integrate=True):
	1341	+ def _prepare_sandbox(self, scope, directory, shell=False, integrate=True, usebuildtree=None):
1342	1342	# bst shell and bst checkout require a local sandbox.
1343	1343	bare_directory = True if directory else False
1344	1344	with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
1345	1345	bare_directory=bare_directory) as sandbox:
	1346	+ sandbox.usebuildtree = usebuildtree
1346	1347
1347	1348	# Configure always comes first, and we need it.
1348	1349	self.__configure_sandbox(sandbox)
...	...	@@ -1386,7 +1387,7 @@ class Element(Plugin):
1386	1387	# Stage all sources that need to be copied
1387	1388	sandbox_vroot = sandbox.get_virtual_directory()
1388	1389	host_vdirectory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True)
1389		- self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces)
	1390	+ self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces, usebuildtree=sandbox.usebuildtree)
1390	1391
1391	1392	# _stage_sources_at():
1392	1393	#
...	...	@@ -1396,9 +1397,8 @@ class Element(Plugin):
1396	1397	# vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into.
1397	1398	# mount_workspaces (bool): mount workspaces if True, copy otherwise
1398	1399	#
1399		- def _stage_sources_at(self, vdirectory, mount_workspaces=True):
	1400	+ def _stage_sources_at(self, vdirectory, mount_workspaces=True, usebuildtree=False):
1400	1401	with self.timed_activity("Staging sources", silent_nested=True):
1401		-
1402	1402	if not isinstance(vdirectory, Directory):
1403	1403	vdirectory = FileBasedDirectory(vdirectory)
1404	1404	if not vdirectory.is_empty():
...	...	@@ -1420,7 +1420,7 @@ class Element(Plugin):
1420	1420	.format(workspace.get_absolute_path())):
1421	1421	workspace.stage(temp_staging_directory)
1422	1422	# Check if we have a cached buildtree to use
1423		- elif self._cached_buildtree():
	1423	+ elif usebuildtree:
1424	1424	artifact_base, _ = self.__extract()
1425	1425	import_dir = os.path.join(artifact_base, 'buildtree')
1426	1426	else:
...	...	@@ -1854,9 +1854,10 @@ class Element(Plugin):
1854	1854	# Returns: Exit code
1855	1855	#
1856	1856	# If directory is not specified, one will be staged using scope
1857		- def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None):
	1857	+ def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None,
	1858	+ usebuildtree=None):
1858	1859
1859		- with self._prepare_sandbox(scope, directory, shell=True) as sandbox:
	1860	+ with self._prepare_sandbox(scope, directory, shell=True, usebuildtree=usebuildtree) as sandbox:
1860	1861	environment = self.get_environment()
1861	1862	environment = copy.copy(environment)
1862	1863	flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
...	...	@@ -2231,7 +2232,6 @@ class Element(Plugin):
2231	2232	specs=self.__remote_execution_specs,
2232	2233	bare_directory=bare_directory,
2233	2234	allow_real_directory=False)
2234		- yield sandbox
2235	2235
2236	2236	elif directory is not None and os.path.exists(directory):
2237	2237	if allow_remote and self.__remote_execution_specs:
...	...	@@ -2249,7 +2249,6 @@ class Element(Plugin):
2249	2249	config=config,
2250	2250	bare_directory=bare_directory,
2251	2251	allow_real_directory=not self.BST_VIRTUAL_DIRECTORY)
2252		- yield sandbox
2253	2252
2254	2253	else:
2255	2254	os.makedirs(context.builddir, exist_ok=True)
...	...	@@ -2263,6 +2262,10 @@ class Element(Plugin):
2263	2262	# Cleanup the build dir
2264	2263	utils._force_rmtree(rootdir)
2265	2264
	2265	+ return
	2266	+ sandbox.usebuildtree = None
	2267	+ yield sandbox
	2268	+
2266	2269	def __compose_default_splits(self, defaults):
2267	2270	project = self._get_project()
2268	2271

tests/integration/build-tree.py

---

...	...	@@ -19,7 +19,9 @@ DATA_DIR = os.path.join(
19	19	@pytest.mark.datafiles(DATA_DIR)
20	20	@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
21	21	def test_buildtree_staged(cli_integration, tmpdir, datafiles):
22		- # i.e. tests that cached build trees are staged by `bst shell --build`
	22	+ # By default we ask about staging build trees but we can only ask
	23	+ # in a interactive session so by default trees are not staged by
	24	+ # `bst shell --build` in the tests
23	25	project = os.path.join(datafiles.dirname, datafiles.basename)
24	26	element_name = 'build-shell/buildtree.bst'
25	27
...	...	@@ -27,15 +29,50 @@ def test_buildtree_staged(cli_integration, tmpdir, datafiles):
27	29	res.assert_success()
28	30
29	31	res = cli_integration.run(project=project, args=[
30		- 'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
	32	+ 'shell', '--build', element_name, '--', 'cat', 'test'
	33	+ ])
	34	+ res.assert_shell_error()
	35	+
	36	+
	37	+@pytest.mark.datafiles(DATA_DIR)
	38	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	39	+def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles):
	40	+ # Test that if we ask for a build tree it is there.
	41	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	42	+ element_name = 'build-shell/buildtree.bst'
	43	+
	44	+ res = cli_integration.run(project=project, args=['build', element_name])
	45	+ res.assert_success()
	46	+
	47	+ res = cli_integration.run(project=project, args=[
	48	+ 'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
31	49	])
32	50	res.assert_success()
	51	+ assert 'Hi' in res.output
	52	+
	53	+
	54	+@pytest.mark.datafiles(DATA_DIR)
	55	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	56	+def test_buildtree_staged_forced_false(cli_integration, tmpdir, datafiles):
	57	+ # Test that if we ask not to have a build tree it is not there
	58	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	59	+ element_name = 'build-shell/buildtree.bst'
	60	+
	61	+ res = cli_integration.run(project=project, args=['build', element_name])
	62	+ res.assert_success()
	63	+
	64	+ res = cli_integration.run(project=project, args=[
	65	+ 'shell', '--build', '--use-buildtree', 'never', element_name, '--', 'cat', 'test'
	66	+ ])
	67	+ res.assert_shell_error()
	68	+
	69	+ assert 'Hi' not in res.output
33	70
34	71
35	72	@pytest.mark.datafiles(DATA_DIR)
36	73	@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
37	74	def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
38		- # i.e. test that on a build failure, we can still shell into it
	75	+ # Test that we can use a build tree after a failure
39	76	project = os.path.join(datafiles.dirname, datafiles.basename)
40	77	element_name = 'build-shell/buildtree-fail.bst'
41	78
...	...	@@ -44,9 +81,10 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
44	81
45	82	# Assert that file has expected contents
46	83	res = cli_integration.run(project=project, args=[
47		- 'shell', '--build', element_name, '--', 'cat', 'test'
	84	+ 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
48	85	])
49	86	res.assert_success()
	87	+ assert "Warning: using a buildtree from a failed build" in res.output
50	88	assert 'Hi' in res.output
51	89
52	90
...	...	@@ -80,6 +118,58 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
80	118
81	119	# Check it's using the cached build tree
82	120	res = cli.run(project=project, args=[
83		- 'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
	121	+ 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
84	122	])
85	123	res.assert_success()
	124	+
	125	+
	126	+# This test checks for correct behaviour if a buildtree is not present.
	127	+@pytest.mark.datafiles(DATA_DIR)
	128	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	129	+def test_buildtree_options(cli, tmpdir, datafiles):
	130	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	131	+ element_name = 'build-shell/buildtree.bst'
	132	+
	133	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
	134	+ # Build the element to push it to cache
	135	+ cli.configure({
	136	+ 'artifacts': {'url': share.repo, 'push': True}
	137	+ })
	138	+ result = cli.run(project=project, args=['build', element_name])
	139	+ result.assert_success()
	140	+ assert cli.get_element_state(project, element_name) == 'cached'
	141	+
	142	+ # Discard the cache
	143	+ cli.configure({
	144	+ 'artifacts': {'url': share.repo, 'push': True},
	145	+ 'artifactdir': os.path.join(cli.directory, 'artifacts2')
	146	+ })
	147	+ assert cli.get_element_state(project, element_name) != 'cached'
	148	+
	149	+ # Pull from cache, but do not include buildtrees.
	150	+ result = cli.run(project=project, args=['pull', '--deps', 'all', element_name])
	151	+ result.assert_success()
	152	+
	153	+ # The above is the simplest way I know to create a local cache without any buildtrees.
	154	+
	155	+ # Check it's not using the cached build tree
	156	+ res = cli.run(project=project, args=[
	157	+ 'shell', '--build', element_name, '--use-buildtree', 'never', '--', 'cat', 'test'
	158	+ ])
	159	+ res.assert_shell_error()
	160	+ assert 'Hi' not in res.output
	161	+
	162	+ # Check it's not using the cached build tree, default is to ask, and fall back to not
	163	+ # for non interactive behavior
	164	+ res = cli.run(project=project, args=[
	165	+ 'shell', '--build', element_name, '--', 'cat', 'test'
	166	+ ])
	167	+ res.assert_shell_error()
	168	+ assert 'Hi' not in res.output
	169	+
	170	+ # Check it's using the cached build tree
	171	+ res = cli.run(project=project, args=[
	172	+ 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
	173	+ ])
	174	+ res.assert_main_error(ErrorDomain.PROG_NOT_FOUND, None)
	175	+ assert 'Hi' not in res.output

tests/testutils/runcli.py

---

...	...	@@ -153,6 +153,20 @@ class Result():
153	153	assert self.task_error_domain == error_domain, fail_message
154	154	assert self.task_error_reason == error_reason, fail_message
155	155
	156	+ # assert_shell_error()
	157	+ #
	158	+ # Asserts that the buildstream created a shell and that the task in the
	159	+ # shell failed.
	160	+ #
	161	+ # Args:
	162	+ # fail_message (str): An optional message to override the automatic
	163	+ # assertion error messages
	164	+ # Raises:
	165	+ # (AssertionError): If any of the assertions fail
	166	+ #
	167	+ def assert_shell_error(self, fail_message=''):
	168	+ assert self.exit_code == 1, fail_message
	169	+
156	170	# get_tracked_elements()
157	171	#
158	172	# Produces a list of element names on which tracking occurred

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.