[Notes] [Git][BuildStream/buildstream][willsalmon/shellBuildTrees] Basic options for shell --build to use buildtrees\n issue #740

[Will Salmon <gitlab mg gitlab com>]

Title: GitLab

Will Salmon pushed to branch willsalmon/shellBuildTrees at BuildStream / buildstream

Commits:

• d1144b8a
  by William Salmon at 2018-12-06T16:47:48Z

  Basic options for shell --build to use buildtrees\n issue #740
  

5 changed files:

• buildstream/_frontend/cli.py
• buildstream/_stream.py
• buildstream/element.py
• tests/integration/build-tree.py
• tests/testutils/runcli.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -582,11 +582,13 @@ def show(app, elements, deps, except_, order, format_):
582	582	help="Mount a file or directory into the sandbox")
583	583	@click.option('--isolate', is_flag=True, default=False,
584	584	help='Create an isolated build sandbox')
	585	+@click.option('--use-buildtree', '-t', type=click.Choice(['ask', 'if_available', 'always', 'never']), default=None,
	586	+ help='Defaults to ask but if set to always the function will fail if a build tree is not available')
585	587	@click.argument('element',
586	588	type=click.Path(readable=False))
587	589	@click.argument('command', type=click.STRING, nargs=-1)
588	590	@click.pass_obj
589		-def shell(app, element, sysroot, mount, isolate, build_, command):
	591	+def shell(app, element, sysroot, mount, isolate, build_, use_buildtree, command):
590	592	"""Run a command in the target element's sandbox environment
591	593
592	594	This will stage a temporary sysroot for running the target
...	...	@@ -611,7 +613,16 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
611	613	scope = Scope.BUILD
612	614	else:
613	615	scope = Scope.RUN
614		-
	616	+
	617	+ use_buildtree_bool = None
	618	+ if use_buildtree is None:
	619	+ use_buildtree = 'ask'
	620	+ else:
	621	+ use_buildtree = use_buildtree.lower().strip()
	622	+ if use_buildtree == 'always':
	623	+ use_buildtree_bool = True
	624	+ elif use_buildtree == 'never':
	625	+ use_buildtree_bool = False
615	626	with app.initialized():
616	627	dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
617	628	element = dependencies[0]
...	...	@@ -620,12 +631,31 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
620	631	HostMount(path, host_path)
621	632	for host_path, path in mount
622	633	]
	634	+
	635	+ if not element._cached_buildtree():
	636	+ if use_buildtree_bool == True:
	637	+ raise AppError("No buildtree when requested")
	638	+ else:
	639	+ if use_buildtree_bool == True:
	640	+ pass
	641	+ elif app.interactive and use_buildtree == 'ask':
	642	+ if click.confirm('Do you want to use the cached buildtree?'):
	643	+ use_buildtree_bool = True
	644	+ else:
	645	+ use_buildtree_bool = False
	646	+ elif use_buildtree == 'if_available':
	647	+ use_buildtree_bool = True
	648	+ else:
	649	+ use_buildtree_bool = False
	650	+ assert type(use_buildtree_bool) == bool
	651	+
623	652	try:
624	653	exitcode = app.stream.shell(element, scope, prompt,
625	654	directory=sysroot,
626	655	mounts=mounts,
627	656	isolate=isolate,
628		- command=command)
	657	+ command=command,
	658	+ usebuildtree=use_buildtree_bool)
629	659	except BstError as e:
630	660	raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e
631	661

buildstream/_stream.py

---

...	...	@@ -132,7 +132,8 @@ class Stream():
132	132	directory=None,
133	133	mounts=None,
134	134	isolate=False,
135		- command=None):
	135	+ command=None,
	136	+ usebuildtree=None):
136	137
137	138	# Assert we have everything we need built, unless the directory is specified
138	139	# in which case we just blindly trust the directory, using the element
...	...	@@ -147,7 +148,8 @@ class Stream():
147	148	raise StreamError("Elements need to be built or downloaded before staging a shell environment",
148	149	detail="\n".join(missing_deps))
149	150
150		- return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command)
	151	+ return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command,
	152	+ usebuildtree=usebuildtree)
151	153
152	154	# build()
153	155	#

buildstream/element.py

---

...	...	@@ -1339,11 +1339,12 @@ class Element(Plugin):
1339	1339	# is used to stage things by the `bst checkout` codepath
1340	1340	#
1341	1341	@contextmanager
1342		- def _prepare_sandbox(self, scope, directory, shell=False, integrate=True):
	1342	+ def _prepare_sandbox(self, scope, directory, shell=False, integrate=True, usebuildtree=None):
1343	1343	# bst shell and bst checkout require a local sandbox.
1344	1344	bare_directory = True if directory else False
1345	1345	with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
1346	1346	bare_directory=bare_directory) as sandbox:
	1347	+ sandbox.usebuildtree = usebuildtree
1347	1348
1348	1349	# Configure always comes first, and we need it.
1349	1350	self.__configure_sandbox(sandbox)
...	...	@@ -1387,7 +1388,7 @@ class Element(Plugin):
1387	1388	# Stage all sources that need to be copied
1388	1389	sandbox_vroot = sandbox.get_virtual_directory()
1389	1390	host_vdirectory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True)
1390		- self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces)
	1391	+ self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces, usebuildtree=sandbox.usebuildtree)
1391	1392
1392	1393	# _stage_sources_at():
1393	1394	#
...	...	@@ -1397,9 +1398,9 @@ class Element(Plugin):
1397	1398	# vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into.
1398	1399	# mount_workspaces (bool): mount workspaces if True, copy otherwise
1399	1400	#
1400		- def _stage_sources_at(self, vdirectory, mount_workspaces=True):
	1401	+ def _stage_sources_at(self, vdirectory, mount_workspaces=True, usebuildtree=None):
1401	1402	with self.timed_activity("Staging sources", silent_nested=True):
1402		-
	1403	+ print("_stage_sources_at(", usebuildtree)
1403	1404	if not isinstance(vdirectory, Directory):
1404	1405	vdirectory = FileBasedDirectory(vdirectory)
1405	1406	if not vdirectory.is_empty():
...	...	@@ -1421,7 +1422,7 @@ class Element(Plugin):
1421	1422	.format(workspace.get_absolute_path())):
1422	1423	workspace.stage(temp_staging_directory)
1423	1424	# Check if we have a cached buildtree to use
1424		- elif self._cached_buildtree():
	1425	+ elif usebuildtree:
1425	1426	artifact_base, _ = self.__extract()
1426	1427	import_dir = os.path.join(artifact_base, 'buildtree')
1427	1428	else:
...	...	@@ -1855,9 +1856,10 @@ class Element(Plugin):
1855	1856	# Returns: Exit code
1856	1857	#
1857	1858	# If directory is not specified, one will be staged using scope
1858		- def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None):
	1859	+ def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None,
	1860	+ usebuildtree=None):
1859	1861
1860		- with self._prepare_sandbox(scope, directory, shell=True) as sandbox:
	1862	+ with self._prepare_sandbox(scope, directory, shell=True, usebuildtree=usebuildtree) as sandbox:
1861	1863	environment = self.get_environment()
1862	1864	environment = copy.copy(environment)
1863	1865	flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
...	...	@@ -2232,7 +2234,6 @@ class Element(Plugin):
2232	2234	specs=self.__remote_execution_specs,
2233	2235	bare_directory=bare_directory,
2234	2236	allow_real_directory=False)
2235		- yield sandbox
2236	2237
2237	2238	elif directory is not None and os.path.exists(directory):
2238	2239	if allow_remote and self.__remote_execution_specs:
...	...	@@ -2250,7 +2251,6 @@ class Element(Plugin):
2250	2251	config=config,
2251	2252	bare_directory=bare_directory,
2252	2253	allow_real_directory=not self.BST_VIRTUAL_DIRECTORY)
2253		- yield sandbox
2254	2254
2255	2255	else:
2256	2256	os.makedirs(context.builddir, exist_ok=True)
...	...	@@ -2264,6 +2264,10 @@ class Element(Plugin):
2264	2264	# Cleanup the build dir
2265	2265	utils._force_rmtree(rootdir)
2266	2266
	2267	+ return
	2268	+ sandbox.usebuildtree = None
	2269	+ yield sandbox
	2270	+
2267	2271	def __compose_default_splits(self, defaults):
2268	2272	project = self._get_project()
2269	2273

tests/integration/build-tree.py

---

...	...	@@ -27,9 +27,44 @@ def test_buildtree_staged(cli_integration, tmpdir, datafiles):
27	27	res.assert_success()
28	28
29	29	res = cli_integration.run(project=project, args=[
30		- 'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
	30	+ 'shell', '--build', element_name, '--', 'cat', 'test'
	31	+ ])
	32	+ res.assert_shell_error()
	33	+
	34	+
	35	+@pytest.mark.datafiles(DATA_DIR)
	36	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	37	+def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles):
	38	+ # i.e. tests that cached build trees are staged by `bst shell --build`
	39	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	40	+ element_name = 'build-shell/buildtree.bst'
	41	+
	42	+ res = cli_integration.run(project=project, args=['build', element_name])
	43	+ res.assert_success()
	44	+
	45	+ res = cli_integration.run(project=project, args=[
	46	+ 'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
31	47	])
32	48	res.assert_success()
	49	+ assert 'Hi' in res.output
	50	+
	51	+
	52	+@pytest.mark.datafiles(DATA_DIR)
	53	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	54	+def test_buildtree_staged_forced_false(cli_integration, tmpdir, datafiles):
	55	+ # i.e. tests that cached build trees are staged by `bst shell --build`
	56	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	57	+ element_name = 'build-shell/buildtree.bst'
	58	+
	59	+ res = cli_integration.run(project=project, args=['build', element_name])
	60	+ res.assert_success()
	61	+
	62	+ res = cli_integration.run(project=project, args=[
	63	+ 'shell', '--build', '--use-buildtree', 'never', element_name, '--', 'cat', 'test'
	64	+ ])
	65	+ res.assert_shell_error()
	66	+
	67	+ assert 'Hi' not in res.output
33	68
34	69
35	70	@pytest.mark.datafiles(DATA_DIR)
...	...	@@ -44,7 +79,7 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
44	79
45	80	# Assert that file has expected contents
46	81	res = cli_integration.run(project=project, args=[
47		- 'shell', '--build', element_name, '--', 'cat', 'test'
	82	+ 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
48	83	])
49	84	res.assert_success()
50	85	assert 'Hi' in res.output
...	...	@@ -80,6 +115,48 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
80	115
81	116	# Check it's using the cached build tree
82	117	res = cli.run(project=project, args=[
83		- 'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
	118	+ 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'grep', '-q', 'Hi', 'test'
84	119	])
85	120	res.assert_success()
	121	+
	122	+
	123	+# Check that build shells work when pulled from a remote cache
	124	+# This is to roughly simulate remote execution
	125	+@pytest.mark.datafiles(DATA_DIR)
	126	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	127	+def test_buildtree_options(cli, tmpdir, datafiles):
	128	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	129	+ element_name = 'build-shell/buildtree.bst'
	130	+
	131	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
	132	+ # Build the element to push it to cache
	133	+ cli.configure({
	134	+ 'artifacts': {'url': share.repo, 'push': True}
	135	+ })
	136	+ result = cli.run(project=project, args=['build', element_name])
	137	+ result.assert_success()
	138	+ assert cli.get_element_state(project, element_name) == 'cached'
	139	+
	140	+ # Discard the cache
	141	+ cli.configure({
	142	+ 'artifacts': {'url': share.repo, 'push': True},
	143	+ 'artifactdir': os.path.join(cli.directory, 'artifacts2')
	144	+ })
	145	+ assert cli.get_element_state(project, element_name) != 'cached'
	146	+
	147	+ # Pull from cache, ensuring cli options is set to pull the buildtree
	148	+ result = cli.run(project=project, args=['pull', '--deps', 'all', element_name])
	149	+ result.assert_success()
	150	+
	151	+ # Check it's using the cached build tree
	152	+ res = cli.run(project=project, args=[
	153	+ 'shell', '--build', element_name, '--use-buildtree', 'never', '--', 'cat', 'test'
	154	+ ])
	155	+ res.assert_shell_error()
	156	+ assert 'Hi' not in res.output
	157	+ # Check it's using the cached build tree
	158	+ res = cli.run(project=project, args=[
	159	+ 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
	160	+ ])
	161	+ res.assert_main_error(ErrorDomain.PROG_NOT_FOUND, None)
	162	+ assert 'Hi' not in res.output

tests/testutils/runcli.py

---

...	...	@@ -153,6 +153,20 @@ class Result():
153	153	assert self.task_error_domain == error_domain, fail_message
154	154	assert self.task_error_reason == error_reason, fail_message
155	155
	156	+ # assert_shell_error()
	157	+ #
	158	+ # Asserts that the buildstream created a shell and that the task in the
	159	+ # shell failed.
	160	+ #
	161	+ # Args:
	162	+ # fail_message (str): An optional message to override the automatic
	163	+ # assertion error messages
	164	+ # Raises:
	165	+ # (AssertionError): If any of the assertions fail
	166	+ #
	167	+ def assert_shell_error(self, fail_message=''):
	168	+ assert self.exit_code == 1, fail_message
	169	+
156	170	# get_tracked_elements()
157	171	#
158	172	# Produces a list of element names on which tracking occurred

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.