[Notes] [Git][BuildStream/buildstream][richardmaw/shell-multi-stage] 5 commits: element: Remove dead _shell method

[richardmaw-codethink <gitlab mg gitlab com>]

Title: GitLab

richardmaw-codethink pushed to branch richardmaw/shell-multi-stage at BuildStream / buildstream

Commits:

• fc1fedc8
  by Richard Maw at 2018-12-03T17:03:19Z

  element: Remove dead _shell method
  

• 3114c562
  by Richard Maw at 2018-12-03T17:03:19Z

  cli: Support multiple elements with -e option
  

• 7cacd32a
  by Richard Maw at 2018-12-03T17:03:19Z

  tests: Move integration.bst to subdirectory
  

• ff81acd2
  by Richard Maw at 2018-12-03T17:03:19Z

  tests: Test multiple-element runtime and workspace shells
  

• df286425
  by Richard Maw at 2018-12-03T17:03:19Z

  Add to NEWS
  

7 changed files:

• NEWS
• buildstream/_frontend/cli.py
• buildstream/element.py
• + tests/integration/project/elements/shell/adds-bar.bst
• + tests/integration/project/elements/shell/adds-foo.bst
• tests/integration/project/elements/integration.bst → tests/integration/project/elements/shell/integration.bst
• tests/integration/shell.py

Changes:

NEWS

---

...	...	@@ -2,6 +2,9 @@
2	2	buildstream 1.3.1
3	3	=================
4	4
	5	+ o `bst shell` learned the `-e` option for staging multiple elements
	6	+ provided the element's kind flags `BST_STAGE_INTEGRATES` as false.
	7	+
5	8	o All elements must now be suffixed with `.bst`
6	9	Attempting to use an element that does not have the `.bst` extension,
7	10	will result in a warning.

buildstream/_frontend/cli.py

---

...	...	@@ -573,7 +573,7 @@ def show(app, elements, deps, except_, order, format_):
573	573	##################################################################
574	574	@cli.command(short_help="Shell into an element's sandbox environment")
575	575	@click.option('--build', '-b', 'build_', is_flag=True, default=False,
576		- help='Stage dependencies and sources to build')
	576	+ help='Stage dependencies and sources to build the first element')
577	577	@click.option('--sysroot', '-s', default=None,
578	578	type=click.Path(exists=True, file_okay=False, readable=True),
579	579	help="An existing sysroot")
...	...	@@ -582,11 +582,15 @@ def show(app, elements, deps, except_, order, format_):
582	582	help="Mount a file or directory into the sandbox")
583	583	@click.option('--isolate', is_flag=True, default=False,
584	584	help='Create an isolated build sandbox')
	585	+@click.option('--element', '-e', 'elements', multiple=True,
	586	+ type=click.Path(readable=False), required=False,
	587	+ help=('Stage all these elements instead of just one. ' +
	588	+ 'Only the build-dependencies and sources of the first will be staged if --build is passed.'))
585	589	@click.argument('element',
586		- type=click.Path(readable=False))
	590	+ type=click.Path(readable=False), required=False)
587	591	@click.argument('command', type=click.STRING, nargs=-1)
588	592	@click.pass_obj
589		-def shell(app, element, sysroot, mount, isolate, build_, command):
	593	+def shell(app, element, elements, sysroot, mount, isolate, build_, command):
590	594	"""Run a command in the target element's sandbox environment
591	595
592	596	This will stage a temporary sysroot for running the target
...	...	@@ -607,13 +611,16 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
607	611	from .._project import HostMount
608	612	from .._pipeline import PipelineSelection
609	613
610		- if build_:
611		- scope = Scope.BUILD
612		- else:
613		- scope = Scope.RUN
	614	+ if elements and element is not None:
	615	+ command = (element,) + command
	616	+ element = None
	617	+ if not elements and element is not None:
	618	+ elements = (element,)
	619	+ if not elements:
	620	+ raise AppError('No elements specified to open a shell in')
614	621
615	622	with app.initialized():
616		- dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
	623	+ dependencies = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
617	624	element = dependencies[0]
618	625	prompt = app.shell_prompt(element)
619	626	mounts = [
...	...	@@ -621,7 +628,9 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
621	628	for host_path, path in mount
622	629	]
623	630	try:
624		- exitcode = app.stream.shell([(element, scope)], prompt,
	631	+ elements = tuple((e, Scope.BUILD if i == 0 and build_ else Scope.RUN)
	632	+ for (i, e) in enumerate(dependencies))
	633	+ exitcode = app.stream.shell(elements, prompt,
625	634	directory=sysroot,
626	635	mounts=mounts,
627	636	isolate=isolate,

buildstream/element.py

---

...	...	@@ -75,7 +75,6 @@ Class Reference
75	75	import os
76	76	import re
77	77	import stat
78		-import copy
79	78	from collections import OrderedDict
80	79	from collections.abc import Mapping
81	80	import contextlib
...	...	@@ -1371,7 +1370,7 @@ class Element(Plugin):
1371	1370	# is used to stage things by the `bst checkout` codepath
1372	1371	#
1373	1372	@contextmanager
1374		- def _prepare_sandbox(self, scope, directory, shell=False, integrate=True):
	1373	+ def _prepare_sandbox(self, scope, directory, integrate=True):
1375	1374	# bst shell and bst checkout require a local sandbox.
1376	1375	bare_directory = True if directory else False
1377	1376	with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
...	...	@@ -1382,20 +1381,15 @@ class Element(Plugin):
1382	1381
1383	1382	# Stage something if we need it
1384	1383	if not directory:
1385		- if shell and scope == Scope.BUILD:
1386		- self.stage(sandbox)
1387		- if not self.BST_STAGE_INTEGRATES:
	1384	+ # Stage deps in the sandbox root
	1385	+ with self.timed_activity("Staging dependencies", silent_nested=True):
	1386	+ self.stage_dependency_artifacts(sandbox, scope)
	1387	+
	1388	+ # Run any integration commands provided by the dependencies
	1389	+ # once they are all staged and ready
	1390	+ if integrate:
	1391	+ with self.timed_activity("Integrating sandbox"):
1388	1392	self.integrate_dependency_artifacts(sandbox, scope)
1389		- else:
1390		- # Stage deps in the sandbox root
1391		- with self.timed_activity("Staging dependencies", silent_nested=True):
1392		- self.stage_dependency_artifacts(sandbox, scope)
1393		-
1394		- # Run any integration commands provided by the dependencies
1395		- # once they are all staged and ready
1396		- if integrate:
1397		- with self.timed_activity("Integrating sandbox"):
1398		- self.integrate_dependency_artifacts(sandbox, scope)
1399	1393
1400	1394	yield sandbox
1401	1395
...	...	@@ -1874,71 +1868,6 @@ class Element(Plugin):
1874	1868	# Notify successful upload
1875	1869	return True
1876	1870
1877		- # _shell():
1878		- #
1879		- # Connects the terminal with a shell running in a staged
1880		- # environment
1881		- #
1882		- # Args:
1883		- # scope (Scope): Either BUILD or RUN scopes are valid, or None
1884		- # directory (str): A directory to an existing sandbox, or None
1885		- # mounts (list): A list of (str, str) tuples, representing host/target paths to mount
1886		- # isolate (bool): Whether to isolate the environment like we do in builds
1887		- # prompt (str): A suitable prompt string for PS1
1888		- # command (list): An argv to launch in the sandbox
1889		- #
1890		- # Returns: Exit code
1891		- #
1892		- # If directory is not specified, one will be staged using scope
1893		- def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None):
1894		-
1895		- with self._prepare_sandbox(scope, directory, shell=True) as sandbox:
1896		- environment = self.get_environment()
1897		- environment = copy.copy(environment)
1898		- flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
1899		-
1900		- # Fetch the main toplevel project, in case this is a junctioned
1901		- # subproject, we want to use the rules defined by the main one.
1902		- context = self._get_context()
1903		- project = context.get_toplevel_project()
1904		- shell_command, shell_environment, shell_host_files = project.get_shell_config()
1905		-
1906		- if prompt is not None:
1907		- environment['PS1'] = prompt
1908		-
1909		- # Special configurations for non-isolated sandboxes
1910		- if not isolate:
1911		-
1912		- # Open the network, and reuse calling uid/gid
1913		- #
1914		- flags |= SandboxFlags.NETWORK_ENABLED | SandboxFlags.INHERIT_UID
1915		-
1916		- # Apply project defined environment vars to set for a shell
1917		- for key, value in _yaml.node_items(shell_environment):
1918		- environment[key] = value
1919		-
1920		- # Setup any requested bind mounts
1921		- if mounts is None:
1922		- mounts = []
1923		-
1924		- for mount in shell_host_files + mounts:
1925		- if not os.path.exists(mount.host_path):
1926		- if not mount.optional:
1927		- self.warn("Not mounting non-existing host file: {}".format(mount.host_path))
1928		- else:
1929		- sandbox.mark_directory(mount.path)
1930		- sandbox._set_mount_source(mount.path, mount.host_path)
1931		-
1932		- if command:
1933		- argv = [arg for arg in command]
1934		- else:
1935		- argv = shell_command
1936		-
1937		- self.status("Running command", detail=" ".join(argv))
1938		-
1939		- # Run shells with network enabled and readonly root.
1940		- return sandbox.run(argv, flags, env=environment)
1941		-
1942	1871	# _open_workspace():
1943	1872	#
1944	1873	# "Open" a workspace for this element

tests/integration/project/elements/shell/adds-bar.bst

---

	1	+kind: manual
	2	+depends:
	3	+- base.bst
	4	+
	5	+config:
	6	+ install-commands:
	7	+ - |
	8	+ install -D -m775 /proc/self/fd/0 %{install-root}%{bindir}/bar <<\EOF
	9	+ #!/bin/sh
	10	+ echo bar
	11	+ EOF

tests/integration/project/elements/shell/adds-foo.bst

---

	1	+kind: manual
	2	+depends:
	3	+- base.bst
	4	+
	5	+config:
	6	+ install-commands:
	7	+ - |
	8	+ install -D -m775 /proc/self/fd/0 %{install-root}%{bindir}/foo <<\EOF
	9	+ #!/bin/sh
	10	+ echo foo
	11	+ EOF

tests/integration/project/elements/integration.bst → tests/integration/project/elements/shell/integration.bst

---

tests/integration/shell.py

---

...	...	@@ -28,11 +28,22 @@ DATA_DIR = os.path.join(
28	28	# config (dict): A project.conf dictionary to composite over the default
29	29	# mount (tuple): A (host, target) tuple for the `--mount` option
30	30	# element (str): The element to build and run a shell with
	31	+# elements (list): Other elements to build and run a shell with
31	32	# isolate (bool): Whether to pass --isolate to `bst shell`
32	33	#
33		-def execute_shell(cli, project, command, *, config=None, mount=None, element='base.bst', isolate=False):
	34	+def execute_shell(cli, project, command, *, config=None, mount=None, element=None, elements=None, isolate=False):
	35	+ assert element is None or elements is None, "Cannot mix single element and multi-element list options"
34	36	# Ensure the element is built
35		- result = cli.run(project=project, project_config=config, args=['build', element])
	37	+ if element is None and elements is None:
	38	+ element = 'base.bst'
	39	+ if elements is None:
	40	+ elements = []
	41	+
	42	+ args = ['build']
	43	+ args.extend(elements)
	44	+ if element is not None:
	45	+ args.append(element)
	46	+ result = cli.run(project=project, project_config=config, args=args)
36	47	assert result.exit_code == 0
37	48
38	49	args = ['shell']
...	...	@@ -41,7 +52,10 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element='ba
41	52	if mount is not None:
42	53	host_path, target_path = mount
43	54	args += ['--mount', host_path, target_path]
44		- args += [element, '--'] + command
	55	+ args += ["-e" + e for e in elements]
	56	+ if element is not None:
	57	+ args.append(element)
	58	+ args += ['--'] + command
45	59
46	60	return cli.run(project=project, project_config=config, args=args)
47	61
...	...	@@ -345,11 +359,56 @@ def test_sysroot(cli, tmpdir, datafiles):
345	359
346	360
347	361	# Test system integration commands can access devices in /dev
	362	+@pytest.mark.integration
348	363	@pytest.mark.datafiles(DATA_DIR)
349	364	@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
350	365	def test_integration_devices(cli, tmpdir, datafiles):
351	366	project = os.path.join(datafiles.dirname, datafiles.basename)
352		- element_name = 'integration.bst'
	367	+ element_name = 'shell/integration.bst'
353	368
354	369	result = execute_shell(cli, project, ["true"], element=element_name)
355	370	assert result.exit_code == 0
	371	+
	372	+
	373	+# Test multiple element shell
	374	+@pytest.mark.integration
	375	+@pytest.mark.datafiles(DATA_DIR)
	376	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	377	+def test_shell_multiple_elements(cli, tmpdir, datafiles):
	378	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	379	+
	380	+ result = execute_shell(cli, project, ["sh", "-c", "foo && bar"],
	381	+ elements=["shell/adds-foo.bst", "shell/adds-bar.bst"])
	382	+ assert result.exit_code == 0
	383	+
	384	+
	385	+# Test multiple element build shell
	386	+@pytest.mark.integration
	387	+@pytest.mark.datafiles(DATA_DIR)
	388	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	389	+def test_shell_multiple_workspace(cli, tmpdir, datafiles):
	390	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	391	+ elements = {'workspace/workspace-mount.bst': os.path.join(cli.directory, 'workspace-mount'),
	392	+ 'make/makehello.bst': os.path.join(cli.directory, 'makehello')}
	393	+
	394	+ for element, workspace in elements.items():
	395	+ res = cli.run(project=project, args=['workspace', 'open', element, '--directory', workspace])
	396	+ assert res.exit_code == 0
	397	+
	398	+ for workspace in elements.values():
	399	+ with open(os.path.join(workspace, "workspace-exists"), "w") as f:
	400	+ pass
	401	+
	402	+ # Ensure the dependencies of our build failing element are built
	403	+ result = cli.run(project=project, args=['build', 'base.bst', 'make/makehello.bst'])
	404	+ assert result.exit_code == 0
	405	+
	406	+ # Test that only the first workspace exists, since only the first element may be staged for build,
	407	+ # additional elements may only be staged as extra dependencies.
	408	+ args = ['shell', '--build'] + ['-e' + e for e in elements]
	409	+ args += ['--', 'sh', '-c',
	410	+ 'test -e /buildstream/test/workspace/workspace-mount.bst/workspace-exists && \
	411	+ test ! -e /buildstream/test/make/makehello.bst/workspace-exists']
	412	+ result = cli.run(project=project, args=args)
	413	+ assert result.exit_code == 0
	414	+ assert result.output == ''

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.