[Notes] [Git][BuildStream/buildstream][jonathan/debug-remote-failed-builds] 3 commits: element.py: Always clean up the rootdir

[Jonathan Maw <gitlab mg gitlab com>]

Title: GitLab

Jonathan Maw pushed to branch jonathan/debug-remote-failed-builds at BuildStream / buildstream

Commits:

• 446f6c64
  by Jonathan Maw at 2018-10-10T14:14:02Z

  element.py: Always clean up the rootdir
  
  We shouldn't need it to persist now that we cache failed build dirs.
  
  This change breaks the test
  `tests/integration/shell.py::test_sysroot_workspace_visible`.
  I can no longer see a use-case for this test.
  AIUI, it tested that the failed build sysroot stored in the builddir has
  the workspace's files in, despite the workspace being unmounted.
  I believe this behaviour is made redundant by cached buildtrees.
  
  This fixes part of #539
  

• bcd8c963
  by Jonathan Maw at 2018-10-10T14:14:48Z

  Element: Use cached buildtree in build shells and failure shells
  
  This includes changes in app.py:
  * Interactive failure shell no longer uses the failed build sysroot,
    defaulting to the cached build tree.
  * Always allow producing a failure shell.
  
  Changes in element.py are:
  * Errors caused by building don't store the failed build sysroot.
  * When staging sources, will stage the element's cached build tree if it
    exists.
  
  This fixes part of #539
  

• 2e8bf330
  by Jonathan Maw at 2018-10-10T14:14:56Z

  tests: Add test that cached build trees are staged in build shells
  
  This is related to #539
  

5 changed files:

• buildstream/_frontend/app.py
• buildstream/element.py
• + tests/integration/build-tree.py
• + tests/integration/project/elements/build-shell/buildtree.bst
• tests/integration/shell.py

Changes:

buildstream/_frontend/app.py

---

...	...	@@ -564,18 +564,15 @@ class App():
564	564	" (c)ontinue - Continue queueing jobs as much as possible\n" +
565	565	" (q)uit - Exit after all ongoing jobs complete\n" +
566	566	" (t)erminate - Terminate any ongoing jobs and exit\n" +
567		- " (r)etry - Retry this job\n")
	567	+ " (r)etry - Retry this job\n" +
	568	+ " (s)hell - Drop into a shell in the failed build sandbox\n")
568	569	if failure.logfile:
569	570	summary += " (l)og - View the full log file\n"
570		- if failure.sandbox:
571		- summary += " (s)hell - Drop into a shell in the failed build sandbox\n"
572	571	summary += "\nPressing ^C will terminate jobs and exit\n"
573	572
574		- choices = ['continue', 'quit', 'terminate', 'retry']
	573	+ choices = ['continue', 'quit', 'terminate', 'retry', 'shell']
575	574	if failure.logfile:
576	575	choices += ['log']
577		- if failure.sandbox:
578		- choices += ['shell']
579	576
580	577	choice = ''
581	578	while choice not in ['continue', 'quit', 'terminate', 'retry']:
...	...	@@ -595,10 +592,10 @@ class App():
595	592	# Handle choices which you can come back from
596	593	#
597	594	if choice == 'shell':
598		- click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True)
	595	+ click.echo("\nDropping into an interactive shell in the failed build tree\n", err=True)
599	596	try:
600	597	prompt = self.shell_prompt(element)
601		- self.stream.shell(element, Scope.BUILD, prompt, directory=failure.sandbox, isolate=True)
	598	+ self.stream.shell(element, Scope.BUILD, prompt, isolate=True)
602	599	except BstError as e:
603	600	click.echo("Error while attempting to create interactive shell: {}".format(e), err=True)
604	601	elif choice == 'log':

buildstream/element.py

---

...	...	@@ -1384,6 +1384,7 @@ class Element(Plugin):
1384	1384	# the same filing system as the rest of our cache.
1385	1385	temp_staging_location = os.path.join(self._get_context().artifactdir, "staging_temp")
1386	1386	temp_staging_directory = tempfile.mkdtemp(prefix=temp_staging_location)
	1387	+ import_dir = temp_staging_directory
1387	1388
1388	1389	try:
1389	1390	workspace = self._get_workspace()
...	...	@@ -1394,12 +1395,16 @@ class Element(Plugin):
1394	1395	with self.timed_activity("Staging local files at {}"
1395	1396	.format(workspace.get_absolute_path())):
1396	1397	workspace.stage(temp_staging_directory)
	1398	+ elif self._cached():
	1399	+ # We have a cached buildtree to use, instead
	1400	+ artifact_base, _ = self.__extract()
	1401	+ import_dir = os.path.join(artifact_base, 'buildtree')
1397	1402	else:
1398	1403	# No workspace, stage directly
1399	1404	for source in self.sources():
1400	1405	source._stage(temp_staging_directory)
1401	1406
1402		- vdirectory.import_files(temp_staging_directory)
	1407	+ vdirectory.import_files(import_dir)
1403	1408
1404	1409	finally:
1405	1410	# Staging may produce directories with less than 'rwx' permissions
...	...	@@ -1565,10 +1570,6 @@ class Element(Plugin):
1565	1570	collect = self.assemble(sandbox)
1566	1571	self.__set_build_result(success=True, description="succeeded")
1567	1572	except BstError as e:
1568		- # If an error occurred assembling an element in a sandbox,
1569		- # then tack on the sandbox directory to the error
1570		- e.sandbox = rootdir
1571		-
1572	1573	# If there is a workspace open on this element, it will have
1573	1574	# been mounted for sandbox invocations instead of being staged.
1574	1575	#
...	...	@@ -1682,8 +1683,8 @@ class Element(Plugin):
1682	1683	"unable to collect artifact contents"
1683	1684	.format(collect))
1684	1685
1685		- # Finally cleanup the build dir
1686		- cleanup_rootdir()
	1686	+ # Finally cleanup the build dir
	1687	+ cleanup_rootdir()
1687	1688
1688	1689	return artifact_size
1689	1690

tests/integration/build-tree.py

---

	1	+import os
	2	+import pytest
	3	+
	4	+from tests.testutils import cli_integration as cli
	5	+
	6	+
	7	+pytestmark = pytest.mark.integration
	8	+
	9	+
	10	+DATA_DIR = os.path.join(
	11	+ os.path.dirname(os.path.realpath(__file__)),
	12	+ "project"
	13	+)
	14	+
	15	+
	16	+@pytest.mark.datafiles(DATA_DIR)
	17	+def test_buildtree_staged(cli, tmpdir, datafiles):
	18	+ # i.e. tests that cached build trees are staged by `bst shell --build`
	19	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	20	+ element_path = os.path.join(project, 'elements')
	21	+ element_name = 'build-shell/buildtree.bst'
	22	+
	23	+ res = cli.run(project=project, args=['build', element_name])
	24	+ res.assert_success()
	25	+
	26	+ res = cli.run(project=project, args=['shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'])
	27	+ res.assert_success()

tests/integration/project/elements/build-shell/buildtree.bst

---

	1	+kind: manual
	2	+description: |
	3	+ Puts a file in the build tree so that build tree caching and staging can be tested.
	4	+
	5	+depends:
	6	+ - filename: base.bst
	7	+ type: build
	8	+
	9	+config:
	10	+ build-commands:
	11	+ - "echo 'Hi' > %{build-root}/test"

tests/integration/shell.py

---

...	...	@@ -302,48 +302,6 @@ def test_workspace_visible(cli, tmpdir, datafiles):
302	302	assert result.output == workspace_hello
303	303
304	304
305		-# Test that we can see the workspace files in a shell
306		-@pytest.mark.integration
307		-@pytest.mark.datafiles(DATA_DIR)
308		-def test_sysroot_workspace_visible(cli, tmpdir, datafiles):
309		- project = os.path.join(datafiles.dirname, datafiles.basename)
310		- workspace = os.path.join(cli.directory, 'workspace')
311		- element_name = 'workspace/workspace-mount-fail.bst'
312		-
313		- # Open a workspace on our build failing element
314		- #
315		- res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
316		- assert res.exit_code == 0
317		-
318		- # Ensure the dependencies of our build failing element are built
319		- result = cli.run(project=project, args=['build', element_name])
320		- result.assert_main_error(ErrorDomain.STREAM, None)
321		-
322		- # Discover the sysroot of the failed build directory, after one
323		- # failed build, there should be only one directory there.
324		- #
325		- build_base = os.path.join(cli.directory, 'build')
326		- build_dirs = os.listdir(path=build_base)
327		- assert len(build_dirs) == 1
328		- build_dir = os.path.join(build_base, build_dirs[0])
329		-
330		- # Obtain a copy of the hello.c content from the workspace
331		- #
332		- workspace_hello_path = os.path.join(cli.directory, 'workspace', 'hello.c')
333		- assert os.path.exists(workspace_hello_path)
334		- with open(workspace_hello_path, 'r') as f:
335		- workspace_hello = f.read()
336		-
337		- # Cat the hello.c file from a bst shell command, and assert
338		- # that we got the same content here
339		- #
340		- result = cli.run(project=project, args=[
341		- 'shell', '--build', '--sysroot', build_dir, element_name, '--', 'cat', 'hello.c'
342		- ])
343		- assert result.exit_code == 0
344		- assert result.output == workspace_hello
345		-
346		-
347	305	# Test system integration commands can access devices in /dev
348	306	@pytest.mark.datafiles(DATA_DIR)
349	307	def test_integration_devices(cli, tmpdir, datafiles):

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.