[Notes] [Git][BuildStream/buildstream][jonathan/debug-remote-failed-buil

Jonathan Maw pushed to branch jonathan/debug-remote-failed-builds at BuildStream / buildstream

Commits:

15a36546

by Jonathan Maw at 2018-10-09T16:38:35Z

element.py: Always clean up the rootdir

We shouldn't need it to persist now that we cache failed build dirs.

This change breaks the test
`tests/integration/shell.py::test_sysroot_workspace_visible`.
I can no longer see a use-case for this test.
AIUI, it tested that the failed build sysroot stored in the builddir has
the workspace's files in, despite the workspace being unmounted.
I believe this behaviour is made redundant by cached buildtrees.

99bcc690

by Jonathan Maw at 2018-10-09T16:39:50Z

Element: Use cached buildtree in build shells and failure shells

This includes changes in app.py:
* Interactive failure shell no longer uses the failed build sysroot,
  defaulting to the cached build tree.
* Always allow producing a failure shell.

Changes in element.py are:
* Errors caused by building don't store the failed build sysroot.
* When staging sources, will stage the element's cached build tree if it
  exists.

fec65e22

by Jonathan Maw at 2018-10-09T16:39:50Z

tests: Add test that cached build trees are staged in build shells

5 changed files:

buildstream/_frontend/app.py
buildstream/element.py
+ tests/integration/build-tree.py
+ tests/integration/project/elements/build-shell/buildtree.bst
tests/integration/shell.py

Changes:

buildstream/_frontend/app.py

@@ -564,18 +564,15 @@ class App():
                         "  (c)ontinue  - Continue queueing jobs as much as possible\n" +
                         "  (q)uit      - Exit after all ongoing jobs complete\n" +
                         "  (t)erminate - Terminate any ongoing jobs and exit\n" +
 -                       "  (r)etry     - Retry this job\n")
 +                       "  (r)etry     - Retry this job\n" +
 +                       "  (s)hell     - Drop into a shell in the failed build sandbox\n")
              if failure.logfile:
                  summary += "  (l)og       - View the full log file\n"
 -            if failure.sandbox:
 -                summary += "  (s)hell     - Drop into a shell in the failed build sandbox\n"
              summary += "\nPressing ^C will terminate jobs and exit\n"
 -            choices = ['continue', 'quit', 'terminate', 'retry']
 +            choices = ['continue', 'quit', 'terminate', 'retry', 'shell']
              if failure.logfile:
                  choices += ['log']
 -            if failure.sandbox:
 -                choices += ['shell']
              choice = ''
              while choice not in ['continue', 'quit', 'terminate', 'retry']:
@@ -595,10 +592,10 @@ class App():
                  # Handle choices which you can come back from
+                 #
                  if choice == 'shell':
 -                    click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True)
 +                    click.echo("\nDropping into an interactive shell in the failed build tree\n", err=True)
                      try:
                          prompt = self.shell_prompt(element)
 -                        self.stream.shell(element, Scope.BUILD, prompt, directory=failure.sandbox, isolate=True)
 +                        self.stream.shell(element, Scope.BUILD, prompt, isolate=True)
                      except BstError as e:
                          click.echo("Error while attempting to create interactive shell: {}".format(e), err=True)
                  elif choice == 'log':

buildstream/element.py

@@ -1384,6 +1384,7 @@ class Element(Plugin):
              # the same filing system as the rest of our cache.
              temp_staging_location = os.path.join(self._get_context().artifactdir, "staging_temp")
              temp_staging_directory = tempfile.mkdtemp(prefix=temp_staging_location)
 +            import_dir = temp_staging_directory
              try:
                  workspace = self._get_workspace()
@@ -1394,12 +1395,16 @@ class Element(Plugin):
                          with self.timed_activity("Staging local files at {}"
                                                   .format(workspace.get_absolute_path())):
                              workspace.stage(temp_staging_directory)
 +                elif self._cached():
 +                    # We have a cached buildtree to use, instead
 +                    artifact_base, _ = self.__extract()
 +                    import_dir = os.path.join(artifact_base, 'buildtree')
                  else:
                      # No workspace, stage directly
                      for source in self.sources():
                          source._stage(temp_staging_directory)
 -                vdirectory.import_files(temp_staging_directory)
 +                vdirectory.import_files(import_dir)
              finally:
                  # Staging may produce directories with less than 'rwx' permissions
@@ -1565,10 +1570,6 @@ class Element(Plugin):
                      collect = self.assemble(sandbox)
                      self.__set_build_result(success=True, description="succeeded")
                  except BstError as e:
 -                    # If an error occurred assembling an element in a sandbox,
 -                    # then tack on the sandbox directory to the error
 -                    e.sandbox = rootdir
+-
                      # If there is a workspace open on this element, it will have
                      # been mounted for sandbox invocations instead of being staged.
+                     #
@@ -1682,8 +1683,8 @@ class Element(Plugin):
                              "unable to collect artifact contents"
                              .format(collect))
 -            # Finally cleanup the build dir
 -            cleanup_rootdir()
 +                    # Finally cleanup the build dir
 +                    cleanup_rootdir()
          return artifact_size

tests/integration/build-tree.py

 +import os
 +import pytest
++
 +from tests.testutils import cli_integration as cli
++
++
 +pytestmark = pytest.mark.integration
++
++
 +DATA_DIR = os.path.join(
 +    os.path.dirname(os.path.realpath(__file__)),
 +    "project"
 +)
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_buildtree_staged(cli, tmpdir, datafiles):
 +    # i.e. tests that cached build trees are staged by `bst shell --build`
 +    project = os.path.join(datafiles.dirname, datafiles.basename)
 +    element_path = os.path.join(project, 'elements')
 +    element_name = 'build-shell/buildtree.bst'
++
 +    res = cli.run(project=project, args=['build', element_name])
 +    res.assert_success()
++
 +    res = cli.run(project=project, args=['shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'])
 +    res.assert_success()

tests/integration/project/elements/build-shell/buildtree.bst

 +kind: manual
 +description: |
 +  Puts a file in the build tree so that build tree caching and staging can be tested.
++
 +depends:
 +  - filename: base.bst
 +    type: build
++
 +config:
 +  build-commands:
 +    - "echo 'Hi' > %{build-root}/test"

tests/integration/shell.py

@@ -302,48 +302,6 @@ def test_workspace_visible(cli, tmpdir, datafiles):
      assert result.output == workspace_hello
 -# Test that we can see the workspace files in a shell
 -@pytest.mark.integration
 -@pytest.mark.datafiles(DATA_DIR)
 -def test_sysroot_workspace_visible(cli, tmpdir, datafiles):
 -    project = os.path.join(datafiles.dirname, datafiles.basename)
 -    workspace = os.path.join(cli.directory, 'workspace')
 -    element_name = 'workspace/workspace-mount-fail.bst'
+-
 -    # Open a workspace on our build failing element
 -    #
 -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
 -    assert res.exit_code == 0
+-
 -    # Ensure the dependencies of our build failing element are built
 -    result = cli.run(project=project, args=['build', element_name])
 -    result.assert_main_error(ErrorDomain.STREAM, None)
+-
 -    # Discover the sysroot of the failed build directory, after one
 -    # failed build, there should be only one directory there.
 -    #
 -    build_base = os.path.join(cli.directory, 'build')
 -    build_dirs = os.listdir(path=build_base)
 -    assert len(build_dirs) == 1
 -    build_dir = os.path.join(build_base, build_dirs[0])
+-
 -    # Obtain a copy of the hello.c content from the workspace
 -    #
 -    workspace_hello_path = os.path.join(cli.directory, 'workspace', 'hello.c')
 -    assert os.path.exists(workspace_hello_path)
 -    with open(workspace_hello_path, 'r') as f:
 -        workspace_hello = f.read()
+-
 -    # Cat the hello.c file from a bst shell command, and assert
 -    # that we got the same content here
 -    #
 -    result = cli.run(project=project, args=[
 -        'shell', '--build', '--sysroot', build_dir, element_name, '--', 'cat', 'hello.c'
 -    ])
 -    assert result.exit_code == 0
 -    assert result.output == workspace_hello
+-
+-
  # Test system integration commands can access devices in /dev
  @pytest.mark.datafiles(DATA_DIR)
  def test_integration_devices(cli, tmpdir, datafiles):

[Notes] [Git][BuildStream/buildstream][jonathan/debug-remote-failed-builds] 3 commits: element.py: Always clean up the rootdir

Jonathan Maw pushed to branch jonathan/debug-remote-failed-builds at BuildStream / buildstream

Commits:

5 changed files:

Changes: