Jonathan Maw pushed to branch jonathan/debug-remote-failed-builds at BuildStream / buildstream
Commits:
-
1088de96
by Jonathan Maw at 2018-10-29T17:38:13Z
-
5621bc49
by Jonathan Maw at 2018-10-29T17:38:14Z
-
5d7b9a47
by Jonathan Maw at 2018-10-29T17:38:14Z
-
726c01cd
by Jonathan Maw at 2018-10-29T17:38:14Z
-
d79ca68e
by Jonathan Maw at 2018-10-29T17:38:14Z
8 changed files:
- NEWS
- buildstream/element.py
- buildstream/sandbox/_mount.py
- buildstream/sandbox/sandbox.py
- + tests/integration/build-tree.py
- + tests/integration/project/elements/build-shell/buildtree-fail.bst
- + tests/integration/project/elements/build-shell/buildtree.bst
- tests/integration/shell.py
Changes:
... | ... | @@ -35,6 +35,10 @@ buildstream 1.3.1 |
35 | 35 |
will now use the cached build tree. It is now easier to debug local build
|
36 | 36 |
failures.
|
37 | 37 |
|
38 |
+ o `bst shell --sysroot` now takes any directory that contains a sysroot,
|
|
39 |
+ instead of just a specially-formatted build-root with a `root` and `scratch`
|
|
40 |
+ subdirectory.
|
|
41 |
+ |
|
38 | 42 |
|
39 | 43 |
=================
|
40 | 44 |
buildstream 1.1.5
|
... | ... | @@ -1318,7 +1318,9 @@ class Element(Plugin): |
1318 | 1318 |
@contextmanager
|
1319 | 1319 |
def _prepare_sandbox(self, scope, directory, deps='run', integrate=True):
|
1320 | 1320 |
# bst shell and bst checkout require a local sandbox.
|
1321 |
- with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False) as sandbox:
|
|
1321 |
+ bare_directory = True if directory else False
|
|
1322 |
+ with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
|
|
1323 |
+ bare_directory=bare_directory) as sandbox:
|
|
1322 | 1324 |
|
1323 | 1325 |
# Configure always comes first, and we need it.
|
1324 | 1326 |
self.configure_sandbox(sandbox)
|
... | ... | @@ -2156,12 +2158,14 @@ class Element(Plugin): |
2156 | 2158 |
# stderr (fileobject): The stream for stderr for the sandbox
|
2157 | 2159 |
# config (SandboxConfig): The SandboxConfig object
|
2158 | 2160 |
# allow_remote (bool): Whether the sandbox is allowed to be remote
|
2161 |
+ # bare_directory (bool): Whether the directory is bare i.e. doesn't have
|
|
2162 |
+ # a separate 'root' subdir
|
|
2159 | 2163 |
#
|
2160 | 2164 |
# Yields:
|
2161 | 2165 |
# (Sandbox): A usable sandbox
|
2162 | 2166 |
#
|
2163 | 2167 |
@contextmanager
|
2164 |
- def __sandbox(self, directory, stdout=None, stderr=None, config=None, allow_remote=True):
|
|
2168 |
+ def __sandbox(self, directory, stdout=None, stderr=None, config=None, allow_remote=True, bare_directory=False):
|
|
2165 | 2169 |
context = self._get_context()
|
2166 | 2170 |
project = self._get_project()
|
2167 | 2171 |
platform = Platform.get_platform()
|
... | ... | @@ -2192,6 +2196,7 @@ class Element(Plugin): |
2192 | 2196 |
stdout=stdout,
|
2193 | 2197 |
stderr=stderr,
|
2194 | 2198 |
config=config,
|
2199 |
+ bare_directory=bare_directory,
|
|
2195 | 2200 |
allow_real_directory=not self.BST_VIRTUAL_DIRECTORY)
|
2196 | 2201 |
yield sandbox
|
2197 | 2202 |
|
... | ... | @@ -2201,7 +2206,7 @@ class Element(Plugin): |
2201 | 2206 |
|
2202 | 2207 |
# Recursive contextmanager...
|
2203 | 2208 |
with self.__sandbox(rootdir, stdout=stdout, stderr=stderr, config=config,
|
2204 |
- allow_remote=allow_remote) as sandbox:
|
|
2209 |
+ allow_remote=allow_remote, bare_directory=False) as sandbox:
|
|
2205 | 2210 |
yield sandbox
|
2206 | 2211 |
|
2207 | 2212 |
# Cleanup the build dir
|
... | ... | @@ -31,7 +31,6 @@ from .._fuse import SafeHardlinks |
31 | 31 |
#
|
32 | 32 |
class Mount():
|
33 | 33 |
def __init__(self, sandbox, mount_point, safe_hardlinks, fuse_mount_options=None):
|
34 |
- scratch_directory = sandbox._get_scratch_directory()
|
|
35 | 34 |
# Getting _get_underlying_directory() here is acceptable as
|
36 | 35 |
# we're part of the sandbox code. This will fail if our
|
37 | 36 |
# directory is CAS-based.
|
... | ... | @@ -51,6 +50,7 @@ class Mount(): |
51 | 50 |
# a regular mount point within the parent's redirected mount.
|
52 | 51 |
#
|
53 | 52 |
if self.safe_hardlinks:
|
53 |
+ scratch_directory = sandbox._get_scratch_directory()
|
|
54 | 54 |
# Redirected mount
|
55 | 55 |
self.mount_origin = os.path.join(root_directory, mount_point.lstrip(os.sep))
|
56 | 56 |
self.mount_base = os.path.join(scratch_directory, utils.url_directory_name(mount_point))
|
... | ... | @@ -98,16 +98,23 @@ class Sandbox(): |
98 | 98 |
self.__config = kwargs['config']
|
99 | 99 |
self.__stdout = kwargs['stdout']
|
100 | 100 |
self.__stderr = kwargs['stderr']
|
101 |
+ self.__bare_directory = kwargs['bare_directory']
|
|
101 | 102 |
|
102 | 103 |
# Setup the directories. Root and output_directory should be
|
103 | 104 |
# available to subclasses, hence being single-underscore. The
|
104 | 105 |
# others are private to this class.
|
105 |
- self._root = os.path.join(directory, 'root')
|
|
106 |
+ # If the directory is bare, it probably doesn't need scratch
|
|
107 |
+ if self.__bare_directory:
|
|
108 |
+ self._root = directory
|
|
109 |
+ self.__scratch = None
|
|
110 |
+ os.makedirs(self._root, exist_ok=True)
|
|
111 |
+ else:
|
|
112 |
+ self._root = os.path.join(directory, 'root')
|
|
113 |
+ self.__scratch = os.path.join(directory, 'scratch')
|
|
114 |
+ for directory_ in [self._root, self.__scratch]:
|
|
115 |
+ os.makedirs(directory_, exist_ok=True)
|
|
116 |
+ |
|
106 | 117 |
self._output_directory = None
|
107 |
- self.__directory = directory
|
|
108 |
- self.__scratch = os.path.join(self.__directory, 'scratch')
|
|
109 |
- for directory_ in [self._root, self.__scratch]:
|
|
110 |
- os.makedirs(directory_, exist_ok=True)
|
|
111 | 118 |
self._vdir = None
|
112 | 119 |
|
113 | 120 |
# This is set if anyone requests access to the underlying
|
... | ... | @@ -334,6 +341,7 @@ class Sandbox(): |
334 | 341 |
# Returns:
|
335 | 342 |
# (str): The sandbox scratch directory
|
336 | 343 |
def _get_scratch_directory(self):
|
344 |
+ assert not self.__bare_directory, "Scratch is not going to work with bare directories"
|
|
337 | 345 |
return self.__scratch
|
338 | 346 |
|
339 | 347 |
# _get_output()
|
1 |
+import os
|
|
2 |
+import pytest
|
|
3 |
+import shutil
|
|
4 |
+ |
|
5 |
+from tests.testutils import cli, cli_integration, create_artifact_share
|
|
6 |
+from buildstream._exceptions import ErrorDomain
|
|
7 |
+ |
|
8 |
+ |
|
9 |
+pytestmark = pytest.mark.integration
|
|
10 |
+ |
|
11 |
+ |
|
12 |
+DATA_DIR = os.path.join(
|
|
13 |
+ os.path.dirname(os.path.realpath(__file__)),
|
|
14 |
+ "project"
|
|
15 |
+)
|
|
16 |
+ |
|
17 |
+ |
|
18 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
19 |
+def test_buildtree_staged(cli_integration, tmpdir, datafiles):
|
|
20 |
+ # i.e. tests that cached build trees are staged by `bst shell --build`
|
|
21 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
22 |
+ element_name = 'build-shell/buildtree.bst'
|
|
23 |
+ |
|
24 |
+ res = cli_integration.run(project=project, args=['build', element_name])
|
|
25 |
+ res.assert_success()
|
|
26 |
+ |
|
27 |
+ res = cli_integration.run(project=project, args=[
|
|
28 |
+ 'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
|
|
29 |
+ ])
|
|
30 |
+ res.assert_success()
|
|
31 |
+ |
|
32 |
+ |
|
33 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
34 |
+def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
|
|
35 |
+ # i.e. test that on a build failure, we can still shell into it
|
|
36 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
37 |
+ element_name = 'build-shell/buildtree-fail.bst'
|
|
38 |
+ |
|
39 |
+ res = cli_integration.run(project=project, args=['build', element_name])
|
|
40 |
+ res.assert_main_error(ErrorDomain.STREAM, None)
|
|
41 |
+ |
|
42 |
+ # Assert that file has expected contents
|
|
43 |
+ res = cli_integration.run(project=project, args=[
|
|
44 |
+ 'shell', '--build', element_name, '--', 'cat', 'test'
|
|
45 |
+ ])
|
|
46 |
+ res.assert_success()
|
|
47 |
+ assert 'Hi' in res.output
|
|
48 |
+ |
|
49 |
+ |
|
50 |
+# Check that build shells work when pulled from a remote cache
|
|
51 |
+# This is to roughly simulate remote execution
|
|
52 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
53 |
+def test_buildtree_pulled(cli, tmpdir, datafiles):
|
|
54 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
55 |
+ element_name = 'build-shell/buildtree.bst'
|
|
56 |
+ |
|
57 |
+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
|
|
58 |
+ # Build the element to push it to cache
|
|
59 |
+ cli.configure({
|
|
60 |
+ 'artifacts': {'url': share.repo, 'push': True}
|
|
61 |
+ })
|
|
62 |
+ result = cli.run(project=project, args=['build', element_name])
|
|
63 |
+ result.assert_success()
|
|
64 |
+ assert cli.get_element_state(project, element_name) == 'cached'
|
|
65 |
+ |
|
66 |
+ # Discard the cache
|
|
67 |
+ cli.configure({
|
|
68 |
+ 'artifacts': {'url': share.repo, 'push': True},
|
|
69 |
+ 'artifactdir': os.path.join(cli.directory, 'artifacts2')
|
|
70 |
+ })
|
|
71 |
+ assert cli.get_element_state(project, element_name) != 'cached'
|
|
72 |
+ |
|
73 |
+ # Pull from cache
|
|
74 |
+ result = cli.run(project=project, args=['pull', '--deps', 'all', element_name])
|
|
75 |
+ result.assert_success()
|
|
76 |
+ |
|
77 |
+ # Check it's using the cached build tree
|
|
78 |
+ res = cli.run(project=project, args=[
|
|
79 |
+ 'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
|
|
80 |
+ ])
|
|
81 |
+ res.assert_success()
|
1 |
+kind: manual
|
|
2 |
+description: |
|
|
3 |
+ Puts a file in the build tree so that build tree caching and staging can be tested,
|
|
4 |
+ then deliberately failing to build so we can check the output.
|
|
5 |
+ |
|
6 |
+depends:
|
|
7 |
+ - filename: base.bst
|
|
8 |
+ type: build
|
|
9 |
+ |
|
10 |
+config:
|
|
11 |
+ build-commands:
|
|
12 |
+ - "echo 'Hi' > %{build-root}/test"
|
|
13 |
+ - "false"
|
1 |
+kind: manual
|
|
2 |
+description: |
|
|
3 |
+ Puts a file in the build tree so that build tree caching and staging can be tested.
|
|
4 |
+ |
|
5 |
+depends:
|
|
6 |
+ - filename: base.bst
|
|
7 |
+ type: build
|
|
8 |
+ |
|
9 |
+config:
|
|
10 |
+ build-commands:
|
|
11 |
+ - "echo 'Hi' > %{build-root}/test"
|
... | ... | @@ -302,6 +302,35 @@ def test_workspace_visible(cli, tmpdir, datafiles): |
302 | 302 |
assert result.output == workspace_hello
|
303 | 303 |
|
304 | 304 |
|
305 |
+# Test that '--sysroot' works
|
|
306 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
307 |
+def test_sysroot(cli, tmpdir, datafiles):
|
|
308 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
309 |
+ base_element = "base/base-alpine.bst"
|
|
310 |
+ # test element only needs to be something lightweight for this test
|
|
311 |
+ test_element = "script/script.bst"
|
|
312 |
+ checkout_dir = os.path.join(str(tmpdir), 'alpine-sysroot')
|
|
313 |
+ test_file = 'hello'
|
|
314 |
+ |
|
315 |
+ # Build and check out a sysroot
|
|
316 |
+ res = cli.run(project=project, args=['build', base_element])
|
|
317 |
+ res.assert_success()
|
|
318 |
+ res = cli.run(project=project, args=['checkout', base_element, checkout_dir])
|
|
319 |
+ res.assert_success()
|
|
320 |
+ |
|
321 |
+ # Mutate the sysroot
|
|
322 |
+ test_path = os.path.join(checkout_dir, test_file)
|
|
323 |
+ with open(test_path, 'w') as f:
|
|
324 |
+ f.write('hello\n')
|
|
325 |
+ |
|
326 |
+ # Shell into the sysroot and check the test file exists
|
|
327 |
+ res = cli.run(project=project, args=[
|
|
328 |
+ 'shell', '--build', '--sysroot', checkout_dir, test_element, '--',
|
|
329 |
+ 'grep', '-q', 'hello', '/' + test_file
|
|
330 |
+ ])
|
|
331 |
+ res.assert_success()
|
|
332 |
+ |
|
333 |
+ |
|
305 | 334 |
# Test system integration commands can access devices in /dev
|
306 | 335 |
@pytest.mark.datafiles(DATA_DIR)
|
307 | 336 |
def test_integration_devices(cli, tmpdir, datafiles):
|