[Notes] [Git][BuildStream/buildstream][jonathan/debug-remote-failed-builds] 5 commits: tests: Add a test for the interactive failure shell



Title: GitLab

Jonathan Maw pushed to branch jonathan/debug-remote-failed-builds at BuildStream / buildstream

Commits:

7 changed files:

Changes:

  • NEWS
    ... ... @@ -35,6 +35,10 @@ buildstream 1.3.1
    35 35
         will now use the cached build tree. It is now *easier* to debug local build
    
    36 36
         failures, and *possible* to debug remote build failures.
    
    37 37
     
    
    38
    +  o `bst shell --sysroot` now takes any directory that contains a sysroot,
    
    39
    +    instead of just a specially-formatted build-root with a `root` and `scratch`
    
    40
    +    subdirectory.
    
    41
    +
    
    38 42
     
    
    39 43
     =================
    
    40 44
     buildstream 1.1.5
    

  • buildstream/element.py
    ... ... @@ -1317,7 +1317,9 @@ class Element(Plugin):
    1317 1317
         @contextmanager
    
    1318 1318
         def _prepare_sandbox(self, scope, directory, deps='run', integrate=True):
    
    1319 1319
             # bst shell and bst checkout require a local sandbox.
    
    1320
    -        with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False) as sandbox:
    
    1320
    +        bare_directory = True if directory else False
    
    1321
    +        with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
    
    1322
    +                            bare_directory=True if directory else False) as sandbox:
    
    1321 1323
     
    
    1322 1324
                 # Configure always comes first, and we need it.
    
    1323 1325
                 self.configure_sandbox(sandbox)
    
    ... ... @@ -2152,12 +2154,14 @@ class Element(Plugin):
    2152 2154
         #    stderr (fileobject): The stream for stderr for the sandbox
    
    2153 2155
         #    config (SandboxConfig): The SandboxConfig object
    
    2154 2156
         #    allow_remote (bool): Whether the sandbox is allowed to be remote
    
    2157
    +    #    bare_directory (bool): Whether the directory is bare i.e. doesn't have
    
    2158
    +    #                           a separate 'root' subdir
    
    2155 2159
         #
    
    2156 2160
         # Yields:
    
    2157 2161
         #    (Sandbox): A usable sandbox
    
    2158 2162
         #
    
    2159 2163
         @contextmanager
    
    2160
    -    def __sandbox(self, directory, stdout=None, stderr=None, config=None, allow_remote=True):
    
    2164
    +    def __sandbox(self, directory, stdout=None, stderr=None, config=None, allow_remote=True, bare_directory=False):
    
    2161 2165
             context = self._get_context()
    
    2162 2166
             project = self._get_project()
    
    2163 2167
             platform = Platform.get_platform()
    
    ... ... @@ -2188,6 +2192,7 @@ class Element(Plugin):
    2188 2192
                                                   stdout=stdout,
    
    2189 2193
                                                   stderr=stderr,
    
    2190 2194
                                                   config=config,
    
    2195
    +                                              bare_directory=bare_directory,
    
    2191 2196
                                                   allow_real_directory=not self.BST_VIRTUAL_DIRECTORY)
    
    2192 2197
                 yield sandbox
    
    2193 2198
     
    
    ... ... @@ -2197,7 +2202,7 @@ class Element(Plugin):
    2197 2202
     
    
    2198 2203
                 # Recursive contextmanager...
    
    2199 2204
                 with self.__sandbox(rootdir, stdout=stdout, stderr=stderr, config=config,
    
    2200
    -                                allow_remote=allow_remote) as sandbox:
    
    2205
    +                                allow_remote=allow_remote, bare_directory=False) as sandbox:
    
    2201 2206
                     yield sandbox
    
    2202 2207
     
    
    2203 2208
                 # Cleanup the build dir
    

  • buildstream/sandbox/_mount.py
    ... ... @@ -31,7 +31,6 @@ from .._fuse import SafeHardlinks
    31 31
     #
    
    32 32
     class Mount():
    
    33 33
         def __init__(self, sandbox, mount_point, safe_hardlinks, fuse_mount_options={}):
    
    34
    -        scratch_directory = sandbox._get_scratch_directory()
    
    35 34
             # Getting _get_underlying_directory() here is acceptable as
    
    36 35
             # we're part of the sandbox code. This will fail if our
    
    37 36
             # directory is CAS-based.
    
    ... ... @@ -51,6 +50,7 @@ class Mount():
    51 50
             #        a regular mount point within the parent's redirected mount.
    
    52 51
             #
    
    53 52
             if self.safe_hardlinks:
    
    53
    +            scratch_directory = sandbox._get_scratch_directory()
    
    54 54
                 # Redirected mount
    
    55 55
                 self.mount_origin = os.path.join(root_directory, mount_point.lstrip(os.sep))
    
    56 56
                 self.mount_base = os.path.join(scratch_directory, utils.url_directory_name(mount_point))
    

  • buildstream/sandbox/sandbox.py
    ... ... @@ -98,16 +98,23 @@ class Sandbox():
    98 98
             self.__config = kwargs['config']
    
    99 99
             self.__stdout = kwargs['stdout']
    
    100 100
             self.__stderr = kwargs['stderr']
    
    101
    +        self.__bare_directory = kwargs['bare_directory']
    
    101 102
     
    
    102 103
             # Setup the directories. Root and output_directory should be
    
    103 104
             # available to subclasses, hence being single-underscore. The
    
    104 105
             # others are private to this class.
    
    105
    -        self._root = os.path.join(directory, 'root')
    
    106
    +        # If the directory is bare, it probably doesn't need scratch
    
    107
    +        if self.__bare_directory:
    
    108
    +            self._root = directory
    
    109
    +            self.__scratch = None
    
    110
    +            os.makedirs(self._root, exist_ok=True)
    
    111
    +        else:
    
    112
    +            self._root = os.path.join(directory, 'root')
    
    113
    +            self.__scratch = os.path.join(directory, 'scratch')
    
    114
    +            for directory_ in [self._root, self.__scratch]:
    
    115
    +                os.makedirs(directory_, exist_ok=True)
    
    116
    +
    
    106 117
             self._output_directory = None
    
    107
    -        self.__directory = directory
    
    108
    -        self.__scratch = os.path.join(self.__directory, 'scratch')
    
    109
    -        for directory_ in [self._root, self.__scratch]:
    
    110
    -            os.makedirs(directory_, exist_ok=True)
    
    111 118
             self._vdir = None
    
    112 119
     
    
    113 120
             # This is set if anyone requests access to the underlying
    
    ... ... @@ -334,6 +341,7 @@ class Sandbox():
    334 341
         # Returns:
    
    335 342
         #    (str): The sandbox scratch directory
    
    336 343
         def _get_scratch_directory(self):
    
    344
    +        assert not self.__bare_directory, "Scratch is not going to work with bare directories"
    
    337 345
             return self.__scratch
    
    338 346
     
    
    339 347
         # _get_output()
    

  • setup.py
    ... ... @@ -303,6 +303,7 @@ setup(name='BuildStream',
    303 303
               'jinja2 >= 2.10',
    
    304 304
               'protobuf >= 3.5',
    
    305 305
               'grpcio >= 1.10',
    
    306
    +          'pexpect',
    
    306 307
           ],
    
    307 308
           entry_points=bst_install_entry_points,
    
    308 309
           setup_requires=['pytest-runner'],
    

  • tests/integration/build-tree.py
    1 1
     import os
    
    2 2
     import pytest
    
    3 3
     import shutil
    
    4
    +import pexpect
    
    4 5
     
    
    5 6
     from tests.testutils import cli, cli_integration, create_artifact_share
    
    6 7
     
    
    ... ... @@ -29,6 +30,31 @@ def test_buildtree_staged(cli_integration, tmpdir, datafiles):
    29 30
         res.assert_success()
    
    30 31
     
    
    31 32
     
    
    33
    +# Check that a failed build causes a shell that stages the buildtree
    
    34
    +@pytest.mark.datafiles(DATA_DIR)
    
    35
    +def test_buildtree_staged_interactive(cli_integration, tmpdir, datafiles):
    
    36
    +    # i.e. tests that cached build trees are staged by `bst shell --build`
    
    37
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    38
    +    element_name = 'build-shell/buildtree-fail.bst'
    
    39
    +
    
    40
    +    # Start an interactive session and intercept the interactive prompt
    
    41
    +    child = pexpect.spawn('bst', ['--no-colors', 'build', element_name], cwd=project)
    
    42
    +
    
    43
    +    # Ensure starting a shell is a valid option
    
    44
    +    child.expect('\(s\)hell')
    
    45
    +
    
    46
    +    # Wait for the interactive prompt
    
    47
    +    child.expect('Choice: \[continue\]:')
    
    48
    +
    
    49
    +    # Start a shell and wait for the shell's prompt
    
    50
    +    child.sendline('s')
    
    51
    +    child.expect('\]\$')
    
    52
    +
    
    53
    +    # Read the contents of the file we added
    
    54
    +    child.sendline('cat test')
    
    55
    +    child.expect('Hi')
    
    56
    +
    
    57
    +
    
    32 58
     # Check that build shells work when pulled from a remote cache
    
    33 59
     # This is to roughly simulate remote execution
    
    34 60
     @pytest.mark.datafiles(DATA_DIR)
    

  • tests/integration/shell.py
    ... ... @@ -302,6 +302,35 @@ def test_workspace_visible(cli, tmpdir, datafiles):
    302 302
         assert result.output == workspace_hello
    
    303 303
     
    
    304 304
     
    
    305
    +# Test that '--sysroot' works
    
    306
    +@pytest.mark.datafiles(DATA_DIR)
    
    307
    +def test_sysroot(cli, tmpdir, datafiles):
    
    308
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    309
    +    base_element = "base/base-alpine.bst"
    
    310
    +    # test element only needs to be something lightweight for this test
    
    311
    +    test_element = "script/script.bst"
    
    312
    +    checkout_dir = os.path.join(str(tmpdir), 'alpine-sysroot')
    
    313
    +    test_file = 'hello'
    
    314
    +
    
    315
    +    # Build and check out a sysroot
    
    316
    +    res = cli.run(project=project, args=['build', base_element])
    
    317
    +    res.assert_success()
    
    318
    +    res = cli.run(project=project, args=['checkout', base_element, checkout_dir])
    
    319
    +    res.assert_success()
    
    320
    +
    
    321
    +    # Mutate the sysroot
    
    322
    +    test_path = os.path.join(checkout_dir, test_file)
    
    323
    +    with open(test_path, 'w') as f:
    
    324
    +        f.write('hello\n')
    
    325
    +
    
    326
    +    # Shell into the sysroot and check the test file exists
    
    327
    +    res = cli.run(project=project, args=[
    
    328
    +        'shell', '--build', '--sysroot', checkout_dir, test_element, '--',
    
    329
    +        'grep', '-q', 'hello', '/' + test_file
    
    330
    +    ])
    
    331
    +    res.assert_success()
    
    332
    +
    
    333
    +
    
    305 334
     # Test system integration commands can access devices in /dev
    
    306 335
     @pytest.mark.datafiles(DATA_DIR)
    
    307 336
     def test_integration_devices(cli, tmpdir, datafiles):
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]