[Notes] [Git][BuildStream/buildstream][willsalmon/shellBuildTrees] Basic options for shell --build to use buildtrees



Title: GitLab

Will Salmon pushed to branch willsalmon/shellBuildTrees at BuildStream / buildstream

Commits:

5 changed files:

Changes:

  • buildstream/_frontend/cli.py
    ... ... @@ -582,11 +582,13 @@ def show(app, elements, deps, except_, order, format_):
    582 582
                   help="Mount a file or directory into the sandbox")
    
    583 583
     @click.option('--isolate', is_flag=True, default=False,
    
    584 584
                   help='Create an isolated build sandbox')
    
    585
    +@click.option('--use-buildtree', '-t', type=click.Choice(['ask', 'if_available', 'always', 'never']), default=None,
    
    586
    +              help='Defaults to ask but if set to always the function will fail if a build tree is not available')
    
    585 587
     @click.argument('element',
    
    586 588
                     type=click.Path(readable=False))
    
    587 589
     @click.argument('command', type=click.STRING, nargs=-1)
    
    588 590
     @click.pass_obj
    
    589
    -def shell(app, element, sysroot, mount, isolate, build_, command):
    
    591
    +def shell(app, element, sysroot, mount, isolate, build_, use_buildtree, command):
    
    590 592
         """Run a command in the target element's sandbox environment
    
    591 593
     
    
    592 594
         This will stage a temporary sysroot for running the target
    
    ... ... @@ -612,6 +614,15 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
    612 614
         else:
    
    613 615
             scope = Scope.RUN
    
    614 616
     
    
    617
    +    use_buildtree_bool = None
    
    618
    +    if use_buildtree is None:
    
    619
    +        use_buildtree = 'ask'
    
    620
    +    else:
    
    621
    +        use_buildtree = use_buildtree.lower().strip()
    
    622
    +    if use_buildtree == 'always':
    
    623
    +        use_buildtree_bool = True
    
    624
    +    elif use_buildtree == 'never':
    
    625
    +        use_buildtree_bool = False
    
    615 626
         with app.initialized():
    
    616 627
             dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
    
    617 628
             element = dependencies[0]
    
    ... ... @@ -620,12 +631,33 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
    620 631
                 HostMount(path, host_path)
    
    621 632
                 for host_path, path in mount
    
    622 633
             ]
    
    634
    +
    
    635
    +        if not element._cached_buildtree():
    
    636
    +            if use_buildtree_bool:
    
    637
    +                raise AppError("No buildtree when requested")
    
    638
    +            else:
    
    639
    +                use_buildtree_bool = False
    
    640
    +        else:
    
    641
    +            if use_buildtree_bool:
    
    642
    +                pass
    
    643
    +            elif app.interactive and use_buildtree == 'ask':
    
    644
    +                if click.confirm('Do you want to use the cached buildtree?'):  # nopep8
    
    645
    +                    use_buildtree_bool = True
    
    646
    +                else:
    
    647
    +                    use_buildtree_bool = False
    
    648
    +            elif use_buildtree == 'if_available':
    
    649
    +                use_buildtree_bool = True
    
    650
    +            else:
    
    651
    +                use_buildtree_bool = False
    
    652
    +        assert type(use_buildtree_bool) == bool  # nopep8
    
    653
    +
    
    623 654
             try:
    
    624 655
                 exitcode = app.stream.shell(element, scope, prompt,
    
    625 656
                                             directory=sysroot,
    
    626 657
                                             mounts=mounts,
    
    627 658
                                             isolate=isolate,
    
    628
    -                                        command=command)
    
    659
    +                                        command=command,
    
    660
    +                                        usebuildtree=use_buildtree_bool)
    
    629 661
             except BstError as e:
    
    630 662
                 raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e
    
    631 663
     
    

  • buildstream/_stream.py
    ... ... @@ -132,7 +132,8 @@ class Stream():
    132 132
                   directory=None,
    
    133 133
                   mounts=None,
    
    134 134
                   isolate=False,
    
    135
    -              command=None):
    
    135
    +              command=None,
    
    136
    +              usebuildtree=None):
    
    136 137
     
    
    137 138
             # Assert we have everything we need built, unless the directory is specified
    
    138 139
             # in which case we just blindly trust the directory, using the element
    
    ... ... @@ -147,7 +148,8 @@ class Stream():
    147 148
                     raise StreamError("Elements need to be built or downloaded before staging a shell environment",
    
    148 149
                                       detail="\n".join(missing_deps))
    
    149 150
     
    
    150
    -        return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command)
    
    151
    +        return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command,
    
    152
    +                              usebuildtree=usebuildtree)
    
    151 153
     
    
    152 154
         # build()
    
    153 155
         #
    

  • buildstream/element.py
    ... ... @@ -1338,11 +1338,12 @@ class Element(Plugin):
    1338 1338
         # is used to stage things by the `bst checkout` codepath
    
    1339 1339
         #
    
    1340 1340
         @contextmanager
    
    1341
    -    def _prepare_sandbox(self, scope, directory, shell=False, integrate=True):
    
    1341
    +    def _prepare_sandbox(self, scope, directory, shell=False, integrate=True, usebuildtree=None):
    
    1342 1342
             # bst shell and bst checkout require a local sandbox.
    
    1343 1343
             bare_directory = True if directory else False
    
    1344 1344
             with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
    
    1345 1345
                                 bare_directory=bare_directory) as sandbox:
    
    1346
    +            sandbox.usebuildtree = usebuildtree
    
    1346 1347
     
    
    1347 1348
                 # Configure always comes first, and we need it.
    
    1348 1349
                 self.__configure_sandbox(sandbox)
    
    ... ... @@ -1386,7 +1387,7 @@ class Element(Plugin):
    1386 1387
             # Stage all sources that need to be copied
    
    1387 1388
             sandbox_vroot = sandbox.get_virtual_directory()
    
    1388 1389
             host_vdirectory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True)
    
    1389
    -        self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces)
    
    1390
    +        self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces, usebuildtree=sandbox.usebuildtree)
    
    1390 1391
     
    
    1391 1392
         # _stage_sources_at():
    
    1392 1393
         #
    
    ... ... @@ -1396,9 +1397,8 @@ class Element(Plugin):
    1396 1397
         #     vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into.
    
    1397 1398
         #     mount_workspaces (bool): mount workspaces if True, copy otherwise
    
    1398 1399
         #
    
    1399
    -    def _stage_sources_at(self, vdirectory, mount_workspaces=True):
    
    1400
    +    def _stage_sources_at(self, vdirectory, mount_workspaces=True, usebuildtree=False):
    
    1400 1401
             with self.timed_activity("Staging sources", silent_nested=True):
    
    1401
    -
    
    1402 1402
                 if not isinstance(vdirectory, Directory):
    
    1403 1403
                     vdirectory = FileBasedDirectory(vdirectory)
    
    1404 1404
                 if not vdirectory.is_empty():
    
    ... ... @@ -1420,7 +1420,7 @@ class Element(Plugin):
    1420 1420
                                                      .format(workspace.get_absolute_path())):
    
    1421 1421
                                 workspace.stage(temp_staging_directory)
    
    1422 1422
                     # Check if we have a cached buildtree to use
    
    1423
    -                elif self._cached_buildtree():
    
    1423
    +                elif usebuildtree:
    
    1424 1424
                         artifact_base, _ = self.__extract()
    
    1425 1425
                         import_dir = os.path.join(artifact_base, 'buildtree')
    
    1426 1426
                     else:
    
    ... ... @@ -1854,9 +1854,10 @@ class Element(Plugin):
    1854 1854
         # Returns: Exit code
    
    1855 1855
         #
    
    1856 1856
         # If directory is not specified, one will be staged using scope
    
    1857
    -    def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None):
    
    1857
    +    def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None,
    
    1858
    +               usebuildtree=None):
    
    1858 1859
     
    
    1859
    -        with self._prepare_sandbox(scope, directory, shell=True) as sandbox:
    
    1860
    +        with self._prepare_sandbox(scope, directory, shell=True, usebuildtree=usebuildtree) as sandbox:
    
    1860 1861
                 environment = self.get_environment()
    
    1861 1862
                 environment = copy.copy(environment)
    
    1862 1863
                 flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
    
    ... ... @@ -2231,7 +2232,6 @@ class Element(Plugin):
    2231 2232
                                         specs=self.__remote_execution_specs,
    
    2232 2233
                                         bare_directory=bare_directory,
    
    2233 2234
                                         allow_real_directory=False)
    
    2234
    -            yield sandbox
    
    2235 2235
     
    
    2236 2236
             elif directory is not None and os.path.exists(directory):
    
    2237 2237
                 if allow_remote and self.__remote_execution_specs:
    
    ... ... @@ -2249,7 +2249,6 @@ class Element(Plugin):
    2249 2249
                                                   config=config,
    
    2250 2250
                                                   bare_directory=bare_directory,
    
    2251 2251
                                                   allow_real_directory=not self.BST_VIRTUAL_DIRECTORY)
    
    2252
    -            yield sandbox
    
    2253 2252
     
    
    2254 2253
             else:
    
    2255 2254
                 os.makedirs(context.builddir, exist_ok=True)
    
    ... ... @@ -2263,6 +2262,10 @@ class Element(Plugin):
    2263 2262
                 # Cleanup the build dir
    
    2264 2263
                 utils._force_rmtree(rootdir)
    
    2265 2264
     
    
    2265
    +            return
    
    2266
    +        sandbox.usebuildtree = None
    
    2267
    +        yield sandbox
    
    2268
    +
    
    2266 2269
         def __compose_default_splits(self, defaults):
    
    2267 2270
             project = self._get_project()
    
    2268 2271
     
    

  • tests/integration/build-tree.py
    ... ... @@ -27,9 +27,44 @@ def test_buildtree_staged(cli_integration, tmpdir, datafiles):
    27 27
         res.assert_success()
    
    28 28
     
    
    29 29
         res = cli_integration.run(project=project, args=[
    
    30
    -        'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
    
    30
    +        'shell', '--build', element_name, '--', 'cat', 'test'
    
    31
    +    ])
    
    32
    +    res.assert_shell_error()
    
    33
    +
    
    34
    +
    
    35
    +@pytest.mark.datafiles(DATA_DIR)
    
    36
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    37
    +def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles):
    
    38
    +    # i.e. tests that cached build trees are staged by `bst shell --build`
    
    39
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    40
    +    element_name = 'build-shell/buildtree.bst'
    
    41
    +
    
    42
    +    res = cli_integration.run(project=project, args=['build', element_name])
    
    43
    +    res.assert_success()
    
    44
    +
    
    45
    +    res = cli_integration.run(project=project, args=[
    
    46
    +        'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
    
    31 47
         ])
    
    32 48
         res.assert_success()
    
    49
    +    assert 'Hi' in res.output
    
    50
    +
    
    51
    +
    
    52
    +@pytest.mark.datafiles(DATA_DIR)
    
    53
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    54
    +def test_buildtree_staged_forced_false(cli_integration, tmpdir, datafiles):
    
    55
    +    # i.e. tests that cached build trees are staged by `bst shell --build`
    
    56
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    57
    +    element_name = 'build-shell/buildtree.bst'
    
    58
    +
    
    59
    +    res = cli_integration.run(project=project, args=['build', element_name])
    
    60
    +    res.assert_success()
    
    61
    +
    
    62
    +    res = cli_integration.run(project=project, args=[
    
    63
    +        'shell', '--build', '--use-buildtree', 'never', element_name, '--', 'cat', 'test'
    
    64
    +    ])
    
    65
    +    res.assert_shell_error()
    
    66
    +
    
    67
    +    assert 'Hi' not in res.output
    
    33 68
     
    
    34 69
     
    
    35 70
     @pytest.mark.datafiles(DATA_DIR)
    
    ... ... @@ -44,7 +79,7 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
    44 79
     
    
    45 80
         # Assert that file has expected contents
    
    46 81
         res = cli_integration.run(project=project, args=[
    
    47
    -        'shell', '--build', element_name, '--', 'cat', 'test'
    
    82
    +        'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    
    48 83
         ])
    
    49 84
         res.assert_success()
    
    50 85
         assert 'Hi' in res.output
    
    ... ... @@ -80,6 +115,48 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
    80 115
     
    
    81 116
             # Check it's using the cached build tree
    
    82 117
             res = cli.run(project=project, args=[
    
    83
    -            'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
    
    118
    +            'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'grep', '-q', 'Hi', 'test'
    
    84 119
             ])
    
    85 120
             res.assert_success()
    
    121
    +
    
    122
    +
    
    123
    +# Check that build shells work when pulled from a remote cache
    
    124
    +# This is to roughly simulate remote execution
    
    125
    +@pytest.mark.datafiles(DATA_DIR)
    
    126
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    127
    +def test_buildtree_options(cli, tmpdir, datafiles):
    
    128
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    129
    +    element_name = 'build-shell/buildtree.bst'
    
    130
    +
    
    131
    +    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
    
    132
    +        # Build the element to push it to cache
    
    133
    +        cli.configure({
    
    134
    +            'artifacts': {'url': share.repo, 'push': True}
    
    135
    +        })
    
    136
    +        result = cli.run(project=project, args=['build', element_name])
    
    137
    +        result.assert_success()
    
    138
    +        assert cli.get_element_state(project, element_name) == 'cached'
    
    139
    +
    
    140
    +        # Discard the cache
    
    141
    +        cli.configure({
    
    142
    +            'artifacts': {'url': share.repo, 'push': True},
    
    143
    +            'artifactdir': os.path.join(cli.directory, 'artifacts2')
    
    144
    +        })
    
    145
    +        assert cli.get_element_state(project, element_name) != 'cached'
    
    146
    +
    
    147
    +        # Pull from cache, ensuring cli options is set to pull the buildtree
    
    148
    +        result = cli.run(project=project, args=['pull', '--deps', 'all', element_name])
    
    149
    +        result.assert_success()
    
    150
    +
    
    151
    +        # Check it's using the cached build tree
    
    152
    +        res = cli.run(project=project, args=[
    
    153
    +            'shell', '--build', element_name, '--use-buildtree', 'never', '--', 'cat', 'test'
    
    154
    +        ])
    
    155
    +        res.assert_shell_error()
    
    156
    +        assert 'Hi' not in res.output
    
    157
    +        # Check it's using the cached build tree
    
    158
    +        res = cli.run(project=project, args=[
    
    159
    +            'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    
    160
    +        ])
    
    161
    +        res.assert_main_error(ErrorDomain.PROG_NOT_FOUND, None)
    
    162
    +        assert 'Hi' not in res.output

  • tests/testutils/runcli.py
    ... ... @@ -153,6 +153,20 @@ class Result():
    153 153
             assert self.task_error_domain == error_domain, fail_message
    
    154 154
             assert self.task_error_reason == error_reason, fail_message
    
    155 155
     
    
    156
    +    # assert_shell_error()
    
    157
    +    #
    
    158
    +    # Asserts that the buildstream created a shell and that the task in the
    
    159
    +    # shell failed.
    
    160
    +    #
    
    161
    +    # Args:
    
    162
    +    #    fail_message (str): An optional message to override the automatic
    
    163
    +    #                        assertion error messages
    
    164
    +    # Raises:
    
    165
    +    #    (AssertionError): If any of the assertions fail
    
    166
    +    #
    
    167
    +    def assert_shell_error(self, fail_message=''):
    
    168
    +        assert self.exit_code == 1, fail_message
    
    169
    +
    
    156 170
         # get_tracked_elements()
    
    157 171
         #
    
    158 172
         # Produces a list of element names on which tracking occurred
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]