[Notes] [Git][BuildStream/buildstream][willsalmon/shellBuildTrees] Basic options for shell --build to use buildtrees



Title: GitLab

Will Salmon pushed to branch willsalmon/shellBuildTrees at BuildStream / buildstream

Commits:

6 changed files:

Changes:

  • buildstream/_frontend/cli.py
    ... ... @@ -527,11 +527,14 @@ def show(app, elements, deps, except_, order, format_):
    527 527
                   help="Mount a file or directory into the sandbox")
    
    528 528
     @click.option('--isolate', is_flag=True, default=False,
    
    529 529
                   help='Create an isolated build sandbox')
    
    530
    +@click.option('--use-buildtree', '-t', 'cli_buildtree', type=click.Choice(['ask', 'if-available', 'always', 'never']),
    
    531
    +              default='ask',
    
    532
    +              help='Defaults to ask but if set to always the function will fail if a build tree is not available')
    
    530 533
     @click.argument('element', required=False,
    
    531 534
                     type=click.Path(readable=False))
    
    532 535
     @click.argument('command', type=click.STRING, nargs=-1)
    
    533 536
     @click.pass_obj
    
    534
    -def shell(app, element, sysroot, mount, isolate, build_, command):
    
    537
    +def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command):
    
    535 538
         """Run a command in the target element's sandbox environment
    
    536 539
     
    
    537 540
         This will stage a temporary sysroot for running the target
    
    ... ... @@ -557,6 +560,8 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
    557 560
         else:
    
    558 561
             scope = Scope.RUN
    
    559 562
     
    
    563
    +    use_buildtree = False
    
    564
    +
    
    560 565
         with app.initialized():
    
    561 566
             if not element:
    
    562 567
                 element = app.context.guess_element()
    
    ... ... @@ -570,12 +575,30 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
    570 575
                 HostMount(path, host_path)
    
    571 576
                 for host_path, path in mount
    
    572 577
             ]
    
    578
    +
    
    579
    +        cached = element._cached_buildtree()
    
    580
    +        if cli_buildtree == "always":
    
    581
    +            if cached:
    
    582
    +                use_buildtree = True
    
    583
    +            else:
    
    584
    +                raise AppError("No buildtree is cached but the use buildtree option was specified")
    
    585
    +        elif cli_buildtree == "never":
    
    586
    +            pass
    
    587
    +        elif cli_buildtree == "if-available":
    
    588
    +            use_buildtree = cached
    
    589
    +        else:
    
    590
    +            if app.interactive and cached:
    
    591
    +                use_buildtree = bool(click.confirm('Do you want to use the cached buildtree?'))
    
    592
    +        if use_buildtree and not element._cached_success():
    
    593
    +            click.echo("Warning: using a buildtree from a failed build.")
    
    594
    +
    
    573 595
             try:
    
    574 596
                 exitcode = app.stream.shell(element, scope, prompt,
    
    575 597
                                             directory=sysroot,
    
    576 598
                                             mounts=mounts,
    
    577 599
                                             isolate=isolate,
    
    578
    -                                        command=command)
    
    600
    +                                        command=command,
    
    601
    +                                        usebuildtree=use_buildtree)
    
    579 602
             except BstError as e:
    
    580 603
                 raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e
    
    581 604
     
    

  • buildstream/_stream.py
    ... ... @@ -124,6 +124,7 @@ class Stream():
    124 124
         #    mounts (list of HostMount): Additional directories to mount into the sandbox
    
    125 125
         #    isolate (bool): Whether to isolate the environment like we do in builds
    
    126 126
         #    command (list): An argv to launch in the sandbox, or None
    
    127
    +    #    usebuildtree (bool): Wheather to use a buildtree as the source.
    
    127 128
         #
    
    128 129
         # Returns:
    
    129 130
         #    (int): The exit code of the launched shell
    
    ... ... @@ -132,7 +133,8 @@ class Stream():
    132 133
                   directory=None,
    
    133 134
                   mounts=None,
    
    134 135
                   isolate=False,
    
    135
    -              command=None):
    
    136
    +              command=None,
    
    137
    +              usebuildtree=False):
    
    136 138
     
    
    137 139
             # Assert we have everything we need built, unless the directory is specified
    
    138 140
             # in which case we just blindly trust the directory, using the element
    
    ... ... @@ -147,7 +149,8 @@ class Stream():
    147 149
                     raise StreamError("Elements need to be built or downloaded before staging a shell environment",
    
    148 150
                                       detail="\n".join(missing_deps))
    
    149 151
     
    
    150
    -        return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command)
    
    152
    +        return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command,
    
    153
    +                              usebuildtree=usebuildtree)
    
    151 154
     
    
    152 155
         # build()
    
    153 156
         #
    

  • buildstream/element.py
    ... ... @@ -1338,11 +1338,12 @@ class Element(Plugin):
    1338 1338
         # is used to stage things by the `bst checkout` codepath
    
    1339 1339
         #
    
    1340 1340
         @contextmanager
    
    1341
    -    def _prepare_sandbox(self, scope, directory, shell=False, integrate=True):
    
    1341
    +    def _prepare_sandbox(self, scope, directory, shell=False, integrate=True, usebuildtree=False):
    
    1342 1342
             # bst shell and bst checkout require a local sandbox.
    
    1343 1343
             bare_directory = True if directory else False
    
    1344 1344
             with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
    
    1345 1345
                                 bare_directory=bare_directory) as sandbox:
    
    1346
    +            sandbox._usebuildtree = usebuildtree
    
    1346 1347
     
    
    1347 1348
                 # Configure always comes first, and we need it.
    
    1348 1349
                 self.__configure_sandbox(sandbox)
    
    ... ... @@ -1386,7 +1387,7 @@ class Element(Plugin):
    1386 1387
             # Stage all sources that need to be copied
    
    1387 1388
             sandbox_vroot = sandbox.get_virtual_directory()
    
    1388 1389
             host_vdirectory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True)
    
    1389
    -        self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces)
    
    1390
    +        self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces, usebuildtree=sandbox._usebuildtree)
    
    1390 1391
     
    
    1391 1392
         # _stage_sources_at():
    
    1392 1393
         #
    
    ... ... @@ -1395,10 +1396,10 @@ class Element(Plugin):
    1395 1396
         # Args:
    
    1396 1397
         #     vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into.
    
    1397 1398
         #     mount_workspaces (bool): mount workspaces if True, copy otherwise
    
    1399
    +    #     usebuildtree (bool): use a the elements build tree as its source.
    
    1398 1400
         #
    
    1399
    -    def _stage_sources_at(self, vdirectory, mount_workspaces=True):
    
    1401
    +    def _stage_sources_at(self, vdirectory, mount_workspaces=True, usebuildtree=False):
    
    1400 1402
             with self.timed_activity("Staging sources", silent_nested=True):
    
    1401
    -
    
    1402 1403
                 if not isinstance(vdirectory, Directory):
    
    1403 1404
                     vdirectory = FileBasedDirectory(vdirectory)
    
    1404 1405
                 if not vdirectory.is_empty():
    
    ... ... @@ -1420,7 +1421,7 @@ class Element(Plugin):
    1420 1421
                                                      .format(workspace.get_absolute_path())):
    
    1421 1422
                                 workspace.stage(temp_staging_directory)
    
    1422 1423
                     # Check if we have a cached buildtree to use
    
    1423
    -                elif self._cached_buildtree():
    
    1424
    +                elif usebuildtree:
    
    1424 1425
                         artifact_base, _ = self.__extract()
    
    1425 1426
                         import_dir = os.path.join(artifact_base, 'buildtree')
    
    1426 1427
                     else:
    
    ... ... @@ -1850,13 +1851,15 @@ class Element(Plugin):
    1850 1851
         #    isolate (bool): Whether to isolate the environment like we do in builds
    
    1851 1852
         #    prompt (str): A suitable prompt string for PS1
    
    1852 1853
         #    command (list): An argv to launch in the sandbox
    
    1854
    +    #    usebuildtree (bool): Use the buildtree as its source
    
    1853 1855
         #
    
    1854 1856
         # Returns: Exit code
    
    1855 1857
         #
    
    1856 1858
         # If directory is not specified, one will be staged using scope
    
    1857
    -    def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None):
    
    1859
    +    def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None,
    
    1860
    +               usebuildtree=False):
    
    1858 1861
     
    
    1859
    -        with self._prepare_sandbox(scope, directory, shell=True) as sandbox:
    
    1862
    +        with self._prepare_sandbox(scope, directory, shell=True, usebuildtree=usebuildtree) as sandbox:
    
    1860 1863
                 environment = self.get_environment()
    
    1861 1864
                 environment = copy.copy(environment)
    
    1862 1865
                 flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
    

  • buildstream/sandbox/sandbox.py
    ... ... @@ -116,6 +116,7 @@ class Sandbox():
    116 116
             self.__env = None
    
    117 117
             self.__mount_sources = {}
    
    118 118
             self.__allow_real_directory = kwargs['allow_real_directory']
    
    119
    +        self.usebuildtree = False
    
    119 120
     
    
    120 121
             # Plugin ID for logging
    
    121 122
             plugin = kwargs.get('plugin', None)
    
    ... ... @@ -146,6 +147,7 @@ class Sandbox():
    146 147
     
    
    147 148
             self._output_directory = None
    
    148 149
             self._vdir = None
    
    150
    +        self._usebuildtree = False
    
    149 151
     
    
    150 152
             # This is set if anyone requests access to the underlying
    
    151 153
             # directory via get_directory.
    

  • tests/integration/build-tree.py
    ... ... @@ -19,7 +19,9 @@ DATA_DIR = os.path.join(
    19 19
     @pytest.mark.datafiles(DATA_DIR)
    
    20 20
     @pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    21 21
     def test_buildtree_staged(cli_integration, tmpdir, datafiles):
    
    22
    -    # i.e. tests that cached build trees are staged by `bst shell --build`
    
    22
    +    # We can only test the non interacitve case
    
    23
    +    # The non interactive case defaults to not using buildtrees
    
    24
    +    # for `bst shell --build`
    
    23 25
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    24 26
         element_name = 'build-shell/buildtree.bst'
    
    25 27
     
    
    ... ... @@ -27,15 +29,67 @@ def test_buildtree_staged(cli_integration, tmpdir, datafiles):
    27 29
         res.assert_success()
    
    28 30
     
    
    29 31
         res = cli_integration.run(project=project, args=[
    
    30
    -        'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
    
    32
    +        'shell', '--build', element_name, '--', 'cat', 'test'
    
    33
    +    ])
    
    34
    +    res.assert_shell_error()
    
    35
    +
    
    36
    +
    
    37
    +@pytest.mark.datafiles(DATA_DIR)
    
    38
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    39
    +def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles):
    
    40
    +    # Test that if we ask for a build tree it is there.
    
    41
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    42
    +    element_name = 'build-shell/buildtree.bst'
    
    43
    +
    
    44
    +    res = cli_integration.run(project=project, args=['build', element_name])
    
    45
    +    res.assert_success()
    
    46
    +
    
    47
    +    res = cli_integration.run(project=project, args=[
    
    48
    +        'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
    
    49
    +    ])
    
    50
    +    res.assert_success()
    
    51
    +    assert 'Hi' in res.output
    
    52
    +
    
    53
    +
    
    54
    +@pytest.mark.datafiles(DATA_DIR)
    
    55
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    56
    +def test_buildtree_staged_if_available(cli_integration, tmpdir, datafiles):
    
    57
    +    # Test that a build tree can be correctly detected.
    
    58
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    59
    +    element_name = 'build-shell/buildtree.bst'
    
    60
    +
    
    61
    +    res = cli_integration.run(project=project, args=['build', element_name])
    
    62
    +    res.assert_success()
    
    63
    +
    
    64
    +    res = cli_integration.run(project=project, args=[
    
    65
    +        'shell', '--build', '--use-buildtree', 'if-available', element_name, '--', 'cat', 'test'
    
    31 66
         ])
    
    32 67
         res.assert_success()
    
    68
    +    assert 'Hi' in res.output
    
    69
    +
    
    70
    +
    
    71
    +@pytest.mark.datafiles(DATA_DIR)
    
    72
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    73
    +def test_buildtree_staged_forced_false(cli_integration, tmpdir, datafiles):
    
    74
    +    # Test that if we ask not to have a build tree it is not there
    
    75
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    76
    +    element_name = 'build-shell/buildtree.bst'
    
    77
    +
    
    78
    +    res = cli_integration.run(project=project, args=['build', element_name])
    
    79
    +    res.assert_success()
    
    80
    +
    
    81
    +    res = cli_integration.run(project=project, args=[
    
    82
    +        'shell', '--build', '--use-buildtree', 'never', element_name, '--', 'cat', 'test'
    
    83
    +    ])
    
    84
    +    res.assert_shell_error()
    
    85
    +
    
    86
    +    assert 'Hi' not in res.output
    
    33 87
     
    
    34 88
     
    
    35 89
     @pytest.mark.datafiles(DATA_DIR)
    
    36 90
     @pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    37 91
     def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
    
    38
    -    # i.e. test that on a build failure, we can still shell into it
    
    92
    +    # Test that we can use a build tree after a failure
    
    39 93
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    40 94
         element_name = 'build-shell/buildtree-fail.bst'
    
    41 95
     
    
    ... ... @@ -44,9 +98,10 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
    44 98
     
    
    45 99
         # Assert that file has expected contents
    
    46 100
         res = cli_integration.run(project=project, args=[
    
    47
    -        'shell', '--build', element_name, '--', 'cat', 'test'
    
    101
    +        'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    
    48 102
         ])
    
    49 103
         res.assert_success()
    
    104
    +    assert "Warning: using a buildtree from a failed build" in res.output
    
    50 105
         assert 'Hi' in res.output
    
    51 106
     
    
    52 107
     
    
    ... ... @@ -80,6 +135,65 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
    80 135
     
    
    81 136
             # Check it's using the cached build tree
    
    82 137
             res = cli.run(project=project, args=[
    
    83
    -            'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
    
    138
    +            'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    
    84 139
             ])
    
    85 140
             res.assert_success()
    
    141
    +
    
    142
    +
    
    143
    +# This test checks for correct behaviour if a buildtree is not present.
    
    144
    +@pytest.mark.datafiles(DATA_DIR)
    
    145
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    146
    +def test_buildtree_options(cli, tmpdir, datafiles):
    
    147
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    148
    +    element_name = 'build-shell/buildtree.bst'
    
    149
    +
    
    150
    +    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
    
    151
    +        # Build the element to push it to cache
    
    152
    +        cli.configure({
    
    153
    +            'artifacts': {'url': share.repo, 'push': True}
    
    154
    +        })
    
    155
    +        result = cli.run(project=project, args=['build', element_name])
    
    156
    +        result.assert_success()
    
    157
    +        assert cli.get_element_state(project, element_name) == 'cached'
    
    158
    +
    
    159
    +        # Discard the cache
    
    160
    +        cli.configure({
    
    161
    +            'artifacts': {'url': share.repo, 'push': True},
    
    162
    +            'artifactdir': os.path.join(cli.directory, 'artifacts2')
    
    163
    +        })
    
    164
    +        assert cli.get_element_state(project, element_name) != 'cached'
    
    165
    +
    
    166
    +        # Pull from cache, but do not include buildtrees.
    
    167
    +        result = cli.run(project=project, args=['pull', '--deps', 'all', element_name])
    
    168
    +        result.assert_success()
    
    169
    +
    
    170
    +        # The above is the simplest way I know to create a local cache without any buildtrees.
    
    171
    +
    
    172
    +        # Check it's not using the cached build tree
    
    173
    +        res = cli.run(project=project, args=[
    
    174
    +            'shell', '--build', element_name, '--use-buildtree', 'never', '--', 'cat', 'test'
    
    175
    +        ])
    
    176
    +        res.assert_shell_error()
    
    177
    +        assert 'Hi' not in res.output
    
    178
    +
    
    179
    +        # Check it's not correctly handling the lack of buildtree
    
    180
    +        res = cli.run(project=project, args=[
    
    181
    +            'shell', '--build', element_name, '--use-buildtree', 'if-available', '--', 'cat', 'test'
    
    182
    +        ])
    
    183
    +        res.assert_shell_error()
    
    184
    +        assert 'Hi' not in res.output
    
    185
    +
    
    186
    +        # Check it's not using the cached build tree, default is to ask, and fall back to not
    
    187
    +        # for non interactive behavior
    
    188
    +        res = cli.run(project=project, args=[
    
    189
    +            'shell', '--build', element_name, '--', 'cat', 'test'
    
    190
    +        ])
    
    191
    +        res.assert_shell_error()
    
    192
    +        assert 'Hi' not in res.output
    
    193
    +
    
    194
    +        # Check it's using the cached build tree
    
    195
    +        res = cli.run(project=project, args=[
    
    196
    +            'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    
    197
    +        ])
    
    198
    +        res.assert_main_error(ErrorDomain.PROG_NOT_FOUND, None)
    
    199
    +        assert 'Hi' not in res.output

  • tests/testutils/runcli.py
    ... ... @@ -153,6 +153,20 @@ class Result():
    153 153
             assert self.task_error_domain == error_domain, fail_message
    
    154 154
             assert self.task_error_reason == error_reason, fail_message
    
    155 155
     
    
    156
    +    # assert_shell_error()
    
    157
    +    #
    
    158
    +    # Asserts that the buildstream created a shell and that the task in the
    
    159
    +    # shell failed.
    
    160
    +    #
    
    161
    +    # Args:
    
    162
    +    #    fail_message (str): An optional message to override the automatic
    
    163
    +    #                        assertion error messages
    
    164
    +    # Raises:
    
    165
    +    #    (AssertionError): If any of the assertions fail
    
    166
    +    #
    
    167
    +    def assert_shell_error(self, fail_message=''):
    
    168
    +        assert self.exit_code == 1, fail_message
    
    169
    +
    
    156 170
         # get_tracked_elements()
    
    157 171
         #
    
    158 172
         # Produces a list of element names on which tracking occurred
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]