GNOME.org
 

[Notes] [Git][BuildStream/buildstream][master] 17 commits: tests/integration/manual.py: Add test for command logging

Title: GitLab

Jürg Billeter pushed to branch master at BuildStream / buildstream

Commits:

14 changed files:

Changes:

  • NEWS
    ... ... @@ -67,6 +67,9 @@ buildstream 1.3.1
    67 67 
         allows the user to set a default location for their creation. This has meant
    68 68 
         that the new CLI is no longer backwards compatible with buildstream 1.2.
    69 69
    70 
    +  o Add sandbox API for command batching and use it for build, script, and
    71 
    +    compose elements.
    72 
    +
    70 73
    71 74 
     =================
    72 75 
     buildstream 1.1.5

  • buildstream/__init__.py
    ... ... @@ -27,7 +27,7 @@ if "_BST_COMPLETION" not in os.environ:
    27 27 
         del get_versions
    28 28
    29 29 
         from .utils import UtilError, ProgramNotFoundError
    30 
    -    from .sandbox import Sandbox, SandboxFlags
    30 
    +    from .sandbox import Sandbox, SandboxFlags, SandboxCommandError
    31 31 
         from .types import Scope, Consistency
    32 32 
         from .plugin import Plugin
    33 33 
         from .source import Source, SourceError, SourceFetcher

  • buildstream/buildelement.py
    ... ... @@ -127,7 +127,7 @@ artifact collection purposes.
    127 127 
     """
    128 128
    129 129 
     import os
    130 
    -from . import Element, Scope, ElementError
    130 
    +from . import Element, Scope
    131 131 
     from . import SandboxFlags
    132 132
    133 133
    ... ... @@ -207,6 +207,10 @@ class BuildElement(Element):
    207 207 
             # Setup environment
    208 208 
             sandbox.set_environment(self.get_environment())
    209 209
    210 
    +        # Enable command batching across prepare() and assemble()
    211 
    +        self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
    212 
    +                                    collect=self.get_variable('install-root'))
    213 
    +
    210 214 
         def stage(self, sandbox):
    211 215
    212 216 
             # Stage deps in the sandbox root
    ... ... @@ -215,7 +219,7 @@ class BuildElement(Element):
    215 219
    216 220 
             # Run any integration commands provided by the dependencies
    217 221 
             # once they are all staged and ready
    218 
    -        with self.timed_activity("Integrating sandbox"):
    222 
    +        with sandbox.batch(SandboxFlags.NONE, label="Integrating sandbox"):
    219 223 
                 for dep in self.dependencies(Scope.BUILD):
    220 224 
                     dep.integrate(sandbox)
    221 225
    ... ... @@ -223,14 +227,13 @@ class BuildElement(Element):
    223 227 
             self.stage_sources(sandbox, self.get_variable('build-root'))
    224 228
    225 229 
         def assemble(self, sandbox):
    226 
    -
    227 230 
             # Run commands
    228 231 
             for command_name in _command_steps:
    229 232 
                 commands = self.__commands[command_name]
    230 233 
                 if not commands or command_name == 'configure-commands':
    231 234 
                     continue
    232 235
    233 
    -            with self.timed_activity("Running {}".format(command_name)):
    236 
    +            with sandbox.batch(SandboxFlags.ROOT_READ_ONLY, label="Running {}".format(command_name)):
    234 237 
                     for cmd in commands:
    235 238 
                         self.__run_command(sandbox, cmd, command_name)
    236 239
    ... ... @@ -254,7 +257,7 @@ class BuildElement(Element):
    254 257 
         def prepare(self, sandbox):
    255 258 
             commands = self.__commands['configure-commands']
    256 259 
             if commands:
    257 
    -            with self.timed_activity("Running configure-commands"):
    260 
    +            with sandbox.batch(SandboxFlags.ROOT_READ_ONLY, label="Running configure-commands"):
    258 261 
                     for cmd in commands:
    259 262 
                         self.__run_command(sandbox, cmd, 'configure-commands')
    260 263
    ... ... @@ -282,13 +285,9 @@ class BuildElement(Element):
    282 285 
             return commands
    283 286
    284 287 
         def __run_command(self, sandbox, cmd, cmd_name):
    285 
    -        self.status("Running {}".format(cmd_name), detail=cmd)
    286 
    -
    287 288 
             # Note the -e switch to 'sh' means to exit with an error
    288 289 
             # if any untested command fails.
    289 290 
             #
    290 
    -        exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    291 
    -                               SandboxFlags.ROOT_READ_ONLY)
    292 
    -        if exitcode != 0:
    293 
    -            raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
    294 
    -                               collect=self.get_variable('install-root'))
    291 
    +        sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    292 
    +                    SandboxFlags.ROOT_READ_ONLY,
    293 
    +                    label=cmd)

  • buildstream/element.py
    ... ... @@ -78,6 +78,7 @@ import stat
    78 78 
     import copy
    79 79 
     from collections import OrderedDict
    80 80 
     from collections.abc import Mapping
    81 
    +import contextlib
    81 82 
     from contextlib import contextmanager
    82 83 
     import tempfile
    83 84 
     import shutil
    ... ... @@ -89,7 +90,7 @@ from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \
    89 90 
         ErrorDomain
    90 91 
     from .utils import UtilError
    91 92 
     from . import Plugin, Consistency, Scope
    92 
    -from . import SandboxFlags
    93 
    +from . import SandboxFlags, SandboxCommandError
    93 94 
     from . import utils
    94 95 
     from . import _cachekey
    95 96 
     from . import _signals
    ... ... @@ -217,6 +218,10 @@ class Element(Plugin):
    217 218 
             self.__build_result = None              # The result of assembling this Element (success, description, detail)
    218 219 
             self._build_log_path = None            # The path of the build log for this Element
    219 220
    221 
    +        self.__batch_prepare_assemble = False         # Whether batching across prepare()/assemble() is configured
    222 
    +        self.__batch_prepare_assemble_flags = 0       # Sandbox flags for batching across prepare()/assemble()
    223 
    +        self.__batch_prepare_assemble_collect = None  # Collect dir for batching across prepare()/assemble()
    224 
    +
    220 225 
             # hash tables of loaded artifact metadata, hashed by key
    221 226 
             self.__metadata_keys = {}                     # Strong and weak keys for this key
    222 227 
             self.__metadata_dependencies = {}             # Dictionary of dependency strong keys
    ... ... @@ -770,13 +775,13 @@ class Element(Plugin):
    770 775 
             environment = self.get_environment()
    771 776
    772 777 
             if bstdata is not None:
    773 
    -            commands = self.node_get_member(bstdata, list, 'integration-commands', [])
    774 
    -            for i in range(len(commands)):
    775 
    -                cmd = self.node_subst_list_element(bstdata, 'integration-commands', [i])
    776 
    -                self.status("Running integration command", detail=cmd)
    777 
    -                exitcode = sandbox.run(['sh', '-e', '-c', cmd], 0, env=environment, cwd='/')
    778 
    -                if exitcode != 0:
    779 
    -                    raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode))
    778 
    +            with sandbox.batch(SandboxFlags.NONE):
    779 
    +                commands = self.node_get_member(bstdata, list, 'integration-commands', [])
    780 
    +                for i in range(len(commands)):
    781 
    +                    cmd = self.node_subst_list_element(bstdata, 'integration-commands', [i])
    782 
    +
    783 
    +                    sandbox.run(['sh', '-e', '-c', cmd], 0, env=environment, cwd='/',
    784 
    +                                label=cmd)
    780 785
    781 786 
         def stage_sources(self, sandbox, directory):
    782 787 
             """Stage this element's sources to a directory in the sandbox
    ... ... @@ -863,6 +868,24 @@ class Element(Plugin):
    863 868
    864 869 
             return None
    865 870
    871 
    +    def batch_prepare_assemble(self, flags, *, collect=None):
    872 
    +        """ Configure command batching across prepare() and assemble()
    873 
    +
    874 
    +        Args:
    875 
    +           flags (:class:`.SandboxFlags`): The sandbox flags for the command batch
    876 
    +           collect (str): An optional directory containing partial install contents
    877 
    +                          on command failure.
    878 
    +
    879 
    +        This may be called in :func:`Element.configure_sandbox() <buildstream.element.Element.configure_sandbox>`
    880 
    +        to enable batching of all sandbox commands issued in prepare() and assemble().
    881 
    +        """
    882 
    +        if self.__batch_prepare_assemble:
    883 
    +            raise ElementError("{}: Command batching for prepare/assemble is already configured".format(self))
    884 
    +
    885 
    +        self.__batch_prepare_assemble = True
    886 
    +        self.__batch_prepare_assemble_flags = flags
    887 
    +        self.__batch_prepare_assemble_collect = collect
    888 
    +
    866 889 
         #############################################################
    867 890 
         #            Private Methods used in BuildStream            #
    868 891 
         #############################################################
    ... ... @@ -1323,7 +1346,7 @@ class Element(Plugin):
    1323 1346 
                                 bare_directory=bare_directory) as sandbox:
    1324 1347
    1325 1348 
                 # Configure always comes first, and we need it.
    1326 
    -            self.configure_sandbox(sandbox)
    1349 
    +            self.__configure_sandbox(sandbox)
    1327 1350
    1328 1351 
                 # Stage something if we need it
    1329 1352 
                 if not directory:
    ... ... @@ -1556,15 +1579,24 @@ class Element(Plugin):
    1556 1579 
                     # Call the abstract plugin methods
    1557 1580 
                     try:
    1558 1581 
                         # Step 1 - Configure
    1559 
    -                    self.configure_sandbox(sandbox)
    1582 
    +                    self.__configure_sandbox(sandbox)
    1560 1583 
                         # Step 2 - Stage
    1561 1584 
                         self.stage(sandbox)
    1562 
    -                    # Step 3 - Prepare
    1563 
    -                    self.__prepare(sandbox)
    1564 
    -                    # Step 4 - Assemble
    1565 
    -                    collect = self.assemble(sandbox)  # pylint: disable=assignment-from-no-return
    1585 
    +
    1586 
    +                    if self.__batch_prepare_assemble:
    1587 
    +                        cm = sandbox.batch(self.__batch_prepare_assemble_flags,
    1588 
    +                                           collect=self.__batch_prepare_assemble_collect)
    1589 
    +                    else:
    1590 
    +                        cm = contextlib.suppress()
    1591 
    +
    1592 
    +                    with cm:
    1593 
    +                        # Step 3 - Prepare
    1594 
    +                        self.__prepare(sandbox)
    1595 
    +                        # Step 4 - Assemble
    1596 
    +                        collect = self.assemble(sandbox)  # pylint: disable=assignment-from-no-return
    1597 
    +
    1566 1598 
                         self.__set_build_result(success=True, description="succeeded")
    1567 
    -                except ElementError as e:
    1599 
    +                except (ElementError, SandboxCommandError) as e:
    1568 1600 
                         # Shelling into a sandbox is useful to debug this error
    1569 1601 
                         e.sandbox = True
    1570 1602
    ... ... @@ -2059,6 +2091,15 @@ class Element(Plugin):
    2059 2091 
         def __can_build_incrementally(self):
    2060 2092 
             return bool(self._get_workspace())
    2061 2093
    2094 
    +    # __configure_sandbox():
    2095 
    +    #
    2096 
    +    # Internal method for calling public abstract configure_sandbox() method.
    2097 
    +    #
    2098 
    +    def __configure_sandbox(self, sandbox):
    2099 
    +        self.__batch_prepare_assemble = False
    2100 
    +
    2101 
    +        self.configure_sandbox(sandbox)
    2102 
    +
    2062 2103 
         # __prepare():
    2063 2104 
         #
    2064 2105 
         # Internal method for calling public abstract prepare() method.
    ... ... @@ -2074,7 +2115,12 @@ class Element(Plugin):
    2074 2115 
                 self.prepare(sandbox)
    2075 2116
    2076 2117 
                 if workspace:
    2077 
    -                workspace.prepared = True
    2118 
    +                def mark_workspace_prepared():
    2119 
    +                    workspace.prepared = True
    2120 
    +
    2121 
    +                # Defer workspace.prepared setting until pending batch commands
    2122 
    +                # have been executed.
    2123 
    +                sandbox._callback(mark_workspace_prepared)
    2078 2124
    2079 2125 
         def __is_cached(self, keystrength):
    2080 2126 
             if keystrength is None:
    ... ... @@ -2157,6 +2203,7 @@ class Element(Plugin):
    2157 2203
    2158 2204 
                 sandbox = SandboxRemote(context, project,
    2159 2205 
                                         directory,
    2206 
    +                                    plugin=self,
    2160 2207 
                                         stdout=stdout,
    2161 2208 
                                         stderr=stderr,
    2162 2209 
                                         config=config,
    ... ... @@ -2175,6 +2222,7 @@ class Element(Plugin):
    2175 2222
    2176 2223 
                 sandbox = platform.create_sandbox(context, project,
    2177 2224 
                                                   directory,
    2225 
    +                                              plugin=self,
    2178 2226 
                                                   stdout=stdout,
    2179 2227 
                                                   stderr=stderr,
    2180 2228 
                                                   config=config,

  • buildstream/plugins/elements/compose.py
    ... ... @@ -122,8 +122,9 @@ class ComposeElement(Element):
    122 122 
                         snapshot = set(vbasedir.list_relative_paths())
    123 123 
                         vbasedir.mark_unmodified()
    124 124
    125 
    -                for dep in self.dependencies(Scope.BUILD):
    126 
    -                    dep.integrate(sandbox)
    125 
    +                with sandbox.batch(0):
    126 
    +                    for dep in self.dependencies(Scope.BUILD):
    127 
    +                        dep.integrate(sandbox)
    127 128
    128 129 
                     if require_split:
    129 130 
                         # Calculate added, modified and removed files

  • buildstream/sandbox/__init__.py
    ... ... @@ -17,6 +17,6 @@
    17 17 
     #  Authors:
    18 18 
     #        Tristan Maat <tristan maat codethink co uk>
    19 19
    20 
    -from .sandbox import Sandbox, SandboxFlags
    20 
    +from .sandbox import Sandbox, SandboxFlags, SandboxCommandError
    21 21 
     from ._sandboxremote import SandboxRemote
    22 22 
     from ._sandboxdummy import SandboxDummy

  • buildstream/sandbox/_sandboxbwrap.py
    ... ... @@ -58,22 +58,12 @@ class SandboxBwrap(Sandbox):
    58 58 
             self.die_with_parent_available = kwargs['die_with_parent_available']
    59 59 
             self.json_status_available = kwargs['json_status_available']
    60 60
    61 
    -    def run(self, command, flags, *, cwd=None, env=None):
    61 
    +    def _run(self, command, flags, *, cwd, env):
    62 62 
             stdout, stderr = self._get_output()
    63 63
    64 64 
             # Allowable access to underlying storage as we're part of the sandbox
    65 65 
             root_directory = self.get_virtual_directory()._get_underlying_directory()
    66 66
    67 
    -        # Fallback to the sandbox default settings for
    68 
    -        # the cwd and env.
    69 
    -        #
    70 
    -        cwd = self._get_work_directory(cwd=cwd)
    71 
    -        env = self._get_environment(cwd=cwd, env=env)
    72 
    -
    73 
    -        # Convert single-string argument to a list
    74 
    -        if isinstance(command, str):
    75 
    -            command = [command]
    76 
    -
    77 67 
             if not self._has_command(command[0], env):
    78 68 
                 raise SandboxError("Staged artifacts do not provide command "
    79 69 
                                    "'{}'".format(command[0]),

  • buildstream/sandbox/_sandboxchroot.py
    ... ... @@ -49,17 +49,7 @@ class SandboxChroot(Sandbox):
    49 49
    50 50 
             self.mount_map = None
    51 51
    52 
    -    def run(self, command, flags, *, cwd=None, env=None):
    53 
    -
    54 
    -        # Fallback to the sandbox default settings for
    55 
    -        # the cwd and env.
    56 
    -        #
    57 
    -        cwd = self._get_work_directory(cwd=cwd)
    58 
    -        env = self._get_environment(cwd=cwd, env=env)
    59 
    -
    60 
    -        # Convert single-string argument to a list
    61 
    -        if isinstance(command, str):
    62 
    -            command = [command]
    52 
    +    def _run(self, command, flags, *, cwd, env):
    63 53
    64 54 
             if not self._has_command(command[0], env):
    65 55 
                 raise SandboxError("Staged artifacts do not provide command "

  • buildstream/sandbox/_sandboxdummy.py
    ... ... @@ -25,17 +25,7 @@ class SandboxDummy(Sandbox):
    25 25 
             super().__init__(*args, **kwargs)
    26 26 
             self._reason = kwargs.get("dummy_reason", "no reason given")
    27 27
    28 
    -    def run(self, command, flags, *, cwd=None, env=None):
    29 
    -
    30 
    -        # Fallback to the sandbox default settings for
    31 
    -        # the cwd and env.
    32 
    -        #
    33 
    -        cwd = self._get_work_directory(cwd=cwd)
    34 
    -        env = self._get_environment(cwd=cwd, env=env)
    35 
    -
    36 
    -        # Convert single-string argument to a list
    37 
    -        if isinstance(command, str):
    38 
    -            command = [command]
    28 
    +    def _run(self, command, flags, *, cwd, env):
    39 29
    40 30 
             if not self._has_command(command[0], env):
    41 31 
                 raise SandboxError("Staged artifacts do not provide command "

  • buildstream/sandbox/_sandboxremote.py
    ... ... @@ -19,12 +19,14 @@
    19 19 
     #        Jim MacArthur <jim macarthur codethink co uk>
    20 20
    21 21 
     import os
    22 
    +import shlex
    22 23 
     from urllib.parse import urlparse
    23 24 
     from functools import partial
    24 25
    25 26 
     import grpc
    26 27
    27 
    -from . import Sandbox
    28 
    +from . import Sandbox, SandboxCommandError
    29 
    +from .sandbox import _SandboxBatch
    28 30 
     from ..storage._filebaseddirectory import FileBasedDirectory
    29 31 
     from ..storage._casbaseddirectory import CasBasedDirectory
    30 32 
     from .. import _signals
    ... ... @@ -212,7 +214,7 @@ class SandboxRemote(Sandbox):
    212 214 
             new_dir = CasBasedDirectory(self._get_context().artifactcache.cas, ref=dir_digest)
    213 215 
             self._set_virtual_directory(new_dir)
    214 216
    215 
    -    def run(self, command, flags, *, cwd=None, env=None):
    217 
    +    def _run(self, command, flags, *, cwd, env):
    216 218 
             # Upload sources
    217 219 
             upload_vdir = self.get_virtual_directory()
    218 220
    ... ... @@ -230,16 +232,6 @@ class SandboxRemote(Sandbox):
    230 232 
             if not cascache.verify_digest_pushed(self._get_project(), upload_vdir.ref):
    231 233 
                 raise SandboxError("Failed to verify that source has been pushed to the remote artifact cache.")
    232 234
    233 
    -        # Fallback to the sandbox default settings for
    234 
    -        # the cwd and env.
    235 
    -        #
    236 
    -        cwd = self._get_work_directory(cwd=cwd)
    237 
    -        env = self._get_environment(cwd=cwd, env=env)
    238 
    -
    239 
    -        # We want command args as a list of strings
    240 
    -        if isinstance(command, str):
    241 
    -            command = [command]
    242 
    -
    243 235 
             # Now transmit the command to execute
    244 236 
             operation = self.run_remote_command(command, upload_vdir.ref, cwd, env)
    245 237
    ... ... @@ -275,3 +267,69 @@ class SandboxRemote(Sandbox):
    275 267 
             self.process_job_output(action_result.output_directories, action_result.output_files)
    276 268
    277 269 
             return 0
    270 
    +
    271 
    +    def _create_batch(self, main_group, flags, *, collect=None):
    272 
    +        return _SandboxRemoteBatch(self, main_group, flags, collect=collect)
    273 
    +
    274 
    +
    275 
    +# _SandboxRemoteBatch()
    276 
    +#
    277 
    +# Command batching by shell script generation.
    278 
    +#
    279 
    +class _SandboxRemoteBatch(_SandboxBatch):
    280 
    +
    281 
    +    def __init__(self, sandbox, main_group, flags, *, collect=None):
    282 
    +        super().__init__(sandbox, main_group, flags, collect=collect)
    283 
    +
    284 
    +        self.script = None
    285 
    +        self.first_command = None
    286 
    +        self.cwd = None
    287 
    +        self.env = None
    288 
    +
    289 
    +    def execute(self):
    290 
    +        self.script = ""
    291 
    +
    292 
    +        self.main_group.execute(self)
    293 
    +
    294 
    +        first = self.first_command
    295 
    +        if first and self.sandbox.run(['sh', '-c', '-e', self.script], self.flags, cwd=first.cwd, env=first.env) != 0:
    296 
    +            raise SandboxCommandError("Command execution failed", collect=self.collect)
    297 
    +
    298 
    +    def execute_group(self, group):
    299 
    +        group.execute_children(self)
    300 
    +
    301 
    +    def execute_command(self, command):
    302 
    +        if self.first_command is None:
    303 
    +            # First command in batch
    304 
    +            # Initial working directory and environment of script already matches
    305 
    +            # the command configuration.
    306 
    +            self.first_command = command
    307 
    +        else:
    308 
    +            # Change working directory for this command
    309 
    +            if command.cwd != self.cwd:
    310 
    +                self.script += "mkdir -p {}\n".format(command.cwd)
    311 
    +                self.script += "cd {}\n".format(command.cwd)
    312 
    +
    313 
    +            # Update environment for this command
    314 
    +            for key in self.env.keys():
    315 
    +                if key not in command.env:
    316 
    +                    self.script += "unset {}\n".format(key)
    317 
    +            for key, value in command.env.items():
    318 
    +                if key not in self.env or self.env[key] != value:
    319 
    +                    self.script += "export {}={}\n".format(key, shlex.quote(value))
    320 
    +
    321 
    +        # Keep track of current working directory and environment
    322 
    +        self.cwd = command.cwd
    323 
    +        self.env = command.env
    324 
    +
    325 
    +        # Actual command execution
    326 
    +        cmdline = ' '.join(shlex.quote(cmd) for cmd in command.command)
    327 
    +        self.script += "(set -ex; {})".format(cmdline)
    328 
    +
    329 
    +        # Error handling
    330 
    +        label = command.label or cmdline
    331 
    +        quoted_label = shlex.quote("'{}'".format(label))
    332 
    +        self.script += " || (echo Command {} failed with exitcode $? >&2 ; exit 1)\n".format(quoted_label)
    333 
    +
    334 
    +    def execute_call(self, call):
    335 
    +        raise SandboxError("SandboxRemote does not support callbacks in command batches")

  • buildstream/sandbox/sandbox.py
    1 1 
     #
    2 2 
     #  Copyright (C) 2017 Codethink Limited
    3 
    +#  Copyright (C) 2018 Bloomberg Finance LP
    3 4 
     #
    4 5 
     #  This program is free software; you can redistribute it and/or
    5 6 
     #  modify it under the terms of the GNU Lesser General Public
    ... ... @@ -29,7 +30,12 @@ See also: :ref:`sandboxing`.
    29 30 
     """
    30 31
    31 32 
     import os
    32 
    -from .._exceptions import ImplError, BstError
    33 
    +import shlex
    34 
    +import contextlib
    35 
    +from contextlib import contextmanager
    36 
    +
    37 
    +from .._exceptions import ImplError, BstError, SandboxError
    38 
    +from .._message import Message, MessageType
    33 39 
     from ..storage._filebaseddirectory import FileBasedDirectory
    34 40 
     from ..storage._casbaseddirectory import CasBasedDirectory
    35 41
    ... ... @@ -38,6 +44,10 @@ class SandboxFlags():
    38 44 
         """Flags indicating how the sandbox should be run.
    39 45 
         """
    40 46
    47 
    +    NONE = 0
    48 
    +    """Use default sandbox configuration.
    49 
    +    """
    50 
    +
    41 51 
         ROOT_READ_ONLY = 0x01
    42 52 
         """The root filesystem is read only.
    43 53
    ... ... @@ -71,6 +81,19 @@ class SandboxFlags():
    71 81 
         """
    72 82
    73 83
    84 
    +class SandboxCommandError(SandboxError):
    85 
    +    """Raised by :class:`.Sandbox` implementations when a command fails.
    86 
    +
    87 
    +    Args:
    88 
    +       message (str): The error message to report to the user
    89 
    +       collect (str): An optional directory containing partial install contents
    90 
    +    """
    91 
    +    def __init__(self, message, *, collect=None):
    92 
    +        super().__init__(message, reason='command-failed')
    93 
    +
    94 
    +        self.collect = collect
    95 
    +
    96 
    +
    74 97 
     class Sandbox():
    75 98 
         """Sandbox()
    76 99
    ... ... @@ -94,6 +117,13 @@ class Sandbox():
    94 117 
             self.__mount_sources = {}
    95 118 
             self.__allow_real_directory = kwargs['allow_real_directory']
    96 119
    120 
    +        # Plugin ID for logging
    121 
    +        plugin = kwargs.get('plugin', None)
    122 
    +        if plugin:
    123 
    +            self.__plugin_id = plugin._get_unique_id()
    124 
    +        else:
    125 
    +            self.__plugin_id = None
    126 
    +
    97 127 
             # Configuration from kwargs common to all subclasses
    98 128 
             self.__config = kwargs['config']
    99 129 
             self.__stdout = kwargs['stdout']
    ... ... @@ -121,6 +151,9 @@ class Sandbox():
    121 151 
             # directory via get_directory.
    122 152 
             self._never_cache_vdirs = False
    123 153
    154 
    +        # Pending command batch
    155 
    +        self.__batch = None
    156 
    +
    124 157 
         def get_directory(self):
    125 158 
             """Fetches the sandbox root directory
    126 159
    ... ... @@ -209,9 +242,16 @@ class Sandbox():
    209 242 
                 'artifact': artifact
    210 243 
             })
    211 244
    212 
    -    def run(self, command, flags, *, cwd=None, env=None):
    245 
    +    def run(self, command, flags, *, cwd=None, env=None, label=None):
    213 246 
             """Run a command in the sandbox.
    214 247
    248 
    +        If this is called outside a batch context, the command is immediately
    249 
    +        executed.
    250 
    +
    251 
    +        If this is called in a batch context, the command is added to the batch
    252 
    +        for later execution. If the command fails, later commands will not be
    253 
    +        executed. Command flags must match batch flags.
    254 
    +
    215 255 
             Args:
    216 256 
                 command (list): The command to run in the sandboxed environment, as a list
    217 257 
                                 of strings starting with the binary to run.
    ... ... @@ -219,9 +259,10 @@ class Sandbox():
    219 259 
                 cwd (str): The sandbox relative working directory in which to run the command.
    220 260 
                 env (dict): A dictionary of string key, value pairs to set as environment
    221 261 
                             variables inside the sandbox environment.
    262 
    +            label (str): An optional label for the command, used for logging. (*Since: 1.4*)
    222 263
    223 264 
             Returns:
    224 
    -            (int): The program exit code.
    265 
    +            (int|None): The program exit code, or None if running in batch context.
    225 266
    226 267 
             Raises:
    227 268 
                 (:class:`.ProgramNotFoundError`): If a host tool which the given sandbox
    ... ... @@ -234,9 +275,115 @@ class Sandbox():
    234 275 
                function must make sure the directory will be created if it does
    235 276 
                not exist yet, even if a workspace is being used.
    236 277 
             """
    237 
    -        raise ImplError("Sandbox of type '{}' does not implement run()"
    278 
    +
    279 
    +        # Fallback to the sandbox default settings for
    280 
    +        # the cwd and env.
    281 
    +        #
    282 
    +        cwd = self._get_work_directory(cwd=cwd)
    283 
    +        env = self._get_environment(cwd=cwd, env=env)
    284 
    +
    285 
    +        # Convert single-string argument to a list
    286 
    +        if isinstance(command, str):
    287 
    +            command = [command]
    288 
    +
    289 
    +        if self.__batch:
    290 
    +            if flags != self.__batch.flags:
    291 
    +                raise SandboxError("Inconsistent sandbox flags in single command batch")
    292 
    +
    293 
    +            batch_command = _SandboxBatchCommand(command, cwd=cwd, env=env, label=label)
    294 
    +
    295 
    +            current_group = self.__batch.current_group
    296 
    +            current_group.append(batch_command)
    297 
    +            return None
    298 
    +        else:
    299 
    +            return self._run(command, flags, cwd=cwd, env=env)
    300 
    +
    301 
    +    @contextmanager
    302 
    +    def batch(self, flags, *, label=None, collect=None):
    303 
    +        """Context manager for command batching
    304 
    +
    305 
    +        This provides a batch context that defers execution of commands until
    306 
    +        the end of the context. If a command fails, the batch will be aborted
    307 
    +        and subsequent commands will not be executed.
    308 
    +
    309 
    +        Command batches may be nested. Execution will start only when the top
    310 
    +        level batch context ends.
    311 
    +
    312 
    +        Args:
    313 
    +            flags (:class:`.SandboxFlags`): The flags for this command batch.
    314 
    +            label (str): An optional label for the batch group, used for logging.
    315 
    +            collect (str): An optional directory containing partial install contents
    316 
    +                           on command failure.
    317 
    +
    318 
    +        Raises:
    319 
    +            (:class:`.SandboxCommandError`): If a command fails.
    320 
    +
    321 
    +        *Since: 1.4*
    322 
    +        """
    323 
    +
    324 
    +        group = _SandboxBatchGroup(label=label)
    325 
    +
    326 
    +        if self.__batch:
    327 
    +            # Nested batch
    328 
    +            if flags != self.__batch.flags:
    329 
    +                raise SandboxError("Inconsistent sandbox flags in single command batch")
    330 
    +
    331 
    +            parent_group = self.__batch.current_group
    332 
    +            parent_group.append(group)
    333 
    +            self.__batch.current_group = group
    334 
    +            try:
    335 
    +                yield
    336 
    +            finally:
    337 
    +                self.__batch.current_group = parent_group
    338 
    +        else:
    339 
    +            # Top-level batch
    340 
    +            batch = self._create_batch(group, flags, collect=collect)
    341 
    +
    342 
    +            self.__batch = batch
    343 
    +            try:
    344 
    +                yield
    345 
    +            finally:
    346 
    +                self.__batch = None
    347 
    +
    348 
    +            batch.execute()
    349 
    +
    350 
    +    #####################################################
    351 
    +    #    Abstract Methods for Sandbox implementations   #
    352 
    +    #####################################################
    353 
    +
    354 
    +    # _run()
    355 
    +    #
    356 
    +    # Abstract method for running a single command
    357 
    +    #
    358 
    +    # Args:
    359 
    +    #    command (list): The command to run in the sandboxed environment, as a list
    360 
    +    #                    of strings starting with the binary to run.
    361 
    +    #    flags (:class:`.SandboxFlags`): The flags for running this command.
    362 
    +    #    cwd (str): The sandbox relative working directory in which to run the command.
    363 
    +    #    env (dict): A dictionary of string key, value pairs to set as environment
    364 
    +    #                variables inside the sandbox environment.
    365 
    +    #
    366 
    +    # Returns:
    367 
    +    #    (int): The program exit code.
    368 
    +    #
    369 
    +    def _run(self, command, flags, *, cwd, env):
    370 
    +        raise ImplError("Sandbox of type '{}' does not implement _run()"
    238 371 
                             .format(type(self).__name__))
    239 372
    373 
    +    # _create_batch()
    374 
    +    #
    375 
    +    # Abstract method for creating a batch object. Subclasses can override
    376 
    +    # this method to instantiate a subclass of _SandboxBatch.
    377 
    +    #
    378 
    +    # Args:
    379 
    +    #    main_group (:class:`_SandboxBatchGroup`): The top level batch group.
    380 
    +    #    flags (:class:`.SandboxFlags`): The flags for commands in this batch.
    381 
    +    #    collect (str): An optional directory containing partial install contents
    382 
    +    #                   on command failure.
    383 
    +    #
    384 
    +    def _create_batch(self, main_group, flags, *, collect=None):
    385 
    +        return _SandboxBatch(self, main_group, flags, collect=collect)
    386 
    +
    240 387 
         ################################################
    241 388 
         #               Private methods                #
    242 389 
         ################################################
    ... ... @@ -385,3 +532,138 @@ class Sandbox():
    385 532 
                     return True
    386 533
    387 534 
             return False
    535 
    +
    536 
    +    # _get_plugin_id()
    537 
    +    #
    538 
    +    # Get the plugin's unique identifier
    539 
    +    #
    540 
    +    def _get_plugin_id(self):
    541 
    +        return self.__plugin_id
    542 
    +
    543 
    +    # _callback()
    544 
    +    #
    545 
    +    # If this is called outside a batch context, the specified function is
    546 
    +    # invoked immediately.
    547 
    +    #
    548 
    +    # If this is called in a batch context, the function is added to the batch
    549 
    +    # for later invocation.
    550 
    +    #
    551 
    +    # Args:
    552 
    +    #    callback (callable): The function to invoke
    553 
    +    #
    554 
    +    def _callback(self, callback):
    555 
    +        if self.__batch:
    556 
    +            batch_call = _SandboxBatchCall(callback)
    557 
    +
    558 
    +            current_group = self.__batch.current_group
    559 
    +            current_group.append(batch_call)
    560 
    +        else:
    561 
    +            callback()
    562 
    +
    563 
    +
    564 
    +# _SandboxBatch()
    565 
    +#
    566 
    +# A batch of sandbox commands.
    567 
    +#
    568 
    +class _SandboxBatch():
    569 
    +
    570 
    +    def __init__(self, sandbox, main_group, flags, *, collect=None):
    571 
    +        self.sandbox = sandbox
    572 
    +        self.main_group = main_group
    573 
    +        self.current_group = main_group
    574 
    +        self.flags = flags
    575 
    +        self.collect = collect
    576 
    +
    577 
    +    def execute(self):
    578 
    +        self.main_group.execute(self)
    579 
    +
    580 
    +    def execute_group(self, group):
    581 
    +        if group.label:
    582 
    +            context = self.sandbox._get_context()
    583 
    +            cm = context.timed_activity(group.label, unique_id=self.sandbox._get_plugin_id())
    584 
    +        else:
    585 
    +            cm = contextlib.suppress()
    586 
    +
    587 
    +        with cm:
    588 
    +            group.execute_children(self)
    589 
    +
    590 
    +    def execute_command(self, command):
    591 
    +        if command.label:
    592 
    +            context = self.sandbox._get_context()
    593 
    +            message = Message(self.sandbox._get_plugin_id(), MessageType.STATUS,
    594 
    +                              'Running {}'.format(command.label))
    595 
    +            context.message(message)
    596 
    +
    597 
    +        exitcode = self.sandbox._run(command.command, self.flags, cwd=command.cwd, env=command.env)
    598 
    +        if exitcode != 0:
    599 
    +            cmdline = ' '.join(shlex.quote(cmd) for cmd in command.command)
    600 
    +            label = command.label or cmdline
    601 
    +            raise SandboxCommandError("Command '{}' failed with exitcode {}".format(label, exitcode),
    602 
    +                                      collect=self.collect)
    603 
    +
    604 
    +    def execute_call(self, call):
    605 
    +        call.callback()
    606 
    +
    607 
    +
    608 
    +# _SandboxBatchItem()
    609 
    +#
    610 
    +# An item in a command batch.
    611 
    +#
    612 
    +class _SandboxBatchItem():
    613 
    +
    614 
    +    def __init__(self, *, label=None):
    615 
    +        self.label = label
    616 
    +
    617 
    +
    618 
    +# _SandboxBatchCommand()
    619 
    +#
    620 
    +# A command item in a command batch.
    621 
    +#
    622 
    +class _SandboxBatchCommand(_SandboxBatchItem):
    623 
    +
    624 
    +    def __init__(self, command, *, cwd, env, label=None):
    625 
    +        super().__init__(label=label)
    626 
    +
    627 
    +        self.command = command
    628 
    +        self.cwd = cwd
    629 
    +        self.env = env
    630 
    +
    631 
    +    def execute(self, batch):
    632 
    +        batch.execute_command(self)
    633 
    +
    634 
    +
    635 
    +# _SandboxBatchGroup()
    636 
    +#
    637 
    +# A group in a command batch.
    638 
    +#
    639 
    +class _SandboxBatchGroup(_SandboxBatchItem):
    640 
    +
    641 
    +    def __init__(self, *, label=None):
    642 
    +        super().__init__(label=label)
    643 
    +
    644 
    +        self.children = []
    645 
    +
    646 
    +    def append(self, item):
    647 
    +        self.children.append(item)
    648 
    +
    649 
    +    def execute(self, batch):
    650 
    +        batch.execute_group(self)
    651 
    +
    652 
    +    def execute_children(self, batch):
    653 
    +        for item in self.children:
    654 
    +            item.execute(batch)
    655 
    +
    656 
    +
    657 
    +# _SandboxBatchCall()
    658 
    +#
    659 
    +# A call item in a command batch.
    660 
    +#
    661 
    +class _SandboxBatchCall(_SandboxBatchItem):
    662 
    +
    663 
    +    def __init__(self, callback):
    664 
    +        super().__init__()
    665 
    +
    666 
    +        self.callback = callback
    667 
    +
    668 
    +    def execute(self, batch):
    669 
    +        batch.execute_call(self)

  • buildstream/scriptelement.py
    ... ... @@ -226,10 +226,11 @@ class ScriptElement(Element):
    226 226 
                                              .format(build_dep.name), silent_nested=True):
    227 227 
                         build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path="/")
    228 228
    229 
    -            for build_dep in self.dependencies(Scope.BUILD, recurse=False):
    230 
    -                with self.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True):
    231 
    -                    for dep in build_dep.dependencies(Scope.RUN):
    232 
    -                        dep.integrate(sandbox)
    229 
    +            with sandbox.batch(SandboxFlags.NONE):
    230 
    +                for build_dep in self.dependencies(Scope.BUILD, recurse=False):
    231 
    +                    with self.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True):
    232 
    +                        for dep in build_dep.dependencies(Scope.RUN):
    233 
    +                            dep.integrate(sandbox)
    233 234 
             else:
    234 235 
                 # If layout, follow its rules.
    235 236 
                 for item in self.__layout:
    ... ... @@ -251,37 +252,40 @@ class ScriptElement(Element):
    251 252 
                             virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True)
    252 253 
                             element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination'])
    253 254
    254 
    -            for item in self.__layout:
    255 
    +            with sandbox.batch(SandboxFlags.NONE):
    256 
    +                for item in self.__layout:
    255 257
    256 
    -                # Skip layout members which dont stage an element
    257 
    -                if not item['element']:
    258 
    -                    continue
    258 
    +                    # Skip layout members which dont stage an element
    259 
    +                    if not item['element']:
    260 
    +                        continue
    259 261
    260 
    -                element = self.search(Scope.BUILD, item['element'])
    262 
    +                    element = self.search(Scope.BUILD, item['element'])
    261 263
    262 
    -                # Integration commands can only be run for elements staged to /
    263 
    -                if item['destination'] == '/':
    264 
    -                    with self.timed_activity("Integrating {}".format(element.name),
    265 
    -                                             silent_nested=True):
    266 
    -                        for dep in element.dependencies(Scope.RUN):
    267 
    -                            dep.integrate(sandbox)
    264 
    +                    # Integration commands can only be run for elements staged to /
    265 
    +                    if item['destination'] == '/':
    266 
    +                        with self.timed_activity("Integrating {}".format(element.name),
    267 
    +                                                 silent_nested=True):
    268 
    +                            for dep in element.dependencies(Scope.RUN):
    269 
    +                                dep.integrate(sandbox)
    268 270
    269 271 
             install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep)
    270 272 
             sandbox.get_virtual_directory().descend(install_root_path_components, create=True)
    271 273
    272 274 
         def assemble(self, sandbox):
    273 275
    274 
    -        for groupname, commands in self.__commands.items():
    275 
    -            with self.timed_activity("Running '{}'".format(groupname)):
    276 
    -                for cmd in commands:
    277 
    -                    self.status("Running command", detail=cmd)
    278 
    -                    # Note the -e switch to 'sh' means to exit with an error
    279 
    -                    # if any untested command fails.
    280 
    -                    exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    281 
    -                                           SandboxFlags.ROOT_READ_ONLY if self.__root_read_only else 0)
    282 
    -                    if exitcode != 0:
    283 
    -                        raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
    284 
    -                                           collect=self.__install_root)
    276 
    +        flags = SandboxFlags.NONE
    277 
    +        if self.__root_read_only:
    278 
    +            flags |= SandboxFlags.ROOT_READ_ONLY
    279 
    +
    280 
    +        with sandbox.batch(flags, collect=self.__install_root):
    281 
    +            for groupname, commands in self.__commands.items():
    282 
    +                with sandbox.batch(flags, label="Running '{}'".format(groupname)):
    283 
    +                    for cmd in commands:
    284 
    +                        # Note the -e switch to 'sh' means to exit with an error
    285 
    +                        # if any untested command fails.
    286 
    +                        sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    287 
    +                                    flags,
    288 
    +                                    label=cmd)
    285 289
    286 290 
             # Return where the result can be collected from
    287 291 
             return self.__install_root

  • tests/integration/manual.py
    ... ... @@ -128,3 +128,28 @@ def test_manual_element_noparallel(cli, tmpdir, datafiles):
    128 128 
         assert text == """-j1 -Wall
    129 129 
     2
    130 130 
     """
    131 
    +
    132 
    +
    133 
    +@pytest.mark.datafiles(DATA_DIR)
    134 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    135 
    +def test_manual_element_logging(cli, tmpdir, datafiles):
    136 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    137 
    +    checkout = os.path.join(cli.directory, 'checkout')
    138 
    +    element_path = os.path.join(project, 'elements')
    139 
    +    element_name = 'import/import.bst'
    140 
    +
    141 
    +    create_manual_element(element_name, element_path, {
    142 
    +        'configure-commands': ["echo configure"],
    143 
    +        'build-commands': ["echo build"],
    144 
    +        'install-commands': ["echo install"],
    145 
    +        'strip-commands': ["echo strip"]
    146 
    +    }, {}, {})
    147 
    +
    148 
    +    res = cli.run(project=project, args=['build', element_name])
    149 
    +    assert res.exit_code == 0
    150 
    +
    151 
    +    # Verify that individual commands are logged
    152 
    +    assert "echo configure" in res.stderr
    153 
    +    assert "echo build" in res.stderr
    154 
    +    assert "echo install" in res.stderr
    155 
    +    assert "echo strip" in res.stderr

  • tests/integration/sandbox-bwrap.py
    ... ... @@ -58,5 +58,5 @@ def test_sandbox_bwrap_return_subprocess(cli, tmpdir, datafiles):
    58 58 
         })
    59 59
    60 60 
         result = cli.run(project=project, args=['build', element_name])
    61 
    -    result.assert_task_error(error_domain=ErrorDomain.ELEMENT, error_reason=None)
    61 
    +    result.assert_task_error(error_domain=ErrorDomain.SANDBOX, error_reason="command-failed")
    62 62 
         assert "sandbox-bwrap/command-exit-42.bst|Command 'exit 42' failed with exitcode 42" in result.stderr

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]