[Notes] [Git][BuildStream/buildstream][mac_fixes] Adding Dummy sandbox and nolocal platform



Title: GitLab

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

5 changed files:

Changes:

  • buildstream/_platform/darwin.py
    ... ... @@ -19,7 +19,7 @@ import os
    19 19
     import resource
    
    20 20
     
    
    21 21
     from .._exceptions import PlatformError
    
    22
    -from ..sandbox import SandboxChroot
    
    22
    +from ..sandbox import SandboxChroot, DummySandbox
    
    23 23
     
    
    24 24
     from . import Platform
    
    25 25
     
    
    ... ... @@ -38,7 +38,10 @@ class Darwin(Platform):
    38 38
             return self._artifact_cache
    
    39 39
     
    
    40 40
         def create_sandbox(self, *args, **kwargs):
    
    41
    -        return SandboxChroot(*args, **kwargs)
    
    41
    +        if os.path.exists('/dev/fuse'):
    
    42
    +            return SandboxChroot(*args, **kwargs)
    
    43
    +        else:
    
    44
    +            return DummySandbox(*args, **kwargs)
    
    42 45
     
    
    43 46
         def get_cpu_count(self, cap=None):
    
    44 47
             if cap < os.cpu_count():
    

  • buildstream/_platform/linux.py
    ... ... @@ -24,7 +24,7 @@ from .. import _site
    24 24
     from .. import utils
    
    25 25
     from .._artifactcache.cascache import CASCache
    
    26 26
     from .._message import Message, MessageType
    
    27
    -from ..sandbox import SandboxBwrap
    
    27
    +from ..sandbox import SandboxBwrap, DummySandbox
    
    28 28
     
    
    29 29
     from . import Platform
    
    30 30
     
    
    ... ... @@ -48,10 +48,13 @@ class Linux(Platform):
    48 48
             return self._artifact_cache
    
    49 49
     
    
    50 50
         def create_sandbox(self, *args, **kwargs):
    
    51
    -        # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
    
    52
    -        kwargs['user_ns_available'] = self._user_ns_available
    
    53
    -        kwargs['die_with_parent_available'] = self._die_with_parent_available
    
    54
    -        return SandboxBwrap(*args, **kwargs)
    
    51
    +        if not (os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')):
    
    52
    +            return DummySandbox(*args, **kwargs)
    
    53
    +        else:
    
    54
    +            # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
    
    55
    +            kwargs['user_ns_available'] = self._user_ns_available
    
    56
    +            kwargs['die_with_parent_available'] = self._die_with_parent_available
    
    57
    +            return SandboxBwrap(*args, **kwargs)
    
    55 58
     
    
    56 59
         ################################################
    
    57 60
         #              Private Methods                 #
    

  • buildstream/_platform/platform.py
    ... ... @@ -54,6 +54,8 @@ class Platform():
    54 54
                 backend = 'linux'
    
    55 55
             elif sys.platform.startswith('darwin'):
    
    56 56
                 backend = 'darwin'
    
    57
    +        elif not (os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')):
    
    58
    +            backend = 'no_local'
    
    57 59
             else:
    
    58 60
                 backend = 'unix'
    
    59 61
     
    
    ... ... @@ -63,6 +65,8 @@ class Platform():
    63 65
                 from .darwin import Darwin as PlatformImpl
    
    64 66
             elif backend == 'unix':
    
    65 67
                 from .unix import Unix as PlatformImpl
    
    68
    +        elif backend == 'no_local':
    
    69
    +            from .nolocal import Nolocal as PlatformImpl
    
    66 70
             else:
    
    67 71
                 raise PlatformError("No such platform: '{}'".format(backend))
    
    68 72
     
    

  • buildstream/sandbox/__init__.py
    ... ... @@ -21,3 +21,4 @@ from .sandbox import Sandbox, SandboxFlags
    21 21
     from ._sandboxchroot import SandboxChroot
    
    22 22
     from ._sandboxbwrap import SandboxBwrap
    
    23 23
     from ._sandboxremote import SandboxRemote
    
    24
    +from ._dummysandbox import DummySandbox

  • buildstream/sandbox/_dummysandbox.py
    1
    +#
    
    2
    +#  Copyright (C) 2017 Codethink Limited
    
    3
    +#
    
    4
    +#  This program is free software; you can redistribute it and/or
    
    5
    +#  modify it under the terms of the GNU Lesser General Public
    
    6
    +#  License as published by the Free Software Foundation; either
    
    7
    +#  version 2 of the License, or (at your option) any later version.
    
    8
    +#
    
    9
    +#  This library is distributed in the hope that it will be useful,
    
    10
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    
    11
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    
    12
    +#  Lesser General Public License for more details.
    
    13
    +#
    
    14
    +#  You should have received a copy of the GNU Lesser General Public
    
    15
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    
    16
    +#
    
    17
    +#  Authors:
    
    18
    +#        Tristan Maat <tristan maat codethink co uk>
    
    19
    +#        Tristan Van Berkom <tristan vanberkom codethink co uk>
    
    20
    +
    
    21
    +import os
    
    22
    +import sys
    
    23
    +import stat
    
    24
    +import signal
    
    25
    +import subprocess
    
    26
    +from contextlib import contextmanager, ExitStack
    
    27
    +import psutil
    
    28
    +
    
    29
    +from .._exceptions import SandboxError
    
    30
    +from .. import utils
    
    31
    +from .. import _signals
    
    32
    +from ._mounter import Mounter
    
    33
    +from ._mount import MountMap
    
    34
    +from . import Sandbox, SandboxFlags
    
    35
    +
    
    36
    +
    
    37
    +class DummySandbox(Sandbox):
    
    38
    +    def __init__(self, *args, **kwargs):
    
    39
    +        super().__init__(*args, **kwargs)
    
    40
    +
    
    41
    +        uid = self._get_config().build_uid
    
    42
    +        gid = self._get_config().build_gid
    
    43
    +        if uid != 0 or gid != 0:
    
    44
    +            raise SandboxError("Chroot sandboxes cannot specify a non-root uid/gid "
    
    45
    +                               "({},{} were supplied via config)".format(uid, gid))
    
    46
    +
    
    47
    +        self.mount_map = None
    
    48
    +
    
    49
    +    def run(self, command, flags, *, cwd=None, env=None):
    
    50
    +
    
    51
    +        # Default settings
    
    52
    +        if cwd is None:
    
    53
    +            cwd = self._get_work_directory()
    
    54
    +
    
    55
    +        if cwd is None:
    
    56
    +            cwd = '/'
    
    57
    +
    
    58
    +        if env is None:
    
    59
    +            env = self._get_environment()
    
    60
    +
    
    61
    +        # Naive getcwd implementations can break when bind-mounts to different
    
    62
    +        # paths on the same filesystem are present. Letting the command know
    
    63
    +        # what directory it is in makes it unnecessary to call the faulty
    
    64
    +        # getcwd.
    
    65
    +        env['PWD'] = cwd
    
    66
    +
    
    67
    +        if not self._has_command(command[0], env):
    
    68
    +            raise SandboxError("Staged artifacts do not provide command "
    
    69
    +                               "'{}'".format(command[0]),
    
    70
    +                               reason='missing-command')
    
    71
    +
    
    72
    +        # Command must be a list
    
    73
    +        if isinstance(command, str):
    
    74
    +            command = [command]
    
    75
    +
    
    76
    +        stdout, stderr = self._get_output()
    
    77
    +
    
    78
    +        # Create the mount map, this will tell us where
    
    79
    +        # each mount point needs to be mounted from and to
    
    80
    +        self.mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY)
    
    81
    +        root_mount_source = self.mount_map.get_mount_source('/')
    
    82
    +
    
    83
    +        # Create a sysroot and run the command inside it
    
    84
    +        with ExitStack() as stack:
    
    85
    +            os.makedirs('/var/run/buildstream', exist_ok=True)
    
    86
    +
    
    87
    +            # FIXME: While we do not currently do anything to prevent
    
    88
    +            # network access, we also don't copy /etc/resolv.conf to
    
    89
    +            # the new rootfs.
    
    90
    +            #
    
    91
    +            # This effectively disables network access, since DNs will
    
    92
    +            # never resolve, so anything a normal process wants to do
    
    93
    +            # will fail. Malicious processes could gain rights to
    
    94
    +            # anything anyway.
    
    95
    +            #
    
    96
    +            # Nonetheless a better solution could perhaps be found.
    
    97
    +
    
    98
    +            rootfs = stack.enter_context(utils._tempdir(dir='/var/run/buildstream'))
    
    99
    +            stack.enter_context(self.create_devices(self._root, flags))
    
    100
    +            stack.enter_context(self.mount_dirs(rootfs, flags, stdout, stderr))
    
    101
    +
    
    102
    +            if flags & SandboxFlags.INTERACTIVE:
    
    103
    +                stdin = sys.stdin
    
    104
    +            else:
    
    105
    +                stdin = stack.enter_context(open(os.devnull, 'r'))
    
    106
    +
    
    107
    +            # Ensure the cwd exists
    
    108
    +            if cwd is not None:
    
    109
    +                workdir = os.path.join(root_mount_source, cwd.lstrip(os.sep))
    
    110
    +                os.makedirs(workdir, exist_ok=True)
    
    111
    +
    
    112
    +            status = self.chroot(rootfs, command, stdin, stdout,
    
    113
    +                                 stderr, cwd, env, flags)
    
    114
    +
    
    115
    +        self._vdir._mark_changed()
    
    116
    +        return status
    
    117
    +
    
    118
    +    # chroot()
    
    119
    +    #
    
    120
    +    # A helper function to chroot into the rootfs.
    
    121
    +    #
    
    122
    +    # Args:
    
    123
    +    #    rootfs (str): The path of the sysroot to chroot into
    
    124
    +    #    command (list): The command to execute in the chroot env
    
    125
    +    #    stdin (file): The stdin
    
    126
    +    #    stdout (file): The stdout
    
    127
    +    #    stderr (file): The stderr
    
    128
    +    #    cwd (str): The current working directory
    
    129
    +    #    env (dict): The environment variables to use while executing the command
    
    130
    +    #    flags (:class:`SandboxFlags`): The flags to enable on the sandbox
    
    131
    +    #
    
    132
    +    # Returns:
    
    133
    +    #    (int): The exit code of the executed command
    
    134
    +    #
    
    135
    +    def chroot(self, rootfs, command, stdin, stdout, stderr, cwd, env, flags):
    
    136
    +        raise SandboxError("This platform does not support local builds")
    
    137
    +
    
    138
    +        def kill_proc():
    
    139
    +            if process:
    
    140
    +                # First attempt to gracefully terminate
    
    141
    +                proc = psutil.Process(process.pid)
    
    142
    +                proc.terminate()
    
    143
    +
    
    144
    +                try:
    
    145
    +                    proc.wait(20)
    
    146
    +                except psutil.TimeoutExpired:
    
    147
    +                    utils._kill_process_tree(process.pid)
    
    148
    +
    
    149
    +        def suspend_proc():
    
    150
    +            group_id = os.getpgid(process.pid)
    
    151
    +            os.killpg(group_id, signal.SIGSTOP)
    
    152
    +
    
    153
    +        def resume_proc():
    
    154
    +            group_id = os.getpgid(process.pid)
    
    155
    +            os.killpg(group_id, signal.SIGCONT)
    
    156
    +
    
    157
    +        try:
    
    158
    +            with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
    
    159
    +                process = subprocess.Popen(
    
    160
    +                    command,
    
    161
    +                    close_fds=True,
    
    162
    +                    cwd=os.path.join(rootfs, cwd.lstrip(os.sep)),
    
    163
    +                    env=env,
    
    164
    +                    stdin=stdin,
    
    165
    +                    stdout=stdout,
    
    166
    +                    stderr=stderr,
    
    167
    +                    # If you try to put gtk dialogs here Tristan (either)
    
    168
    +                    # will personally scald you
    
    169
    +                    preexec_fn=lambda: (os.chroot(rootfs), os.chdir(cwd)),
    
    170
    +                    start_new_session=flags & SandboxFlags.INTERACTIVE
    
    171
    +                )
    
    172
    +
    
    173
    +                # Wait for the child process to finish, ensuring that
    
    174
    +                # a SIGINT has exactly the effect the user probably
    
    175
    +                # expects (i.e. let the child process handle it).
    
    176
    +                try:
    
    177
    +                    while True:
    
    178
    +                        try:
    
    179
    +                            _, status = os.waitpid(process.pid, 0)
    
    180
    +                            # If the process exits due to a signal, we
    
    181
    +                            # brutally murder it to avoid zombies
    
    182
    +                            if not os.WIFEXITED(status):
    
    183
    +                                utils._kill_process_tree(process.pid)
    
    184
    +
    
    185
    +                        # Unlike in the bwrap case, here only the main
    
    186
    +                        # process seems to receive the SIGINT. We pass
    
    187
    +                        # on the signal to the child and then continue
    
    188
    +                        # to wait.
    
    189
    +                        except KeyboardInterrupt:
    
    190
    +                            process.send_signal(signal.SIGINT)
    
    191
    +                            continue
    
    192
    +
    
    193
    +                        break
    
    194
    +                # If we can't find the process, it has already died of
    
    195
    +                # its own accord, and therefore we don't need to check
    
    196
    +                # or kill anything.
    
    197
    +                except psutil.NoSuchProcess:
    
    198
    +                    pass
    
    199
    +
    
    200
    +                # Return the exit code - see the documentation for
    
    201
    +                # os.WEXITSTATUS to see why this is required.
    
    202
    +                if os.WIFEXITED(status):
    
    203
    +                    code = os.WEXITSTATUS(status)
    
    204
    +                else:
    
    205
    +                    code = -1
    
    206
    +
    
    207
    +        except subprocess.SubprocessError as e:
    
    208
    +            # Exceptions in preexec_fn are simply reported as
    
    209
    +            # 'Exception occurred in preexec_fn', turn these into
    
    210
    +            # a more readable message.
    
    211
    +            if '{}'.format(e) == 'Exception occurred in preexec_fn.':
    
    212
    +                raise SandboxError('Could not chroot into {} or chdir into {}. '
    
    213
    +                                   'Ensure you are root and that the relevant directory exists.'
    
    214
    +                                   .format(rootfs, cwd)) from e
    
    215
    +            else:
    
    216
    +                raise SandboxError('Could not run command {}: {}'.format(command, e)) from e
    
    217
    +
    
    218
    +        return code
    
    219
    +
    
    220
    +    # create_devices()
    
    221
    +    #
    
    222
    +    # Create the nodes in /dev/ usually required for builds (null,
    
    223
    +    # none, etc.)
    
    224
    +    #
    
    225
    +    # Args:
    
    226
    +    #    rootfs (str): The path of the sysroot to prepare
    
    227
    +    #    flags (:class:`.SandboxFlags`): The sandbox flags
    
    228
    +    #
    
    229
    +    @contextmanager
    
    230
    +    def create_devices(self, rootfs, flags):
    
    231
    +
    
    232
    +        devices = []
    
    233
    +        # When we are interactive, we'd rather mount /dev due to the
    
    234
    +        # sheer number of devices
    
    235
    +        if not flags & SandboxFlags.INTERACTIVE:
    
    236
    +
    
    237
    +            for device in Sandbox.DEVICES:
    
    238
    +                location = os.path.join(rootfs, device.lstrip(os.sep))
    
    239
    +                os.makedirs(os.path.dirname(location), exist_ok=True)
    
    240
    +                try:
    
    241
    +                    if os.path.exists(location):
    
    242
    +                        os.remove(location)
    
    243
    +
    
    244
    +                    devices.append(self.mknod(device, location))
    
    245
    +                except OSError as err:
    
    246
    +                    if err.errno == 1:
    
    247
    +                        raise SandboxError("Permission denied while creating device node: {}.".format(err) +
    
    248
    +                                           "BuildStream reqiures root permissions for these setttings.")
    
    249
    +                    else:
    
    250
    +                        raise
    
    251
    +
    
    252
    +        yield
    
    253
    +
    
    254
    +        for device in devices:
    
    255
    +            os.remove(device)
    
    256
    +
    
    257
    +    # mount_dirs()
    
    258
    +    #
    
    259
    +    # Mount paths required for the command.
    
    260
    +    #
    
    261
    +    # Args:
    
    262
    +    #    rootfs (str): The path of the sysroot to prepare
    
    263
    +    #    flags (:class:`.SandboxFlags`): The sandbox flags
    
    264
    +    #    stdout (file): The stdout
    
    265
    +    #    stderr (file): The stderr
    
    266
    +    #
    
    267
    +    @contextmanager
    
    268
    +    def mount_dirs(self, rootfs, flags, stdout, stderr):
    
    269
    +
    
    270
    +        # FIXME: This should probably keep track of potentially
    
    271
    +        #        already existing files a la _sandboxwrap.py:239
    
    272
    +
    
    273
    +        @contextmanager
    
    274
    +        def mount_point(point, **kwargs):
    
    275
    +            mount_source_overrides = self._get_mount_sources()
    
    276
    +            if point in mount_source_overrides:
    
    277
    +                mount_source = mount_source_overrides[point]
    
    278
    +            else:
    
    279
    +                mount_source = self.mount_map.get_mount_source(point)
    
    280
    +            mount_point = os.path.join(rootfs, point.lstrip(os.sep))
    
    281
    +
    
    282
    +            with Mounter.bind_mount(mount_point, src=mount_source, stdout=stdout, stderr=stderr, **kwargs):
    
    283
    +                yield
    
    284
    +
    
    285
    +        @contextmanager
    
    286
    +        def mount_src(src, **kwargs):
    
    287
    +            mount_point = os.path.join(rootfs, src.lstrip(os.sep))
    
    288
    +            os.makedirs(mount_point, exist_ok=True)
    
    289
    +
    
    290
    +            with Mounter.bind_mount(mount_point, src=src, stdout=stdout, stderr=stderr, **kwargs):
    
    291
    +                yield
    
    292
    +
    
    293
    +        with ExitStack() as stack:
    
    294
    +            stack.enter_context(self.mount_map.mounted(self))
    
    295
    +
    
    296
    +            stack.enter_context(mount_point('/'))
    
    297
    +
    
    298
    +            if flags & SandboxFlags.INTERACTIVE:
    
    299
    +                stack.enter_context(mount_src('/dev'))
    
    300
    +
    
    301
    +            stack.enter_context(mount_src('/tmp'))
    
    302
    +            stack.enter_context(mount_src('/proc'))
    
    303
    +
    
    304
    +            for mark in self._get_marked_directories():
    
    305
    +                stack.enter_context(mount_point(mark['directory']))
    
    306
    +
    
    307
    +            # Remount root RO if necessary
    
    308
    +            if flags & flags & SandboxFlags.ROOT_READ_ONLY:
    
    309
    +                root_mount = Mounter.mount(rootfs, stdout=stdout, stderr=stderr, remount=True, ro=True, bind=True)
    
    310
    +                # Since the exit stack has already registered a mount
    
    311
    +                # for this path, we do not need to register another
    
    312
    +                # umount call.
    
    313
    +                root_mount.__enter__()
    
    314
    +
    
    315
    +            yield
    
    316
    +
    
    317
    +    # mknod()
    
    318
    +    #
    
    319
    +    # Create a device node equivalent to the given source node
    
    320
    +    #
    
    321
    +    # Args:
    
    322
    +    #    source (str): Path of the device to mimic (e.g. '/dev/null')
    
    323
    +    #    target (str): Location to create the new device in
    
    324
    +    #
    
    325
    +    # Returns:
    
    326
    +    #    target (str): The location of the created node
    
    327
    +    #
    
    328
    +    def mknod(self, source, target):
    
    329
    +        try:
    
    330
    +            dev = os.stat(source)
    
    331
    +            major = os.major(dev.st_rdev)
    
    332
    +            minor = os.minor(dev.st_rdev)
    
    333
    +
    
    334
    +            target_dev = os.makedev(major, minor)
    
    335
    +
    
    336
    +            os.mknod(target, mode=stat.S_IFCHR | dev.st_mode, device=target_dev)
    
    337
    +
    
    338
    +        except PermissionError as e:
    
    339
    +            raise SandboxError('Could not create device {}, ensure that you have root permissions: {}')
    
    340
    +
    
    341
    +        except OSError as e:
    
    342
    +            raise SandboxError('Could not create device {}: {}'
    
    343
    +                               .format(target, e)) from e
    
    344
    +
    
    345
    +        return target



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]