GNOME.org
 

[Notes] [Git][BuildStream/buildstream][raoul/775-execution-environment-reqs] 8 commits: _sandboxremote.py: Add os and arch to command uploaded

Title: GitLab

Raoul Hidalgo Charman pushed to branch raoul/775-execution-environment-reqs at BuildStream / buildstream

Commits:

11 changed files:

Changes:

  • buildstream/_options/optionos.py
    1 
    +
    2 
    +#
    3 
    +#  Copyright (C) 2017 Codethink Limited
    4 
    +#
    5 
    +#  This program is free software; you can redistribute it and/or
    6 
    +#  modify it under the terms of the GNU Lesser General Public
    7 
    +#  License as published by the Free Software Foundation; either
    8 
    +#  version 2 of the License, or (at your option) any later version.
    9 
    +#
    10 
    +#  This library is distributed in the hope that it will be useful,
    11 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    12 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    13 
    +#  Lesser General Public License for more details.
    14 
    +#
    15 
    +#  You should have received a copy of the GNU Lesser General Public
    16 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    17 
    +#
    18 
    +#  Authors:
    19 
    +#        Raoul Hidalgo Charman <raoul hidalgocharman codethink co uk>
    20 
    +
    21 
    +import os
    22 
    +from .optionenum import OptionEnum
    23 
    +
    24 
    +
    25 
    +# OptionOS
    26 
    +#
    27 
    +class OptionOS(OptionEnum):
    28 
    +
    29 
    +    OPTION_TYPE = 'os'
    30 
    +
    31 
    +    def load(self, node):
    32 
    +        super(OptionOS, self).load(node, allow_default_definition=False)
    33 
    +
    34 
    +    def load_default_value(self, node):
    35 
    +        return os.uname()[0]
    36 
    +
    37 
    +    def resolve(self):
    38 
    +
    39 
    +        # Validate that the default OS reported by uname() is explicitly
    40 
    +        # supported by the project, if not overridden by user config or cli.
    41 
    +        self.validate(self.value)

  • buildstream/_options/optionpool.py
    ... ... @@ -28,6 +28,7 @@ from .optionenum import OptionEnum
    28 28 
     from .optionflags import OptionFlags
    29 29 
     from .optioneltmask import OptionEltMask
    30 30 
     from .optionarch import OptionArch
    31 
    +from .optionos import OptionOS
    31 32
    32 33
    33 34 
     _OPTION_TYPES = {
    ... ... @@ -36,6 +37,7 @@ _OPTION_TYPES = {
    36 37 
         OptionFlags.OPTION_TYPE: OptionFlags,
    37 38 
         OptionEltMask.OPTION_TYPE: OptionEltMask,
    38 39 
         OptionArch.OPTION_TYPE: OptionArch,
    40 
    +    OptionOS.OPTION_TYPE: OptionOS,
    39 41 
     }
    40 42
    41 43

  • buildstream/_platform/linux.py
    ... ... @@ -25,6 +25,7 @@ from .. import utils
    25 25 
     from ..sandbox import SandboxDummy
    26 26
    27 27 
     from . import Platform
    28 
    +from .._exceptions import PlatformError
    28 29
    29 30
    30 31 
     class Linux(Platform):
    ... ... @@ -58,6 +59,9 @@ class Linux(Platform):
    58 59 
             else:
    59 60 
                 self._user_ns_available = False
    60 61
    62 
    +        # Set linux32 option
    63 
    +        self._linux32 = False
    64 
    +
    61 65 
         def create_sandbox(self, *args, **kwargs):
    62 66 
             if not self._local_sandbox_available:
    63 67 
                 return self._create_dummy_sandbox(*args, **kwargs)
    ... ... @@ -71,11 +75,23 @@ class Linux(Platform):
    71 75
    72 76 
             if self._user_ns_available:
    73 77 
                 # User namespace support allows arbitrary build UID/GID settings.
    74 
    -            return True
    75 
    -        else:
    78 
    +            pass
    79 
    +        elif (config.build_uid != self._uid or config.build_gid != self._gid):
    76 80 
                 # Without user namespace support, the UID/GID in the sandbox
    77 81 
                 # will match the host UID/GID.
    78 
    -            return config.build_uid == self._uid and config.build_gid == self._gid
    82 
    +            return False
    83 
    +
    84 
    +        # We can't do builds for another host or architecture except x86-32 on
    85 
    +        # x86-64
    86 
    +        if config.build_os != self.get_host_os():
    87 
    +            raise PlatformError("Configured and host OS don't match.")
    88 
    +        elif config.build_arch != self.get_host_arch():
    89 
    +            if config.build_arch == "x86-32" and self.get_host_arch() == "x86-64":
    90 
    +                self._linux32 = True
    91 
    +            else:
    92 
    +                raise PlatformError("Configured architecture and host architecture don't match.")
    93 
    +
    94 
    +        return True
    79 95
    80 96 
         ################################################
    81 97 
         #              Private Methods                 #
    ... ... @@ -100,6 +116,7 @@ class Linux(Platform):
    100 116 
             kwargs['user_ns_available'] = self._user_ns_available
    101 117 
             kwargs['die_with_parent_available'] = self._die_with_parent_available
    102 118 
             kwargs['json_status_available'] = self._json_status_available
    119 
    +        kwargs['linux32'] = self._linux32
    103 120 
             return SandboxBwrap(*args, **kwargs)
    104 121
    105 122 
         def _check_user_ns_available(self):

  • buildstream/_platform/unix.py
    ... ... @@ -44,4 +44,13 @@ class Unix(Platform):
    44 44 
         def check_sandbox_config(self, config):
    45 45 
             # With the chroot sandbox, the UID/GID in the sandbox
    46 46 
             # will match the host UID/GID (typically 0/0).
    47 
    -        return config.build_uid == self._uid and config.build_gid == self._gid
    47 
    +        if config.build_uid != self._uid or config.build_gid != self._gid:
    48 
    +            return False
    49 
    +
    50 
    +        # Check host os and architecture match
    51 
    +        if config.build_os != self.get_host_os():
    52 
    +            raise PlatformError("Configured and host OS don't match.")
    53 
    +        elif config.build_arch != self.get_host_arch():
    54 
    +            raise PlatformError("Configured and host architecture don't match.")
    55 
    +
    56 
    +        return True

  • buildstream/sandbox/_sandboxbwrap.py
    ... ... @@ -57,6 +57,7 @@ class SandboxBwrap(Sandbox):
    57 57 
             self.user_ns_available = kwargs['user_ns_available']
    58 58 
             self.die_with_parent_available = kwargs['die_with_parent_available']
    59 59 
             self.json_status_available = kwargs['json_status_available']
    60 
    +        self.linux32 = kwargs['linux32']
    60 61
    61 62 
         def run(self, command, flags, *, cwd=None, env=None):
    62 63 
             stdout, stderr = self._get_output()
    ... ... @@ -84,8 +85,14 @@ class SandboxBwrap(Sandbox):
    84 85 
             mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY)
    85 86 
             root_mount_source = mount_map.get_mount_source('/')
    86 87
    88 
    +        # start command with linux32 if needed
    89 
    +        if self.linux32:
    90 
    +            bwrap_command = [utils.get_host_tool('linux32')]
    91 
    +        else:
    92 
    +            bwrap_command = []
    93 
    +
    87 94 
             # Grab the full path of the bwrap binary
    88 
    -        bwrap_command = [utils.get_host_tool('bwrap')]
    95 
    +        bwrap_command += [utils.get_host_tool('bwrap')]
    89 96
    90 97 
             for k, v in env.items():
    91 98 
                 bwrap_command += ['--setenv', k, v]

  • buildstream/sandbox/_sandboxremote.py
    ... ... @@ -65,13 +65,20 @@ class SandboxRemote(Sandbox):
    65 65 
                                      EnvironmentVariable(name=k, value=v)
    66 66 
                                      for (k, v) in environment.items()]
    67 67
    68 
    +        config = self._get_config()
    69 
    +        platform = remote_execution_pb2.Platform()
    70 
    +        platform.properties.extend([remote_execution_pb2.Platform.
    71 
    +                                    Property(name="OSFamily", value=config.build_os),
    72 
    +                                    remote_execution_pb2.Platform.
    73 
    +                                    Property(name="ISA", value=config.build_arch)])
    74 
    +
    68 75 
             # Create and send the Command object.
    69 76 
             remote_command = remote_execution_pb2.Command(arguments=command,
    70 77 
                                                           working_directory=working_directory,
    71 78 
                                                           environment_variables=environment_variables,
    72 79 
                                                           output_files=[],
    73 80 
                                                           output_directories=[self._output_directory],
    74 
    -                                                      platform=None)
    81 
    +                                                      platform=platform)
    75 82 
             context = self._get_context()
    76 83 
             cascache = context.artifactcache
    77 84 
             # Upload the Command message to the remote CAS server

  • doc/source/format_declaring.rst
    ... ... @@ -294,8 +294,10 @@ can be viewed in detail in the :ref:`builtin public data <public_builtin>` secti
    294 294 
     Sandbox
    295 295 
     ~~~~~~~
    296 296 
     Configuration for the build sandbox (other than :ref:`environment variables <format_environment>`)
    297 
    -can be placed in the ``sandbox`` configuration. At present, only the
    298 
    -UID and GID used by the user in the group can be specified.
    297 
    +can be placed in the ``sandbox`` configuration. The UID and GID used by the user
    298 
    +in the group can be specified, as well as the desired OS and machine
    299 
    +architecture. Possible machine architecture follow the same list as specified in
    300 
    +the :ref:`architecture option <project_options_arch>`.
    299 301
    300 302 
     .. code:: yaml
    301 303
    ... ... @@ -311,6 +313,23 @@ you can supply a different uid or gid for the sandbox. Only
    311 313 
     bwrap-style sandboxes support custom user IDs at the moment, and hence
    312 314 
     this will only work on Linux host platforms.
    313 315
    316 
    +.. code:: yaml
    317 
    +
    318 
    +   # Specify build OS and architecture
    319 
    +   sandbox:
    320 
    +     build-os: AIX
    321 
    +     build-arch: Power-ISA-BE
    322 
    +
    323 
    +When building locally, if these don't match the host machine then generally the
    324 
    +build will fail. The exception is when the OS is Linux and the architecture
    325 
    +specifies an ``x86-32`` build on an ``x86-64`` machine, in which case the
    326 
    +``linux32`` command is prepended to the bubblewrap command.
    327 
    +
    328 
    +When building remotely, the OS and architecture are added to the ``Platform``
    329 
    +field in the ``Command`` uploaded. Whether this actually results in a building
    330 
    +the element for the desired OS and architecture is dependent on the server
    331 
    +having implemented these options the same as buildstream.
    332 
    +
    314 333 
     .. note::
    315 334
    316 335 
        The ``sandbox`` configuration is available since :ref:`format version 6 <project_format_version>`

  • doc/source/format_project.rst
    ... ... @@ -538,9 +538,22 @@ exported as a comma separated list of selected value strings.
    538 538
    539 539 
     Architecture
    540 540 
     ~~~~~~~~~~~~
    541 
    -The ``arch`` option type is special enumeration option which
    542 
    -defaults to the result of `uname -m`, and does not support
    543 
    -assigning any default in the project configuration.
    541 
    +The ``arch`` option type is a special enumeration option which defaults via
    542 
    +`uname -m` results to the following list.
    543 
    +
    544 
    +* AArch32
    545 
    +* AArch64
    546 
    +* AArch64-BE
    547 
    +* Power-ISA-BE
    548 
    +* Power-ISA-LE
    549 
    +* Sparc-V9
    550 
    +* x86-32
    551 
    +* x86-64
    552 
    +
    553 
    +The reason for this, opposed to using just `uname -m`, is that we want an
    554 
    +OS-independent list, as well as several results mapping to the same architecture
    555 
    +(e.g. i386, i486 etc. are all x86-32). It does not support assigning any default
    556 
    +in the project configuration.
    544 557
    545 558 
     .. code:: yaml
    546 559
    ... ... @@ -549,16 +562,40 @@ assigning any default in the project configuration.
    549 562 
            type: arch
    550 563 
            description: The machine architecture
    551 564 
            values:
    552 
    -       - arm
    553 
    -       - aarch64
    554 
    -       - i386
    555 
    -       - x86_64
    565 
    +       - AArch32
    566 
    +       - AArch64
    567 
    +       - x86-32
    568 
    +       - x86-64
    556 569
    557 570
    558 571 
     Architecture options can be tested with the same expressions
    559 572 
     as other Enumeration options.
    560 573
    561 574
    575 
    +.. _project_options_os:
    576 
    +
    577 
    +OS
    578 
    +~~
    579 
    +
    580 
    +The ``os`` option type is a special enumeration option, which defaults to the
    581 
    +results of `uname -s`. It does not support assigning any default in the project
    582 
    +configuration.
    583 
    +
    584 
    +.. code:: yaml
    585 
    +
    586 
    +    options:
    587 
    +      machine_os:
    588 
    +	type: os
    589 
    +	description: The machine OS
    590 
    +	values:
    591 
    +        - Linux
    592 
    +	- SunOS
    593 
    +	- Darwin
    594 
    +	- FreeBSD
    595 
    +
    596 
    +Os options can be tested with the same expressions as other Enumeration options.
    597 
    +
    598 
    +
    562 599 
     .. _project_options_element_mask:
    563 600
    564 601 
     Element mask

  • tests/format/option-os/element.bst
    1 
    +kind: autotools
    2 
    +variables:
    3 
    +  result: "Nothing"
    4 
    +  (?):
    5 
    +  - machine_os == "Linux":
    6 
    +      result: "Linuxy"
    7 
    +  - machine_os == "Darwin":
    8 
    +      result: "Darwiny"
    9 
    +  - machine_os == "SunOS":
    10 
    +      result: "SunOSy"
    11 
    +  - machine_os == "FreeBSD":
    12 
    +      result: "FreeBSDy"
    \ No newline at end of file

  • tests/format/option-os/project.conf
    1 
    +name: test
    2 
    +
    3 
    +options:
    4 
    +  machine_os:
    5 
    +    type: os
    6 
    +    description: The operating system
    7 
    +    values:
    8 
    +    - Linux
    9 
    +    - Darwin
    10 
    +    - FreeBSD
    11 
    +    - SunOS

  • tests/format/optionos.py
    1 
    +import os
    2 
    +import pytest
    3 
    +from contextlib import contextmanager
    4 
    +
    5 
    +from buildstream import _yaml
    6 
    +from buildstream._exceptions import ErrorDomain, LoadErrorReason
    7 
    +from tests.testutils.runcli import cli
    8 
    +
    9 
    +DATA_DIR = os.path.dirname(os.path.realpath(__file__))
    10 
    +
    11 
    +
    12 
    +@contextmanager
    13 
    +def override_uname_os(name):
    14 
    +    orig_uname = os.uname
    15 
    +    orig_tuple = tuple(os.uname())
    16 
    +    override_result = (name, orig_tuple[1],
    17 
    +                       orig_tuple[2], orig_tuple[3],
    18 
    +                       orig_tuple[4])
    19 
    +
    20 
    +    def override():
    21 
    +        return override_result
    22 
    +
    23 
    +    os.uname = override
    24 
    +    yield
    25 
    +    os.uname = orig_uname
    26 
    +
    27 
    +
    28 
    +@pytest.mark.datafiles(DATA_DIR)
    29 
    +@pytest.mark.parametrize("uname,value,expected", [
    30 
    +    # Test explicitly provided arches
    31 
    +    ('Darwin', 'Linux', 'Linuxy'),
    32 
    +    ('SunOS', 'FreeBSD', 'FreeBSDy'),
    33 
    +
    34 
    +    # Test automatically derived arches
    35 
    +    ('Linux', None, 'Linuxy'),
    36 
    +    ('Darwin', None, 'Darwiny'),
    37 
    +
    38 
    +    # Test that explicitly provided arches dont error out
    39 
    +    # when the `uname` reported arch is not supported
    40 
    +    ('AIX', 'Linux', 'Linuxy'),
    41 
    +    ('HaikuOS', 'SunOS', 'SunOSy'),
    42 
    +])
    43 
    +def test_conditionals(cli, datafiles, uname, value, expected):
    44 
    +    with override_uname_os(uname):
    45 
    +        project = os.path.join(datafiles.dirname, datafiles.basename, 'option-os')
    46 
    +
    47 
    +        bst_args = []
    48 
    +        if value is not None:
    49 
    +            bst_args += ['--option', 'machine_os', value]
    50 
    +
    51 
    +        bst_args += [
    52 
    +            'show',
    53 
    +            '--deps', 'none',
    54 
    +            '--format', '%{vars}',
    55 
    +            'element.bst'
    56 
    +        ]
    57 
    +        result = cli.run(project=project, silent=True, args=bst_args)
    58 
    +        result.assert_success()
    59 
    +
    60 
    +        loaded = _yaml.load_data(result.output)
    61 
    +        assert loaded['result'] == expected
    62 
    +
    63 
    +
    64 
    +@pytest.mark.datafiles(DATA_DIR)
    65 
    +def test_unsupported_arch(cli, datafiles):
    66 
    +
    67 
    +    with override_uname_os("AIX"):
    68 
    +        project = os.path.join(datafiles.dirname, datafiles.basename, 'option-os')
    69 
    +        result = cli.run(project=project, silent=True, args=[
    70 
    +            'show',
    71 
    +            '--deps', 'none',
    72 
    +            '--format', '%{vars}',
    73 
    +            'element.bst'
    74 
    +        ])
    75 
    +
    76 
    +        result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]