GNOME.org
 

[Notes] [Git][BuildStream/buildstream][lachlanmackenzie/debug_benchmark_failure] Deleted 7 commits: _sandboxremote.py: Add os and arch to command uploaded

Title: GitLab

Lachlan pushed to branch lachlanmackenzie/debug_benchmark_failure at BuildStream / buildstream

WARNING: The push did not contain any new commits, but force pushed to delete the commits and changes below.

Deleted commits:

11 changed files:

Changes:

  • buildstream/_options/optionos.py
    1 
    +
    2 
    +#
    3 
    +#  Copyright (C) 2017 Codethink Limited
    4 
    +#
    5 
    +#  This program is free software; you can redistribute it and/or
    6 
    +#  modify it under the terms of the GNU Lesser General Public
    7 
    +#  License as published by the Free Software Foundation; either
    8 
    +#  version 2 of the License, or (at your option) any later version.
    9 
    +#
    10 
    +#  This library is distributed in the hope that it will be useful,
    11 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    12 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    13 
    +#  Lesser General Public License for more details.
    14 
    +#
    15 
    +#  You should have received a copy of the GNU Lesser General Public
    16 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    17 
    +#
    18 
    +#  Authors:
    19 
    +#        Raoul Hidalgo Charman <raoul hidalgocharman codethink co uk>
    20 
    +
    21 
    +import os
    22 
    +from .optionenum import OptionEnum
    23 
    +
    24 
    +
    25 
    +# OptionOS
    26 
    +#
    27 
    +class OptionOS(OptionEnum):
    28 
    +
    29 
    +    OPTION_TYPE = 'os'
    30 
    +
    31 
    +    def load(self, node):
    32 
    +        super(OptionOS, self).load(node, allow_default_definition=False)
    33 
    +
    34 
    +    def load_default_value(self, node):
    35 
    +        return os.uname()[0]
    36 
    +
    37 
    +    def resolve(self):
    38 
    +
    39 
    +        # Validate that the default OS reported by uname() is explicitly
    40 
    +        # supported by the project, if not overridden by user config or cli.
    41 
    +        self.validate(self.value)

  • buildstream/_options/optionpool.py
    ... ... @@ -28,6 +28,7 @@ from .optionenum import OptionEnum
    28 28 
     from .optionflags import OptionFlags
    29 29 
     from .optioneltmask import OptionEltMask
    30 30 
     from .optionarch import OptionArch
    31 
    +from .optionos import OptionOS
    31 32
    32 33
    33 34 
     _OPTION_TYPES = {
    ... ... @@ -36,6 +37,7 @@ _OPTION_TYPES = {
    36 37 
         OptionFlags.OPTION_TYPE: OptionFlags,
    37 38 
         OptionEltMask.OPTION_TYPE: OptionEltMask,
    38 39 
         OptionArch.OPTION_TYPE: OptionArch,
    40 
    +    OptionOS.OPTION_TYPE: OptionOS,
    39 41 
     }
    40 42
    41 43

  • buildstream/_platform/linux.py
    ... ... @@ -25,6 +25,7 @@ from .. import utils
    25 25 
     from ..sandbox import SandboxDummy
    26 26
    27 27 
     from . import Platform
    28 
    +from .._exceptions import PlatformError
    28 29
    29 30
    30 31 
     class Linux(Platform):
    ... ... @@ -58,6 +59,9 @@ class Linux(Platform):
    58 59 
             else:
    59 60 
                 self._user_ns_available = False
    60 61
    62 
    +        # Set linux32 option
    63 
    +        self._linux32 = False
    64 
    +
    61 65 
         def create_sandbox(self, *args, **kwargs):
    62 66 
             if not self._local_sandbox_available:
    63 67 
                 return self._create_dummy_sandbox(*args, **kwargs)
    ... ... @@ -71,11 +75,33 @@ class Linux(Platform):
    71 75
    72 76 
             if self._user_ns_available:
    73 77 
                 # User namespace support allows arbitrary build UID/GID settings.
    74 
    -            return True
    75 
    -        else:
    78 
    +            pass
    79 
    +        elif (config.build_uid != self._uid or config.build_gid != self._gid):
    76 80 
                 # Without user namespace support, the UID/GID in the sandbox
    77 81 
                 # will match the host UID/GID.
    78 
    -            return config.build_uid == self._uid and config.build_gid == self._gid
    82 
    +            return False
    83 
    +
    84 
    +        # We can't do builds for another host or architecture except x86-32 on
    85 
    +        # x86-64
    86 
    +        host_os = self.get_host_os()
    87 
    +        host_arch = self.get_host_arch()
    88 
    +        if config.build_os != host_os:
    89 
    +            raise PlatformError("Configured and host OS don't match.")
    90 
    +        elif config.build_arch != host_arch:
    91 
    +            # We can use linux32 for building 32bit on 64bit machines
    92 
    +            if (host_os == "Linux" and
    93 
    +                    ((config.build_arch == "x86-32" and host_arch == "x86-64") or
    94 
    +                     (config.build_arch == "aarch32" and host_arch == "aarch64"))):
    95 
    +                # check linux32 is available
    96 
    +                try:
    97 
    +                    utils.get_host_tool('linux32')
    98 
    +                    self._linux32 = True
    99 
    +                except utils.ProgramNotFoundError:
    100 
    +                    pass
    101 
    +            else:
    102 
    +                raise PlatformError("Configured architecture and host architecture don't match.")
    103 
    +
    104 
    +        return True
    79 105
    80 106 
         ################################################
    81 107 
         #              Private Methods                 #
    ... ... @@ -100,6 +126,7 @@ class Linux(Platform):
    100 126 
             kwargs['user_ns_available'] = self._user_ns_available
    101 127 
             kwargs['die_with_parent_available'] = self._die_with_parent_available
    102 128 
             kwargs['json_status_available'] = self._json_status_available
    129 
    +        kwargs['linux32'] = self._linux32
    103 130 
             return SandboxBwrap(*args, **kwargs)
    104 131
    105 132 
         def _check_user_ns_available(self):

  • buildstream/_platform/unix.py
    ... ... @@ -44,4 +44,13 @@ class Unix(Platform):
    44 44 
         def check_sandbox_config(self, config):
    45 45 
             # With the chroot sandbox, the UID/GID in the sandbox
    46 46 
             # will match the host UID/GID (typically 0/0).
    47 
    -        return config.build_uid == self._uid and config.build_gid == self._gid
    47 
    +        if config.build_uid != self._uid or config.build_gid != self._gid:
    48 
    +            return False
    49 
    +
    50 
    +        # Check host os and architecture match
    51 
    +        if config.build_os != self.get_host_os():
    52 
    +            raise PlatformError("Configured and host OS don't match.")
    53 
    +        elif config.build_arch != self.get_host_arch():
    54 
    +            raise PlatformError("Configured and host architecture don't match.")
    55 
    +
    56 
    +        return True

  • buildstream/sandbox/_sandboxbwrap.py
    ... ... @@ -57,6 +57,7 @@ class SandboxBwrap(Sandbox):
    57 57 
             self.user_ns_available = kwargs['user_ns_available']
    58 58 
             self.die_with_parent_available = kwargs['die_with_parent_available']
    59 59 
             self.json_status_available = kwargs['json_status_available']
    60 
    +        self.linux32 = kwargs['linux32']
    60 61
    61 62 
         def _run(self, command, flags, *, cwd, env):
    62 63 
             stdout, stderr = self._get_output()
    ... ... @@ -74,8 +75,14 @@ class SandboxBwrap(Sandbox):
    74 75 
             mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY)
    75 76 
             root_mount_source = mount_map.get_mount_source('/')
    76 77
    78 
    +        # start command with linux32 if needed
    79 
    +        if self.linux32:
    80 
    +            bwrap_command = [utils.get_host_tool('linux32')]
    81 
    +        else:
    82 
    +            bwrap_command = []
    83 
    +
    77 84 
             # Grab the full path of the bwrap binary
    78 
    -        bwrap_command = [utils.get_host_tool('bwrap')]
    85 
    +        bwrap_command += [utils.get_host_tool('bwrap')]
    79 86
    80 87 
             for k, v in env.items():
    81 88 
                 bwrap_command += ['--setenv', k, v]

  • buildstream/sandbox/_sandboxremote.py
    ... ... @@ -126,13 +126,20 @@ class SandboxRemote(Sandbox):
    126 126 
                                      EnvironmentVariable(name=k, value=v)
    127 127 
                                      for (k, v) in environment.items()]
    128 128
    129 
    +        config = self._get_config()
    130 
    +        platform = remote_execution_pb2.Platform()
    131 
    +        platform.properties.extend([remote_execution_pb2.Platform.
    132 
    +                                    Property(name="OSFamily", value=config.build_os),
    133 
    +                                    remote_execution_pb2.Platform.
    134 
    +                                    Property(name="ISA", value=config.build_arch)])
    135 
    +
    129 136 
             # Create and send the Command object.
    130 137 
             remote_command = remote_execution_pb2.Command(arguments=command,
    131 138 
                                                           working_directory=working_directory,
    132 139 
                                                           environment_variables=environment_variables,
    133 140 
                                                           output_files=[],
    134 141 
                                                           output_directories=[self._output_directory],
    135 
    -                                                      platform=None)
    142 
    +                                                      platform=platform)
    136 143 
             context = self._get_context()
    137 144 
             cascache = context.get_cascache()
    138 145 
             casremote = CASRemote(self.storage_remote_spec)

  • doc/source/format_declaring.rst
    ... ... @@ -294,8 +294,10 @@ can be viewed in detail in the :ref:`builtin public data <public_builtin>` secti
    294 294 
     Sandbox
    295 295 
     ~~~~~~~
    296 296 
     Configuration for the build sandbox (other than :ref:`environment variables <format_environment>`)
    297 
    -can be placed in the ``sandbox`` configuration. At present, only the
    298 
    -UID and GID used by the user in the group can be specified.
    297 
    +can be placed in the ``sandbox`` configuration. The UID and GID used by the user
    298 
    +in the group can be specified, as well as the desired OS and machine
    299 
    +architecture. Possible machine architecture follow the same list as specified in
    300 
    +the :ref:`architecture option <project_options_arch>`.
    299 301
    300 302 
     .. code:: yaml
    301 303
    ... ... @@ -311,6 +313,24 @@ you can supply a different uid or gid for the sandbox. Only
    311 313 
     bwrap-style sandboxes support custom user IDs at the moment, and hence
    312 314 
     this will only work on Linux host platforms.
    313 315
    316 
    +.. code:: yaml
    317 
    +
    318 
    +   # Specify build OS and architecture
    319 
    +   sandbox:
    320 
    +     build-os: AIX
    321 
    +     build-arch: power-isa-be
    322 
    +
    323 
    +When building locally, if these don't match the host machine then generally the
    324 
    +build will fail. The exception is when the OS is Linux and the architecture
    325 
    +specifies an ``x86-32`` build on an ``x86-64`` machine, or ``aarch32`` build on
    326 
    +a ``aarch64`` machine, in which case the ``linux32`` command is prepended to the
    327 
    +bubblewrap command.
    328 
    +
    329 
    +When building remotely, the OS and architecture are added to the ``Platform``
    330 
    +field in the ``Command`` uploaded. Whether this actually results in a building
    331 
    +the element for the desired OS and architecture is dependent on the server
    332 
    +having implemented these options the same as buildstream.
    333 
    +
    314 334 
     .. note::
    315 335
    316 336 
        The ``sandbox`` configuration is available since :ref:`format version 6 <project_format_version>`

  • doc/source/format_project.rst
    ... ... @@ -552,9 +552,22 @@ exported as a comma separated list of selected value strings.
    552 552
    553 553 
     Architecture
    554 554 
     ~~~~~~~~~~~~
    555 
    -The ``arch`` option type is special enumeration option which
    556 
    -defaults to the result of `uname -m`, and does not support
    557 
    -assigning any default in the project configuration.
    555 
    +The ``arch`` option type is a special enumeration option which defaults via
    556 
    +`uname -m` results to the following list.
    557 
    +
    558 
    +* aarch32
    559 
    +* aarch64
    560 
    +* aarch64-be
    561 
    +* power-isa-be
    562 
    +* power-isa-le
    563 
    +* sparc-v9
    564 
    +* x86-32
    565 
    +* x86-64
    566 
    +
    567 
    +The reason for this, opposed to using just `uname -m`, is that we want an
    568 
    +OS-independent list, as well as several results mapping to the same architecture
    569 
    +(e.g. i386, i486 etc. are all x86-32). It does not support assigning any default
    570 
    +in the project configuration.
    558 571
    559 572 
     .. code:: yaml
    560 573
    ... ... @@ -563,10 +576,10 @@ assigning any default in the project configuration.
    563 576 
            type: arch
    564 577 
            description: The machine architecture
    565 578 
            values:
    566 
    -       - arm
    579 
    +       - aarch32
    567 580 
            - aarch64
    568 
    -       - i386
    569 
    -       - x86_64
    581 
    +       - x86-32
    582 
    +       - x86-64
    570 583
    571 584
    572 585 
     Architecture options can be tested with the same expressions

  • tests/format/option-os/element.bst
    1 
    +kind: autotools
    2 
    +variables:
    3 
    +  result: "Nothing"
    4 
    +  (?):
    5 
    +  - machine_os == "Linux":
    6 
    +      result: "Linuxy"
    7 
    +  - machine_os == "Darwin":
    8 
    +      result: "Darwiny"
    9 
    +  - machine_os == "SunOS":
    10 
    +      result: "SunOSy"
    11 
    +  - machine_os == "FreeBSD":
    12 
    +      result: "FreeBSDy"
    \ No newline at end of file

  • tests/format/option-os/project.conf
    1 
    +name: test
    2 
    +
    3 
    +options:
    4 
    +  machine_os:
    5 
    +    type: os
    6 
    +    description: The operating system
    7 
    +    values:
    8 
    +    - Linux
    9 
    +    - Darwin
    10 
    +    - FreeBSD
    11 
    +    - SunOS

  • tests/format/optionos.py
    1 
    +import os
    2 
    +import pytest
    3 
    +from contextlib import contextmanager
    4 
    +
    5 
    +from buildstream import _yaml
    6 
    +from buildstream._exceptions import ErrorDomain, LoadErrorReason
    7 
    +from tests.testutils.runcli import cli
    8 
    +
    9 
    +DATA_DIR = os.path.dirname(os.path.realpath(__file__))
    10 
    +
    11 
    +
    12 
    +@contextmanager
    13 
    +def override_uname_os(name):
    14 
    +    orig_uname = os.uname
    15 
    +    orig_tuple = tuple(os.uname())
    16 
    +    override_result = (name, orig_tuple[1],
    17 
    +                       orig_tuple[2], orig_tuple[3],
    18 
    +                       orig_tuple[4])
    19 
    +
    20 
    +    def override():
    21 
    +        return override_result
    22 
    +
    23 
    +    os.uname = override
    24 
    +    yield
    25 
    +    os.uname = orig_uname
    26 
    +
    27 
    +
    28 
    +@pytest.mark.datafiles(DATA_DIR)
    29 
    +@pytest.mark.parametrize("uname,value,expected", [
    30 
    +    # Test explicitly provided arches
    31 
    +    ('Darwin', 'Linux', 'Linuxy'),
    32 
    +    ('SunOS', 'FreeBSD', 'FreeBSDy'),
    33 
    +
    34 
    +    # Test automatically derived arches
    35 
    +    ('Linux', None, 'Linuxy'),
    36 
    +    ('Darwin', None, 'Darwiny'),
    37 
    +
    38 
    +    # Test that explicitly provided arches dont error out
    39 
    +    # when the `uname` reported arch is not supported
    40 
    +    ('AIX', 'Linux', 'Linuxy'),
    41 
    +    ('HaikuOS', 'SunOS', 'SunOSy'),
    42 
    +])
    43 
    +def test_conditionals(cli, datafiles, uname, value, expected):
    44 
    +    with override_uname_os(uname):
    45 
    +        project = os.path.join(datafiles.dirname, datafiles.basename, 'option-os')
    46 
    +
    47 
    +        bst_args = []
    48 
    +        if value is not None:
    49 
    +            bst_args += ['--option', 'machine_os', value]
    50 
    +
    51 
    +        bst_args += [
    52 
    +            'show',
    53 
    +            '--deps', 'none',
    54 
    +            '--format', '%{vars}',
    55 
    +            'element.bst'
    56 
    +        ]
    57 
    +        result = cli.run(project=project, silent=True, args=bst_args)
    58 
    +        result.assert_success()
    59 
    +
    60 
    +        loaded = _yaml.load_data(result.output)
    61 
    +        assert loaded['result'] == expected
    62 
    +
    63 
    +
    64 
    +@pytest.mark.datafiles(DATA_DIR)
    65 
    +def test_unsupported_arch(cli, datafiles):
    66 
    +
    67 
    +    with override_uname_os("AIX"):
    68 
    +        project = os.path.join(datafiles.dirname, datafiles.basename, 'option-os')
    69 
    +        result = cli.run(project=project, silent=True, args=[
    70 
    +            'show',
    71 
    +            '--deps', 'none',
    72 
    +            '--format', '%{vars}',
    73 
    +            'element.bst'
    74 
    +        ])
    75 
    +
    76 
    +        result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]