GNOME.org
 

[Notes] [Git][BuildStream/buildstream][chandan/bst-docker-import] 13 commits: sandbox/_sandboxdummy.py: Take a reason for use

Title: GitLab

Jonathan Maw pushed to branch chandan/bst-docker-import at BuildStream / buildstream

Commits:

9 changed files:

Changes:

  • buildstream/_platform/darwin.py
    ... ... @@ -34,6 +34,9 @@ class Darwin(Platform):
    34 34 
             super().__init__()
    35 35
    36 36 
         def create_sandbox(self, *args, **kwargs):
    37 
    +        kwargs['dummy_reason'] = \
    38 
    +            "OSXFUSE is not supported and there are no supported sandbox" + \
    39 
    +            "technologies for OSX at this time"
    37 40 
             return SandboxDummy(*args, **kwargs)
    38 41
    39 42 
         def check_sandbox_config(self, config):

  • buildstream/_platform/linux.py
    ... ... @@ -37,25 +37,27 @@ class Linux(Platform):
    37 37 
             self._uid = os.geteuid()
    38 38 
             self._gid = os.getegid()
    39 39
    40 
    +        self._have_fuse = os.path.exists("/dev/fuse")
    41 
    +        self._bwrap_exists = _site.check_bwrap_version(0, 0, 0)
    42 
    +        self._have_good_bwrap = _site.check_bwrap_version(0, 1, 2)
    43 
    +
    44 
    +        self._local_sandbox_available = self._have_fuse and self._have_good_bwrap
    45 
    +
    40 46 
             self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
    41 47
    42 
    -        if self._local_sandbox_available():
    48 
    +        if self._local_sandbox_available:
    43 49 
                 self._user_ns_available = self._check_user_ns_available()
    44 50 
             else:
    45 51 
                 self._user_ns_available = False
    46 52
    47 53 
         def create_sandbox(self, *args, **kwargs):
    48 
    -        if not self._local_sandbox_available():
    49 
    -            return SandboxDummy(*args, **kwargs)
    54 
    +        if not self._local_sandbox_available:
    55 
    +            return self._create_dummy_sandbox(*args, **kwargs)
    50 56 
             else:
    51 
    -            from ..sandbox._sandboxbwrap import SandboxBwrap
    52 
    -            # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
    53 
    -            kwargs['user_ns_available'] = self._user_ns_available
    54 
    -            kwargs['die_with_parent_available'] = self._die_with_parent_available
    55 
    -            return SandboxBwrap(*args, **kwargs)
    57 
    +            return self._create_bwrap_sandbox(*args, **kwargs)
    56 58
    57 59 
         def check_sandbox_config(self, config):
    58 
    -        if not self._local_sandbox_available():
    60 
    +        if not self._local_sandbox_available:
    59 61 
                 # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run).
    60 62 
                 return True
    61 63
    ... ... @@ -70,11 +72,26 @@ class Linux(Platform):
    70 72 
         ################################################
    71 73 
         #              Private Methods                 #
    72 74 
         ################################################
    73 
    -    def _local_sandbox_available(self):
    74 
    -        try:
    75 
    -            return os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')
    76 
    -        except utils.ProgramNotFoundError:
    77 
    -            return False
    75 
    +
    76 
    +    def _create_dummy_sandbox(self, *args, **kwargs):
    77 
    +        reasons = []
    78 
    +        if not self._have_fuse:
    79 
    +            reasons.append("FUSE is unavailable")
    80 
    +        if not self._have_good_bwrap:
    81 
    +            if self._bwrap_exists:
    82 
    +                reasons.append("`bwrap` is too old (bst needs at least 0.1.2)")
    83 
    +            else:
    84 
    +                reasons.append("`bwrap` executable not found")
    85 
    +
    86 
    +        kwargs['dummy_reason'] = " and ".join(reasons)
    87 
    +        return SandboxDummy(*args, **kwargs)
    88 
    +
    89 
    +    def _create_bwrap_sandbox(self, *args, **kwargs):
    90 
    +        from ..sandbox._sandboxbwrap import SandboxBwrap
    91 
    +        # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
    92 
    +        kwargs['user_ns_available'] = self._user_ns_available
    93 
    +        kwargs['die_with_parent_available'] = self._die_with_parent_available
    94 
    +        return SandboxBwrap(*args, **kwargs)
    78 95
    79 96 
         def _check_user_ns_available(self):
    80 97 
             # Here, lets check if bwrap is able to create user namespaces,

  • buildstream/_site.py
    ... ... @@ -78,18 +78,12 @@ def check_bwrap_version(major, minor, patch):
    78 78 
             if not bwrap_path:
    79 79 
                 return False
    80 80 
             cmd = [bwrap_path, "--version"]
    81 
    -        version = str(subprocess.check_output(cmd).split()[1], "utf-8")
    81 
    +        try:
    82 
    +            version = str(subprocess.check_output(cmd).split()[1], "utf-8")
    83 
    +        except subprocess.CalledProcessError:
    84 
    +            # Failure trying to run bubblewrap
    85 
    +            return False
    82 86 
             _bwrap_major, _bwrap_minor, _bwrap_patch = map(int, version.split("."))
    83 87
    84 88 
         # Check whether the installed version meets the requirements
    85 
    -    if _bwrap_major > major:
    86 
    -        return True
    87 
    -    elif _bwrap_major < major:
    88 
    -        return False
    89 
    -    else:
    90 
    -        if _bwrap_minor > minor:
    91 
    -            return True
    92 
    -        elif _bwrap_minor < minor:
    93 
    -            return False
    94 
    -        else:
    95 
    -            return _bwrap_patch >= patch
    89 
    +    return (_bwrap_major, _bwrap_minor, _bwrap_patch) >= (major, minor, patch)

  • buildstream/element.py
    ... ... @@ -212,7 +212,7 @@ class Element(Plugin):
    212 212 
             self.__staged_sources_directory = None  # Location where Element.stage_sources() was called
    213 213 
             self.__tainted = None                   # Whether the artifact is tainted and should not be shared
    214 214 
             self.__required = False                 # Whether the artifact is required in the current session
    215 
    -        self.__build_result = None              # The result of assembling this Element
    215 
    +        self.__build_result = None              # The result of assembling this Element (success, description, detail)
    216 216 
             self._build_log_path = None            # The path of the build log for this Element
    217 217
    218 218 
             # hash tables of loaded artifact metadata, hashed by key
    ... ... @@ -1479,11 +1479,13 @@ class Element(Plugin):
    1479 1479
    1480 1480 
             self._update_state()
    1481 1481
    1482 
    -        if self._get_workspace() and self._cached():
    1482 
    +        if self._get_workspace() and self._cached_success():
    1483 
    +            assert utils._is_main_process(), \
    1484 
    +                "Attempted to save workspace configuration from child process"
    1483 1485 
                 #
    1484 1486 
                 # Note that this block can only happen in the
    1485 
    -            # main process, since `self._cached()` cannot
    1486 
    -            # be true when assembly is completed in the task.
    1487 
    +            # main process, since `self._cached_success()` cannot
    1488 
    +            # be true when assembly is successful in the task.
    1487 1489 
                 #
    1488 1490 
                 # For this reason, it is safe to update and
    1489 1491 
                 # save the workspaces configuration

  • buildstream/sandbox/_sandboxdummy.py
    ... ... @@ -23,6 +23,7 @@ from . import Sandbox
    23 23 
     class SandboxDummy(Sandbox):
    24 24 
         def __init__(self, *args, **kwargs):
    25 25 
             super().__init__(*args, **kwargs)
    26 
    +        self._reason = kwargs.get("dummy_reason", "no reason given")
    26 27
    27 28 
         def run(self, command, flags, *, cwd=None, env=None):
    28 29
    ... ... @@ -37,4 +38,4 @@ class SandboxDummy(Sandbox):
    37 38 
                                    "'{}'".format(command[0]),
    38 39 
                                    reason='missing-command')
    39 40
    40 
    -        raise SandboxError("This platform does not support local builds")
    41 
    +        raise SandboxError("This platform does not support local builds: {}".format(self._reason))

  • buildstream/utils.py
    ... ... @@ -686,7 +686,7 @@ def _force_rmtree(rootpath, **kwargs):
    686 686
    687 687 
         try:
    688 688 
             shutil.rmtree(rootpath, **kwargs)
    689 
    -    except shutil.Error as e:
    689 
    +    except OSError as e:
    690 690 
             raise UtilError("Failed to remove cache directory '{}': {}"
    691 691 
                             .format(rootpath, e))
    692 692

  • contrib/bst-docker-import
    1 
    +#!/bin/bash
    2 
    +#
    3 
    +#  Copyright 2018 Bloomberg Finance LP
    4 
    +#
    5 
    +#  This program is free software; you can redistribute it and/or
    6 
    +#  modify it under the terms of the GNU Lesser General Public
    7 
    +#  License as published by the Free Software Foundation; either
    8 
    +#  version 2 of the License, or (at your option) any later version.
    9 
    +#
    10 
    +#  This library is distributed in the hope that it will be useful,
    11 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    12 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    13 
    +#  Lesser General Public License for more details.
    14 
    +#
    15 
    +#  You should have received a copy of the GNU Lesser General Public
    16 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    17 
    +#
    18 
    +#  Authors:
    19 
    +#        Chadnan Singh <csingh43 bloomberg net>
    20 
    +
    21 
    +# This is a helper script to generate Docker images using checkouts of
    22 
    +# BuildStream elements.
    23 
    +
    24 
    +usage() {
    25 
    +    cat <<EOF
    26 
    +
    27 
    +USAGE: $(basename "$0") [-c BST_CMD] [-m MESSAGE] [-t TAG] [-h] ELEMENT
    28 
    +
    29 
    +Create a Docker image from bst checkout of an element.
    30 
    +
    31 
    +OPTIONS:
    32 
    +    -c BST_CMD    Path to BuildStream command (default: bst).
    33 
    +    -m MESSAGE    Commit message for the imported image.
    34 
    +    -t TAG        Tag of the imported image.
    35 
    +    -h            Print this help text and exit.
    36 
    +
    37 
    +EXAMPLES:
    38 
    +
    39 
    +    # Import hello.bst as a Docker image with tag "bst-hello" and message "hello"
    40 
    +    $(basename "$0") -m hello -t bst-hello hello.bst
    41 
    +
    42 
    +    # Import hello.bst as a Docker image with tag "bst-hello" using bst-here
    43 
    +    $(basename "$0") -c bst-here -t bst-hello hello.bst
    44 
    +
    45 
    +EOF
    46 
    +    exit "$1"
    47 
    +}
    48 
    +
    49 
    +die() {
    50 
    +    echo "FATAL: $1" >&2
    51 
    +    exit 1
    52 
    +}
    53 
    +
    54 
    +bst_cmd=bst
    55 
    +docker_import_cmd=(docker import)
    56 
    +docker_image_tag=
    57 
    +
    58 
    +while getopts c:m:t:h arg
    59 
    +do
    60 
    +    case $arg in
    61 
    +    c)
    62 
    +        bst_cmd="$OPTARG"
    63 
    +        ;;
    64 
    +    m)
    65 
    +        docker_import_cmd+=('-m' "$OPTARG")
    66 
    +        ;;
    67 
    +    t)
    68 
    +        docker_image_tag="$OPTARG"
    69 
    +        ;;
    70 
    +    h)
    71 
    +        usage 0
    72 
    +        ;;
    73 
    +    \?)
    74 
    +        usage 1
    75 
    +    esac
    76 
    +done
    77 
    +
    78 
    +shift $((OPTIND-1))
    79 
    +if [[ "$#" != 1 ]]; then
    80 
    +    echo "$0: No element specified" >&2
    81 
    +    usage 1
    82 
    +fi
    83 
    +element="$1"
    84 
    +
    85 
    +# Dump to a temporary file in the current directory.
    86 
    +# NOTE: We use current directory to try to ensure compatibility with scripts
    87 
    +# like bst-here, assuming that the current working directory is mounted
    88 
    +# inside the container.
    89 
    +
    90 
    +checkout_tar="bst-checkout-$(basename "$element")-$RANDOM.tar"
    91 
    +
    92 
    +echo "INFO: Checking out $element ..." >&2
    93 
    +$bst_cmd checkout --tar "$element" "$checkout_tar" || die "Failed to checkout $element"
    94 
    +echo "INFO: Successfully checked out $element" >&2
    95 
    +
    96 
    +echo "INFO: Importing Docker image ..."
    97 
    +"${docker_import_cmd[@]}" "$checkout_tar" "$docker_image_tag" || die "Failed to import Docker image from tarball"
    98 
    +echo "INFO: Successfully import Docker image $docker_image_tag"
    99 
    +
    100 
    +echo "INFO: Cleaning up ..."
    101 
    +rm "$checkout_tar" || die "Failed to remove $checkout_tar"
    102 
    +echo "INFO: Clean up finished"

  • setup.py
    ... ... @@ -54,12 +54,13 @@ REQUIRED_BWRAP_MINOR = 1
    54 54 
     REQUIRED_BWRAP_PATCH = 2
    55 55
    56 56
    57 
    -def exit_bwrap(reason):
    57 
    +def warn_bwrap(reason):
    58 58 
         print(reason +
    59 
    -          "\nBuildStream requires Bubblewrap (bwrap) for"
    60 
    -          " sandboxing the build environment. Install it using your package manager"
    61 
    -          " (usually bwrap or bubblewrap)")
    62 
    -    sys.exit(1)
    59 
    +          "\nBuildStream requires Bubblewrap (bwrap {}.{}.{} or better),"
    60 
    +          " during local builds, for"
    61 
    +          " sandboxing the build environment.\nInstall it using your package manager"
    62 
    +          " (usually bwrap or bubblewrap) otherwise you will be limited to"
    63 
    +          " remote builds only.".format(REQUIRED_BWRAP_MAJOR, REQUIRED_BWRAP_MINOR, REQUIRED_BWRAP_PATCH))
    63 64
    64 65
    65 66 
     def bwrap_too_old(major, minor, patch):
    ... ... @@ -76,18 +77,19 @@ def bwrap_too_old(major, minor, patch):
    76 77 
             return False
    77 78
    78 79
    79 
    -def assert_bwrap():
    80 
    +def check_for_bwrap():
    80 81 
         platform = os.environ.get('BST_FORCE_BACKEND', '') or sys.platform
    81 82 
         if platform.startswith('linux'):
    82 83 
             bwrap_path = shutil.which('bwrap')
    83 84 
             if not bwrap_path:
    84 
    -            exit_bwrap("Bubblewrap not found")
    85 
    +            warn_bwrap("Bubblewrap not found")
    86 
    +            return
    85 87
    86 88 
             version_bytes = subprocess.check_output([bwrap_path, "--version"]).split()[1]
    87 89 
             version_string = str(version_bytes, "utf-8")
    88 90 
             major, minor, patch = map(int, version_string.split("."))
    89 91 
             if bwrap_too_old(major, minor, patch):
    90 
    -            exit_bwrap("Bubblewrap too old")
    92 
    +            warn_bwrap("Bubblewrap too old")
    91 93
    92 94
    93 95 
     ###########################################
    ... ... @@ -126,7 +128,7 @@ bst_install_entry_points = {
    126 128 
     }
    127 129
    128 130 
     if not os.environ.get('BST_ARTIFACTS_ONLY', ''):
    129 
    -    assert_bwrap()
    131 
    +    check_for_bwrap()
    130 132 
         bst_install_entry_points['console_scripts'] += [
    131 133 
             'bst = buildstream._frontend:cli'
    132 134 
         ]

  • tests/frontend/workspace.py
    ... ... @@ -43,7 +43,8 @@ DATA_DIR = os.path.join(
    43 43 
     )
    44 44
    45 45
    46 
    -def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None, project_path=None):
    46 
    +def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
    47 
    +                   project_path=None, element_attrs=None):
    47 48 
         if not workspace_dir:
    48 49 
             workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
    49 50 
         if not project_path:
    ... ... @@ -69,6 +70,8 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir
    69 70 
                 repo.source_config(ref=ref)
    70 71 
             ]
    71 72 
         }
    73 
    +    if element_attrs:
    74 
    +        element = {**element, **element_attrs}
    72 75 
         _yaml.dump(element,
    73 76 
                    os.path.join(element_path,
    74 77 
                                 element_name))
    ... ... @@ -854,3 +857,22 @@ def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict):
    854 857
    855 858 
         # Check that the original /usr/bin/hello is not in the checkout
    856 859 
         assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello'))
    860 
    +
    861 
    +
    862 
    +@pytest.mark.datafiles(DATA_DIR)
    863 
    +def test_multiple_failed_builds(cli, tmpdir, datafiles):
    864 
    +    element_config = {
    865 
    +        "kind": "manual",
    866 
    +        "config": {
    867 
    +            "configure-commands": [
    868 
    +                "unknown_command_that_will_fail"
    869 
    +            ]
    870 
    +        }
    871 
    +    }
    872 
    +    element_name, project, _ = open_workspace(cli, tmpdir, datafiles,
    873 
    +                                              "git", False, element_attrs=element_config)
    874 
    +
    875 
    +    for _ in range(2):
    876 
    +        result = cli.run(project=project, args=["build", element_name])
    877 
    +        assert "BUG" not in result.stderr
    878 
    +        assert cli.get_element_state(project, element_name) != "cached"

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]