[Notes] [Git][BuildStream/buildstream][mac_fixes] 7 commits: Added `set_resource_limits()` to platform



Title: GitLab

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

9 changed files:

Changes:

  • buildstream/_frontend/app.py
    ... ... @@ -116,14 +116,6 @@ class App():
    116 116
             else:
    
    117 117
                 self.colors = False
    
    118 118
     
    
    119
    -        # Increase the soft limit for open file descriptors to the maximum.
    
    120
    -        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    
    121
    -        # Avoid hitting the limit too quickly.
    
    122
    -        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    
    123
    -        if limits[0] != limits[1]:
    
    124
    -            # Set soft limit to hard limit
    
    125
    -            resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
    
    126
    -
    
    127 119
         # create()
    
    128 120
         #
    
    129 121
         # Should be used instead of the regular constructor.
    

  • buildstream/_platform/darwin.py
    1
    +#
    
    2
    +#  Copyright (C) 2017 Codethink Limited
    
    3
    +#  Copyright (C) 2018 Bloomberg Finance LP
    
    4
    +#
    
    5
    +#  This program is free software; you can redistribute it and/or
    
    6
    +#  modify it under the terms of the GNU Lesser General Public
    
    7
    +#  License as published by the Free Software Foundation; either
    
    8
    +#  version 2 of the License, or (at your option) any later version.
    
    9
    +#
    
    10
    +#  This library is distributed in the hope that it will be useful,
    
    11
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    
    12
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    
    13
    +#  Lesser General Public License for more details.
    
    14
    +#
    
    15
    +#  You should have received a copy of the GNU Lesser General Public
    
    16
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    
    17
    +
    
    18
    +import os
    
    19
    +import resource
    
    20
    +
    
    21
    +from .._exceptions import PlatformError
    
    22
    +from ..sandbox import SandboxChroot, SandboxDummy
    
    23
    +
    
    24
    +from . import Platform
    
    25
    +
    
    26
    +
    
    27
    +class Darwin(Platform):
    
    28
    +
    
    29
    +    # This value comes from OPEN_MAX in syslimits.h
    
    30
    +    OPEN_MAX = 10240
    
    31
    +
    
    32
    +    def __init__(self, context):
    
    33
    +
    
    34
    +        super().__init__(context)
    
    35
    +
    
    36
    +    @property
    
    37
    +    def artifactcache(self):
    
    38
    +        return self._artifact_cache
    
    39
    +
    
    40
    +    def create_sandbox(self, *args, **kwargs):
    
    41
    +        return SandboxDummy(*args, **kwargs)
    
    42
    +
    
    43
    +    def get_cpu_count(self, cap=None):
    
    44
    +        if cap < os.cpu_count():
    
    45
    +            return cap
    
    46
    +        else:
    
    47
    +            return os.cpu_count()
    
    48
    +
    
    49
    +    def set_resource_limits(self, soft_limit=OPEN_MAX, hard_limit=None):
    
    50
    +        super().set_resource_limits(soft_limit)

  • buildstream/_platform/linux.py
    ... ... @@ -17,13 +17,14 @@
    17 17
     #  Authors:
    
    18 18
     #        Tristan Maat <tristan maat codethink co uk>
    
    19 19
     
    
    20
    +import os
    
    20 21
     import subprocess
    
    21 22
     
    
    22 23
     from .. import _site
    
    23 24
     from .. import utils
    
    24 25
     from .._artifactcache.cascache import CASCache
    
    25 26
     from .._message import Message, MessageType
    
    26
    -from ..sandbox import SandboxBwrap
    
    27
    +from ..sandbox import SandboxBwrap, SandboxDummy
    
    27 28
     
    
    28 29
     from . import Platform
    
    29 30
     
    
    ... ... @@ -32,27 +33,40 @@ class Linux(Platform):
    32 33
     
    
    33 34
         def __init__(self, context):
    
    34 35
     
    
    35
    -        super().__init__(context)
    
    36
    -
    
    37 36
             self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
    
    38
    -        self._user_ns_available = self._check_user_ns_available(context)
    
    39
    -        self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
    
    37
    +
    
    38
    +        if self._local_sandbox_available():
    
    39
    +            self._user_ns_available = self._check_user_ns_available(context)
    
    40
    +        else:
    
    41
    +            self._user_ns_available = False
    
    42
    +
    
    43
    +        # _user_ns_available needs to be set before chaining up to the super class
    
    44
    +        # This is because it will call create_artifact_cache()
    
    45
    +        super().__init__(context)
    
    40 46
     
    
    41 47
         @property
    
    42 48
         def artifactcache(self):
    
    43 49
             return self._artifact_cache
    
    44 50
     
    
    45 51
         def create_sandbox(self, *args, **kwargs):
    
    46
    -        # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
    
    47
    -        kwargs['user_ns_available'] = self._user_ns_available
    
    48
    -        kwargs['die_with_parent_available'] = self._die_with_parent_available
    
    49
    -        return SandboxBwrap(*args, **kwargs)
    
    52
    +        if not self._local_sandbox_available():
    
    53
    +            return SandboxDummy(*args, **kwargs)
    
    54
    +        else:
    
    55
    +            # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
    
    56
    +            kwargs['user_ns_available'] = self._user_ns_available
    
    57
    +            kwargs['die_with_parent_available'] = self._die_with_parent_available
    
    58
    +            return SandboxBwrap(*args, **kwargs)
    
    59
    +
    
    60
    +    def create_artifact_cache(self, context, *, enable_push):
    
    61
    +        return super().create_artifact_cache(context, *, self._user_ns_available)
    
    50 62
     
    
    51 63
         ################################################
    
    52 64
         #              Private Methods                 #
    
    53 65
         ################################################
    
    54
    -    def _check_user_ns_available(self, context):
    
    66
    +    def _local_sandbox_available(self):
    
    67
    +        return os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')
    
    55 68
     
    
    69
    +    def _check_user_ns_available(self, context):
    
    56 70
             # Here, lets check if bwrap is able to create user namespaces,
    
    57 71
             # issue a warning if it's not available, and save the state
    
    58 72
             # locally so that we can inform the sandbox to not try it
    

  • buildstream/_platform/platform.py
    ... ... @@ -19,8 +19,10 @@
    19 19
     
    
    20 20
     import os
    
    21 21
     import sys
    
    22
    +import resource
    
    22 23
     
    
    23 24
     from .._exceptions import PlatformError, ImplError
    
    25
    +from .._artifactcache.cascache import CASCache
    
    24 26
     
    
    25 27
     
    
    26 28
     class Platform():
    
    ... ... @@ -37,22 +39,28 @@ class Platform():
    37 39
         #
    
    38 40
         def __init__(self, context):
    
    39 41
             self.context = context
    
    42
    +        self.set_resource_limits()
    
    43
    +        self._artifact_cache = self.create_artifact_cache(context, True)
    
    40 44
     
    
    41 45
         @classmethod
    
    42 46
         def create_instance(cls, *args, **kwargs):
    
    43
    -        if sys.platform.startswith('linux'):
    
    44
    -            backend = 'linux'
    
    45
    -        else:
    
    46
    -            backend = 'unix'
    
    47 47
     
    
    48 48
             # Meant for testing purposes and therefore hidden in the
    
    49 49
             # deepest corners of the source code. Try not to abuse this,
    
    50 50
             # please?
    
    51 51
             if os.getenv('BST_FORCE_BACKEND'):
    
    52 52
                 backend = os.getenv('BST_FORCE_BACKEND')
    
    53
    +        elif sys.platform.startswith('linux'):
    
    54
    +            backend = 'linux'
    
    55
    +        elif sys.platform.startswith('darwin'):
    
    56
    +            backend = 'darwin'
    
    57
    +        else:
    
    58
    +            backend = 'unix'
    
    53 59
     
    
    54 60
             if backend == 'linux':
    
    55 61
                 from .linux import Linux as PlatformImpl
    
    62
    +        elif backend == 'darwin':
    
    63
    +            from .darwin import Darwin as PlatformImpl
    
    56 64
             elif backend == 'unix':
    
    57 65
                 from .unix import Unix as PlatformImpl
    
    58 66
             else:
    
    ... ... @@ -66,6 +74,9 @@ class Platform():
    66 74
                 raise PlatformError("Platform needs to be initialized first")
    
    67 75
             return cls._instance
    
    68 76
     
    
    77
    +    def get_cpu_count(self, cap=None):
    
    78
    +        return min(len(os.sched_getaffinity(0)), cap)
    
    79
    +
    
    69 80
         ##################################################################
    
    70 81
         #                       Platform properties                      #
    
    71 82
         ##################################################################
    
    ... ... @@ -92,3 +103,18 @@ class Platform():
    92 103
         def create_sandbox(self, *args, **kwargs):
    
    93 104
             raise ImplError("Platform {platform} does not implement create_sandbox()"
    
    94 105
                             .format(platform=type(self).__name__))
    
    106
    +
    
    107
    +    def set_resource_limits(self, soft_limit=None, hard_limit=None):
    
    108
    +        # Need to set resources for _frontend/app.py as this is dependent on the platform
    
    109
    +        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    
    110
    +        # Avoid hitting the limit too quickly.
    
    111
    +        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    
    112
    +        if limits[0] != limits[1]:
    
    113
    +            if soft_limit is None:
    
    114
    +                soft_limit = limits[1]
    
    115
    +            if hard_limit is None:
    
    116
    +                hard_limit = limits[1]
    
    117
    +            resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
    
    118
    +
    
    119
    +    def create_artifact_cache(self, context, *, enable_push=True):
    
    120
    +        return CASCache(context=context, *, enable_push=enable_push)

  • buildstream/_platform/unix.py
    ... ... @@ -31,7 +31,6 @@ class Unix(Platform):
    31 31
         def __init__(self, context):
    
    32 32
     
    
    33 33
             super().__init__(context)
    
    34
    -        self._artifact_cache = CASCache(context)
    
    35 34
     
    
    36 35
             # Not necessarily 100% reliable, but we want to fail early.
    
    37 36
             if os.geteuid() != 0:
    

  • buildstream/_project.py
    ... ... @@ -38,6 +38,7 @@ from ._loader import Loader
    38 38
     from .element import Element
    
    39 39
     from ._message import Message, MessageType
    
    40 40
     from ._includes import Includes
    
    41
    +from ._platform import Platform
    
    41 42
     
    
    42 43
     
    
    43 44
     # Project Configuration file
    
    ... ... @@ -617,7 +618,8 @@ class Project():
    617 618
             # Based on some testing (mainly on AWS), maximum effective
    
    618 619
             # max-jobs value seems to be around 8-10 if we have enough cores
    
    619 620
             # users should set values based on workload and build infrastructure
    
    620
    -        output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
    
    621
    +        platform = Platform.get_platform()
    
    622
    +        output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
    
    621 623
     
    
    622 624
             # Export options into variables, if that was requested
    
    623 625
             output.options.export_variables(output.base_variables)
    

  • buildstream/sandbox/__init__.py
    ... ... @@ -21,3 +21,4 @@ from .sandbox import Sandbox, SandboxFlags
    21 21
     from ._sandboxchroot import SandboxChroot
    
    22 22
     from ._sandboxbwrap import SandboxBwrap
    
    23 23
     from ._sandboxremote import SandboxRemote
    
    24
    +from ._sandboxdummy import SandboxDummy

  • buildstream/sandbox/_sandboxdummy.py
    1
    +#
    
    2
    +#  Copyright (C) 2017 Codethink Limited
    
    3
    +#
    
    4
    +#  This program is free software; you can redistribute it and/or
    
    5
    +#  modify it under the terms of the GNU Lesser General Public
    
    6
    +#  License as published by the Free Software Foundation; either
    
    7
    +#  version 2 of the License, or (at your option) any later version.
    
    8
    +#
    
    9
    +#  This library is distributed in the hope that it will be useful,
    
    10
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    
    11
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    
    12
    +#  Lesser General Public License for more details.
    
    13
    +#
    
    14
    +#  You should have received a copy of the GNU Lesser General Public
    
    15
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    
    16
    +#
    
    17
    +#  Authors:
    
    18
    +
    
    19
    +from .._exceptions import SandboxError
    
    20
    +from . import Sandbox
    
    21
    +
    
    22
    +
    
    23
    +class SandboxDummy(Sandbox):
    
    24
    +    def __init__(self, *args, **kwargs):
    
    25
    +        super().__init__(*args, **kwargs)
    
    26
    +
    
    27
    +    def run(self, command, flags, *, cwd=None, env=None):
    
    28
    +
    
    29
    +        # Fallback to the sandbox default settings for
    
    30
    +        # the cwd and env.
    
    31
    +        #
    
    32
    +        cwd = self._get_work_directory(cwd=cwd)
    
    33
    +        env = self._get_environment(cwd=cwd, env=env)
    
    34
    +
    
    35
    +        if not self._has_command(command[0], env):
    
    36
    +            raise SandboxError("Staged artifacts do not provide command "
    
    37
    +                               "'{}'".format(command[0]),
    
    38
    +                               reason='missing-command')
    
    39
    +
    
    40
    +        raise SandboxError("This platform does not support local builds")

  • buildstream/utils.py
    ... ... @@ -35,6 +35,7 @@ import tempfile
    35 35
     import itertools
    
    36 36
     import functools
    
    37 37
     from contextlib import contextmanager
    
    38
    +from stat import S_ISDIR
    
    38 39
     
    
    39 40
     import psutil
    
    40 41
     
    
    ... ... @@ -328,27 +329,25 @@ def safe_remove(path):
    328 329
         Raises:
    
    329 330
            UtilError: In the case of unexpected system call failures
    
    330 331
         """
    
    331
    -    if os.path.lexists(path):
    
    332
    -
    
    333
    -        # Try to remove anything that is in the way, but issue
    
    334
    -        # a warning instead if it removes a non empty directory
    
    335
    -        try:
    
    332
    +    try:
    
    333
    +        if S_ISDIR(os.lstat(path).st_mode):
    
    334
    +            os.rmdir(path)
    
    335
    +        else:
    
    336 336
                 os.unlink(path)
    
    337
    -        except OSError as e:
    
    338
    -            if e.errno != errno.EISDIR:
    
    339
    -                raise UtilError("Failed to remove '{}': {}"
    
    340
    -                                .format(path, e))
    
    341
    -
    
    342
    -            try:
    
    343
    -                os.rmdir(path)
    
    344
    -            except OSError as e:
    
    345
    -                if e.errno == errno.ENOTEMPTY:
    
    346
    -                    return False
    
    347
    -                else:
    
    348
    -                    raise UtilError("Failed to remove '{}': {}"
    
    349
    -                                    .format(path, e))
    
    350 337
     
    
    351
    -    return True
    
    338
    +        # File removed/unlinked successfully
    
    339
    +        return True
    
    340
    +
    
    341
    +    except OSError as e:
    
    342
    +        if e.errno == errno.ENOTEMPTY:
    
    343
    +            # Path is non-empty directory
    
    344
    +            return False
    
    345
    +        elif e.errno == errno.ENOENT:
    
    346
    +            # Path does not exist
    
    347
    +            return True
    
    348
    +
    
    349
    +    raise UtilError("Failed to remove '{}': {}"
    
    350
    +                    .format(path, e))
    
    352 351
     
    
    353 352
     
    
    354 353
     def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=False):
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]