GNOME.org
 

[Notes] [Git][BuildStream/buildstream][mac_fixes] 4 commits: Moved `resource.getrlimit()`

Title: GitLab

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

7 changed files:

Changes:

  • buildstream/_frontend/app.py
    ... ... @@ -116,14 +116,6 @@ class App():
    116 116 
             else:
    117 117 
                 self.colors = False
    118 118
    119 
    -        # Increase the soft limit for open file descriptors to the maximum.
    120 
    -        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    121 
    -        # Avoid hitting the limit too quickly.
    122 
    -        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    123 
    -        if limits[0] != limits[1]:
    124 
    -            # Set soft limit to hard limit
    125 
    -            resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
    126 
    -
    127 119 
         # create()
    128 120 
         #
    129 121 
         # Should be used instead of the regular constructor.

  • buildstream/_platform/darwin.py
    1 
    +#
    2 
    +#  Copyright (C) 2017 Codethink Limited
    3 
    +#  Copyright (C) 2018 Bloomberg Finance LP
    4 
    +#
    5 
    +#  This program is free software; you can redistribute it and/or
    6 
    +#  modify it under the terms of the GNU Lesser General Public
    7 
    +#  License as published by the Free Software Foundation; either
    8 
    +#  version 2 of the License, or (at your option) any later version.
    9 
    +#
    10 
    +#  This library is distributed in the hope that it will be useful,
    11 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    12 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    13 
    +#  Lesser General Public License for more details.
    14 
    +#
    15 
    +#  You should have received a copy of the GNU Lesser General Public
    16 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    17 
    +
    18 
    +import os
    19 
    +import resource
    20 
    +
    21 
    +from .._exceptions import PlatformError
    22 
    +from ..sandbox import SandboxChroot
    23 
    +
    24 
    +from . import Platform
    25 
    +
    26 
    +
    27 
    +class Darwin(Platform):
    28 
    +
    29 
    +    def __init__(self, context):
    30 
    +
    31 
    +        super().__init__(context)
    32 
    +        self.set_resources(soft_limit=49152)
    33 
    +
    34 
    +    @property
    35 
    +    def artifactcache(self):
    36 
    +        return self._artifact_cache
    37 
    +
    38 
    +    def create_sandbox(self, *args, **kwargs):
    39 
    +        raise ImplError("Platform {platform} does not implement create_sandbox"
    40 
    +                        .format(platform=type(self).__name__))
    41 
    +
    42 
    +    def get_cpu_count(self, cap=None):
    43 
    +        if cap < os.cpu_count():
    44 
    +            return cap
    45 
    +        else:
    46 
    +            return os.cpu_count()

  • buildstream/_platform/linux.py
    ... ... @@ -17,6 +17,7 @@
    17 17 
     #  Authors:
    18 18 
     #        Tristan Maat <tristan maat codethink co uk>
    19 19
    20 
    +import os
    20 21 
     import subprocess
    21 22
    22 23 
     from .. import _site
    ... ... @@ -35,7 +36,12 @@ class Linux(Platform):
    35 36 
             super().__init__(context)
    36 37
    37 38 
             self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
    38 
    -        self._user_ns_available = self._check_user_ns_available(context)
    39 
    +
    40 
    +        if _local_sandbox_available():
    41 
    +            self._user_ns_available = self._check_user_ns_available(context)
    42 
    +        else:
    43 
    +            self._user_ns_available = False
    44 
    +
    39 45 
             self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
    40 46
    41 47 
         @property
    ... ... @@ -51,6 +57,9 @@ class Linux(Platform):
    51 57 
         ################################################
    52 58 
         #              Private Methods                 #
    53 59 
         ################################################
    60 
    +    def _local_sandbox_available(self):
    61 
    +        return os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')
    62 
    +
    54 63 
         def _check_user_ns_available(self, context):
    55 64
    56 65 
             # Here, lets check if bwrap is able to create user namespaces,

  • buildstream/_platform/platform.py
    ... ... @@ -21,6 +21,7 @@ import os
    21 21 
     import sys
    22 22
    23 23 
     from .._exceptions import PlatformError, ImplError
    24 
    +from .._artifactcache.cascache import CASCache
    24 25
    25 26
    26 27 
     class Platform():
    ... ... @@ -37,22 +38,28 @@ class Platform():
    37 38 
         #
    38 39 
         def __init__(self, context):
    39 40 
             self.context = context
    41 
    +        self.set_resources()
    42 
    +        self._artifact_cache = CASCache(context)
    40 43
    41 44 
         @classmethod
    42 45 
         def create_instance(cls, *args, **kwargs):
    43 
    -        if sys.platform.startswith('linux'):
    44 
    -            backend = 'linux'
    45 
    -        else:
    46 
    -            backend = 'unix'
    47 46
    48 47 
             # Meant for testing purposes and therefore hidden in the
    49 48 
             # deepest corners of the source code. Try not to abuse this,
    50 49 
             # please?
    51 50 
             if os.getenv('BST_FORCE_BACKEND'):
    52 51 
                 backend = os.getenv('BST_FORCE_BACKEND')
    52 
    +        elif sys.platform.startswith('linux'):
    53 
    +            backend = 'linux'
    54 
    +        elif sys.platform.startswith('darwin'):
    55 
    +            backend = 'darwin'
    56 
    +        else:
    57 
    +            backend = 'unix'
    53 58
    54 59 
             if backend == 'linux':
    55 60 
                 from .linux import Linux as PlatformImpl
    61 
    +        elif backend == 'darwin':
    62 
    +            from .darwin import Darwin as PlatformImpl
    56 63 
             elif backend == 'unix':
    57 64 
                 from .unix import Unix as PlatformImpl
    58 65 
             else:
    ... ... @@ -92,3 +99,15 @@ class Platform():
    92 99 
         def create_sandbox(self, *args, **kwargs):
    93 100 
             raise ImplError("Platform {platform} does not implement create_sandbox()"
    94 101 
                             .format(platform=type(self).__name__))
    102 
    +
    103 
    +    def set_resources(self, soft_limit=None, hard_limit=None):
    104 
    +        # Need to set resources for _frontend/app.py as this is dependent on the platform
    105 
    +        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    106 
    +        # Avoid hitting the limit too quickly.
    107 
    +        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    108 
    +        if limits[0] != limits[1]:
    109 
    +            if soft_limit is None:
    110 
    +                soft_limit = limits[1]
    111 
    +            if hard_limit is None:
    112 
    +                hard_limit = limits[1]
    113 
    +            resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))

  • buildstream/_platform/unix.py
    ... ... @@ -18,8 +18,8 @@
    18 18 
     #        Tristan Maat <tristan maat codethink co uk>
    19 19
    20 20 
     import os
    21 
    +import resource
    21 22
    22 
    -from .._artifactcache.cascache import CASCache
    23 23 
     from .._exceptions import PlatformError
    24 24 
     from ..sandbox import SandboxChroot
    25 25
    ... ... @@ -31,7 +31,6 @@ class Unix(Platform):
    31 31 
         def __init__(self, context):
    32 32
    33 33 
             super().__init__(context)
    34 
    -        self._artifact_cache = CASCache(context)
    35 34
    36 35 
             # Not necessarily 100% reliable, but we want to fail early.
    37 36 
             if os.geteuid() != 0:

  • buildstream/_project.py
    ... ... @@ -19,6 +19,7 @@
    19 19 
     #        Tiago Gomes <tiago gomes codethink co uk>
    20 20
    21 21 
     import os
    22 
    +import sys
    22 23 
     from collections import Mapping, OrderedDict
    23 24 
     from pluginbase import PluginBase
    24 25 
     from . import utils
    ... ... @@ -38,6 +39,7 @@ from ._loader import Loader
    38 39 
     from .element import Element
    39 40 
     from ._message import Message, MessageType
    40 41 
     from ._includes import Includes
    42 
    +from ._platform import Platform
    41 43
    42 44
    43 45 
     # Project Configuration file
    ... ... @@ -605,7 +607,8 @@ class Project():
    605 607 
             # Based on some testing (mainly on AWS), maximum effective
    606 608 
             # max-jobs value seems to be around 8-10 if we have enough cores
    607 609 
             # users should set values based on workload and build infrastructure
    608 
    -        output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
    610 
    +        platform = Platform.get_platform()
    611 
    +        output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
    609 612
    610 613 
             # Export options into variables, if that was requested
    611 614 
             output.options.export_variables(output.base_variables)

  • buildstream/utils.py
    ... ... @@ -335,7 +335,7 @@ def safe_remove(path):
    335 335 
             try:
    336 336 
                 os.unlink(path)
    337 337 
             except OSError as e:
    338 
    -            if e.errno != errno.EISDIR:
    338 
    +            if e.errno != errno.EISDIR and (e.errno != errno.EPERM or not os.path.isdir(path)):
    339 339 
                     raise UtilError("Failed to remove '{}': {}"
    340 340 
                                     .format(path, e))
    341 341

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]