GNOME.org
 

[Notes] [Git][BuildStream/buildstream][mac_fixes] :wAdded MacOSX fixes

Title: GitLab

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

7 changed files:

Changes:

  • buildstream/_frontend/app.py
    ... ... @@ -120,9 +120,10 @@ class App():
    120 120 
             # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    121 121 
             # Avoid hitting the limit too quickly.
    122 122 
             limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    123 
    -        if limits[0] != limits[1]:
    123 
    +        # This does not work on MacOSX
    124 
    +        if limits[0] != limits[1] and not os.getenv('BST_FORCE_BACKEND', sys.platform).startswith('darwin'):
    124 125 
                 # Set soft limit to hard limit
    125 
    -            resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
    126 
    +            resource.setrlimit(resource.RLIMIT_NOFILE, (49152, limits[1]))
    126 127
    127 128 
         # create()
    128 129 
         #

  • buildstream/_platform/darwin.py
    1 
    +#
    2 
    +#  Copyright (C) 2017 Codethink Limited
    3 
    +#
    4 
    +#  This program is free software; you can redistribute it and/or
    5 
    +#  modify it under the terms of the GNU Lesser General Public
    6 
    +#  License as published by the Free Software Foundation; either
    7 
    +#  version 2 of the License, or (at your option) any later version.
    8 
    +#
    9 
    +#  This library is distributed in the hope that it will be useful,
    10 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    11 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    12 
    +#  Lesser General Public License for more details.
    13 
    +#
    14 
    +#  You should have received a copy of the GNU Lesser General Public
    15 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    16 
    +#
    17 
    +#  Authors:
    18 
    +#        Phillip Smyth <Phillip Smyth codethink co uk>
    19 
    +
    20 
    +import os
    21 
    +import resource
    22 
    +
    23 
    +from .._artifactcache.cascache import CASCache
    24 
    +from .._exceptions import PlatformError
    25 
    +from ..sandbox import SandboxChroot
    26 
    +
    27 
    +from . import Platform
    28 
    +
    29 
    +
    30 
    +class Darwin(Platform):
    31 
    +
    32 
    +    def __init__(self, context):
    33 
    +
    34 
    +        super().__init__(context)
    35 
    +        self._artifact_cache = CASCache(context)
    36 
    +
    37 
    +        self.set_resources_limit()
    38 
    +        # Not necessarily 100% reliable, but we want to fail early.
    39 
    +        if os.geteuid() != 0:
    40 
    +            raise PlatformError("Root privileges are required to run without bubblewrap.")
    41 
    +
    42 
    +    @property
    43 
    +    def artifactcache(self):
    44 
    +        return self._artifact_cache
    45 
    +
    46 
    +    def create_sandbox(self, *args, **kwargs):
    47 
    +        return SandboxChroot(*args, **kwargs)
    48 
    +
    49 
    +    def set_resources_limit(self):
    50 
    +        # Need to set resources for _frontend/app.py as this is dependent on the platform
    51 
    +        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    52 
    +        # Avoid hitting the limit too quickly.
    53 
    +        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    54 
    +        if limits[0] != limits[1]:
    55 
    +            resource.setrlimit(resource.RLIMIT_NOFILE, (49152, limits[1]))
    56 
    +
    57 
    +

  • buildstream/_platform/linux.py
    ... ... @@ -21,6 +21,7 @@ import subprocess
    21 21
    22 22 
     from .. import _site
    23 23 
     from .. import utils
    24 
    +import resource
    24 25 
     from .._artifactcache.cascache import CASCache
    25 26 
     from .._message import Message, MessageType
    26 27 
     from ..sandbox import SandboxBwrap
    ... ... @@ -37,6 +38,7 @@ class Linux(Platform):
    37 38 
             self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
    38 39 
             self._user_ns_available = self._check_user_ns_available(context)
    39 40 
             self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
    41 
    +        self.set_resources_limit()
    40 42
    41 43 
         @property
    42 44 
         def artifactcache(self):
    ... ... @@ -48,10 +50,21 @@ class Linux(Platform):
    48 50 
             kwargs['die_with_parent_available'] = self._die_with_parent_available
    49 51 
             return SandboxBwrap(*args, **kwargs)
    50 52
    53 
    +    def set_resources_limit(self):
    54 
    +            # Need to set resources for _frontend/app.py as this is dependent on the platform
    55 
    +            # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    56 
    +            # Avoid hitting the limit too quickly.
    57 
    +            limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    58 
    +            if limits[0] != limits[1]:
    59 
    +                resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
    60 
    +
    51 61 
         ################################################
    52 62 
         #              Private Methods                 #
    53 63 
         ################################################
    54 64 
         def _check_user_ns_available(self, context):
    65 
    +        import os
    66 
    +        if not os.path.exists('/dev/fuse')
    67 
    +            return False
    55 68
    56 69 
             # Here, lets check if bwrap is able to create user namespaces,
    57 70 
             # issue a warning if it's not available, and save the state

  • buildstream/_platform/platform.py
    ... ... @@ -40,19 +40,22 @@ class Platform():
    40 40
    41 41 
         @classmethod
    42 42 
         def create_instance(cls, *args, **kwargs):
    43 
    -        if sys.platform.startswith('linux'):
    44 
    -            backend = 'linux'
    45 
    -        else:
    46 
    -            backend = 'unix'
    47 
    -
    48 43 
             # Meant for testing purposes and therefore hidden in the
    49 44 
             # deepest corners of the source code. Try not to abuse this,
    50 45 
             # please?
    51 46 
             if os.getenv('BST_FORCE_BACKEND'):
    52 47 
                 backend = os.getenv('BST_FORCE_BACKEND')
    48 
    +        elif sys.platform.startswith('linux'):
    49 
    +            backend = 'linux'
    50 
    +        elif sys.platform.startswith('darwin'):
    51 
    +            backend = 'darwin'
    52 
    +        else:
    53 
    +            backend = 'unix'
    53 54
    54 55 
             if backend == 'linux':
    55 56 
                 from .linux import Linux as PlatformImpl
    57 
    +        elif backend == 'darwin':
    58 
    +            from .darwin import Darwin as PlatformImpl
    56 59 
             elif backend == 'unix':
    57 60 
                 from .unix import Unix as PlatformImpl
    58 61 
             else:
    ... ... @@ -92,3 +95,7 @@ class Platform():
    92 95 
         def create_sandbox(self, *args, **kwargs):
    93 96 
             raise ImplError("Platform {platform} does not implement create_sandbox()"
    94 97 
                             .format(platform=type(self).__name__))
    98 
    +
    99 
    +    def set_resources_limit(self):
    100 
    +        raise ImplError("Platform {platform} does not implement set_resources_limit()"
    101 
    +                        .format(platform=type(self).__name__))

  • buildstream/_platform/unix.py
    ... ... @@ -18,6 +18,7 @@
    18 18 
     #        Tristan Maat <tristan maat codethink co uk>
    19 19
    20 20 
     import os
    21 
    +import resource
    21 22
    22 23 
     from .._artifactcache.cascache import CASCache
    23 24 
     from .._exceptions import PlatformError
    ... ... @@ -32,6 +33,7 @@ class Unix(Platform):
    32 33
    33 34 
             super().__init__(context)
    34 35 
             self._artifact_cache = CASCache(context)
    36 
    +        self.set_resources_limit()
    35 37
    36 38 
             # Not necessarily 100% reliable, but we want to fail early.
    37 39 
             if os.geteuid() != 0:
    ... ... @@ -43,3 +45,11 @@ class Unix(Platform):
    43 45
    44 46 
         def create_sandbox(self, *args, **kwargs):
    45 47 
             return SandboxChroot(*args, **kwargs)
    48 
    +
    49 
    +    def set_resources_limit(self):
    50 
    +            # Need to set resources for _frontend/app.py as this is dependent on the platform
    51 
    +            # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    52 
    +            # Avoid hitting the limit too quickly.
    53 
    +            limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    54 
    +            if limits[0] != limits[1]:
    55 
    +                resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))

  • buildstream/_project.py
    ... ... @@ -19,6 +19,7 @@
    19 19 
     #        Tiago Gomes <tiago gomes codethink co uk>
    20 20
    21 21 
     import os
    22 
    +import sys
    22 23 
     from collections import Mapping, OrderedDict
    23 24 
     from pluginbase import PluginBase
    24 25 
     from . import utils
    ... ... @@ -574,7 +575,7 @@ class Project():
    574 575 
             # Based on some testing (mainly on AWS), maximum effective
    575 576 
             # max-jobs value seems to be around 8-10 if we have enough cores
    576 577 
             # users should set values based on workload and build infrastructure
    577 
    -        output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
    578 
    +        output.base_variables['max-jobs'] = str(min(os.cpu_count(), 8))
    578 579
    579 580 
             # Export options into variables, if that was requested
    580 581 
             output.options.export_variables(output.base_variables)

  • tests/testutils/repo/bzr.py
    ... ... @@ -20,12 +20,12 @@ class Bzr(Repo):
    20 20 
         def create(self, directory):
    21 21 
             branch_dir = os.path.join(self.repo, 'trunk')
    22 22
    23 
    -        subprocess.call(['bzr', 'init-repo', self.repo], env=BZR_ENV)
    24 
    -        subprocess.call(['bzr', 'init', branch_dir], env=BZR_ENV)
    23 
    +        subprocess.call(['bzr', 'init-repo', self.repo], env=BZR_ENV, shell=True)
    24 
    +        subprocess.call(['bzr', 'init', branch_dir], env=BZR_ENV, shell=True)
    25 25 
             self.copy_directory(directory, branch_dir)
    26 
    -        subprocess.call(['bzr', 'add', '.'], env=BZR_ENV, cwd=branch_dir)
    26 
    +        subprocess.call(['bzr', 'add', '.'], env=BZR_ENV, cwd=branch_dir, shell=True)
    27 27 
             subprocess.call(['bzr', 'commit', '--message="Initial commit"'],
    28 
    -                        env=BZR_ENV, cwd=branch_dir)
    28 
    +                        env=BZR_ENV, cwd=branch_dir, shell=True)
    29 29
    30 30 
             return self.latest_commit()
    31 31
    ... ... @@ -45,5 +45,5 @@ class Bzr(Repo):
    45 45 
                 'bzr', 'version-info',
    46 46 
                 '--custom', '--template={revno}',
    47 47 
                 os.path.join(self.repo, 'trunk')
    48 
    -        ], env=BZR_ENV)
    48 
    +        ], env=BZR_ENV, shell=True)
    49 49 
             return output.decode('UTF-8').strip()

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]