[Notes] [Git][BuildStream/buildstream][mac_fixes] Added MacOSX fixes



Title: GitLab

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

7 changed files:

Changes:

  • buildstream/_frontend/app.py
    ... ... @@ -116,14 +116,6 @@ class App():
    116 116
             else:
    
    117 117
                 self.colors = False
    
    118 118
     
    
    119
    -        # Increase the soft limit for open file descriptors to the maximum.
    
    120
    -        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    
    121
    -        # Avoid hitting the limit too quickly.
    
    122
    -        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    
    123
    -        if limits[0] != limits[1]:
    
    124
    -            # Set soft limit to hard limit
    
    125
    -            resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
    
    126
    -
    
    127 119
         # create()
    
    128 120
         #
    
    129 121
         # Should be used instead of the regular constructor.
    

  • buildstream/_platform/darwin.py
    1
    +#
    
    2
    +#  Copyright (C) 2017 Codethink Limited
    
    3
    +#
    
    4
    +#  This program is free software; you can redistribute it and/or
    
    5
    +#  modify it under the terms of the GNU Lesser General Public
    
    6
    +#  License as published by the Free Software Foundation; either
    
    7
    +#  version 2 of the License, or (at your option) any later version.
    
    8
    +#
    
    9
    +#  This library is distributed in the hope that it will be useful,
    
    10
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    
    11
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    
    12
    +#  Lesser General Public License for more details.
    
    13
    +#
    
    14
    +#  You should have received a copy of the GNU Lesser General Public
    
    15
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    
    16
    +#
    
    17
    +#  Authors:
    
    18
    +#        Phillip Smyth <Phillip Smyth codethink co uk>
    
    19
    +
    
    20
    +import os
    
    21
    +import resource
    
    22
    +
    
    23
    +from .._artifactcache.cascache import CASCache
    
    24
    +from .._exceptions import PlatformError
    
    25
    +from ..sandbox import SandboxChroot
    
    26
    +
    
    27
    +from . import Platform
    
    28
    +
    
    29
    +
    
    30
    +class Darwin(Platform):
    
    31
    +
    
    32
    +    def __init__(self, context):
    
    33
    +
    
    34
    +        super().__init__(context)
    
    35
    +        self._artifact_cache = CASCache(context)
    
    36
    +
    
    37
    +        self.set_resources_limit()
    
    38
    +        # Not necessarily 100% reliable, but we want to fail early.
    
    39
    +        if os.geteuid() != 0:
    
    40
    +            raise PlatformError("Root privileges are required to run without bubblewrap.")
    
    41
    +
    
    42
    +    @property
    
    43
    +    def artifactcache(self):
    
    44
    +        return self._artifact_cache
    
    45
    +
    
    46
    +    def create_sandbox(self, *args, **kwargs):
    
    47
    +        return SandboxChroot(*args, **kwargs)
    
    48
    +
    
    49
    +    def set_resources_limit(self):
    
    50
    +        # Need to set resources for _frontend/app.py as this is dependent on the platform
    
    51
    +        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    
    52
    +        # Avoid hitting the limit too quickly.
    
    53
    +        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    
    54
    +        if limits[0] != limits[1]:
    
    55
    +            resource.setrlimit(resource.RLIMIT_NOFILE, (49152, limits[1]))

  • buildstream/_platform/linux.py
    ... ... @@ -21,6 +21,7 @@ import subprocess
    21 21
     
    
    22 22
     from .. import _site
    
    23 23
     from .. import utils
    
    24
    +import resource
    
    24 25
     from .._artifactcache.cascache import CASCache
    
    25 26
     from .._message import Message, MessageType
    
    26 27
     from ..sandbox import SandboxBwrap
    
    ... ... @@ -37,6 +38,7 @@ class Linux(Platform):
    37 38
             self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
    
    38 39
             self._user_ns_available = self._check_user_ns_available(context)
    
    39 40
             self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
    
    41
    +        self.set_resources_limit()
    
    40 42
     
    
    41 43
         @property
    
    42 44
         def artifactcache(self):
    
    ... ... @@ -48,10 +50,21 @@ class Linux(Platform):
    48 50
             kwargs['die_with_parent_available'] = self._die_with_parent_available
    
    49 51
             return SandboxBwrap(*args, **kwargs)
    
    50 52
     
    
    53
    +    def set_resources_limit(self):
    
    54
    +            # Need to set resources for _frontend/app.py as this is dependent on the platform
    
    55
    +            # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    
    56
    +            # Avoid hitting the limit too quickly.
    
    57
    +            limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    
    58
    +            if limits[0] != limits[1]:
    
    59
    +                resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
    
    60
    +
    
    51 61
         ################################################
    
    52 62
         #              Private Methods                 #
    
    53 63
         ################################################
    
    54 64
         def _check_user_ns_available(self, context):
    
    65
    +        import os
    
    66
    +        if not os.path.exists('/dev/fuse'):
    
    67
    +            return False
    
    55 68
     
    
    56 69
             # Here, lets check if bwrap is able to create user namespaces,
    
    57 70
             # issue a warning if it's not available, and save the state
    

  • buildstream/_platform/platform.py
    ... ... @@ -40,19 +40,22 @@ class Platform():
    40 40
     
    
    41 41
         @classmethod
    
    42 42
         def create_instance(cls, *args, **kwargs):
    
    43
    -        if sys.platform.startswith('linux'):
    
    44
    -            backend = 'linux'
    
    45
    -        else:
    
    46
    -            backend = 'unix'
    
    47
    -
    
    48 43
             # Meant for testing purposes and therefore hidden in the
    
    49 44
             # deepest corners of the source code. Try not to abuse this,
    
    50 45
             # please?
    
    51 46
             if os.getenv('BST_FORCE_BACKEND'):
    
    52 47
                 backend = os.getenv('BST_FORCE_BACKEND')
    
    48
    +        elif sys.platform.startswith('linux'):
    
    49
    +            backend = 'linux'
    
    50
    +        elif sys.platform.startswith('darwin'):
    
    51
    +            backend = 'darwin'
    
    52
    +        else:
    
    53
    +            backend = 'unix'
    
    53 54
     
    
    54 55
             if backend == 'linux':
    
    55 56
                 from .linux import Linux as PlatformImpl
    
    57
    +        elif backend == 'darwin':
    
    58
    +            from .darwin import Darwin as PlatformImpl
    
    56 59
             elif backend == 'unix':
    
    57 60
                 from .unix import Unix as PlatformImpl
    
    58 61
             else:
    
    ... ... @@ -92,3 +95,7 @@ class Platform():
    92 95
         def create_sandbox(self, *args, **kwargs):
    
    93 96
             raise ImplError("Platform {platform} does not implement create_sandbox()"
    
    94 97
                             .format(platform=type(self).__name__))
    
    98
    +
    
    99
    +    def set_resources_limit(self):
    
    100
    +        raise ImplError("Platform {platform} does not implement set_resources_limit()"
    
    101
    +                        .format(platform=type(self).__name__))

  • buildstream/_platform/unix.py
    ... ... @@ -18,6 +18,7 @@
    18 18
     #        Tristan Maat <tristan maat codethink co uk>
    
    19 19
     
    
    20 20
     import os
    
    21
    +import resource
    
    21 22
     
    
    22 23
     from .._artifactcache.cascache import CASCache
    
    23 24
     from .._exceptions import PlatformError
    
    ... ... @@ -32,6 +33,7 @@ class Unix(Platform):
    32 33
     
    
    33 34
             super().__init__(context)
    
    34 35
             self._artifact_cache = CASCache(context)
    
    36
    +        self.set_resources_limit()
    
    35 37
     
    
    36 38
             # Not necessarily 100% reliable, but we want to fail early.
    
    37 39
             if os.geteuid() != 0:
    
    ... ... @@ -43,3 +45,11 @@ class Unix(Platform):
    43 45
     
    
    44 46
         def create_sandbox(self, *args, **kwargs):
    
    45 47
             return SandboxChroot(*args, **kwargs)
    
    48
    +
    
    49
    +    def set_resources_limit(self):
    
    50
    +            # Need to set resources for _frontend/app.py as this is dependent on the platform
    
    51
    +            # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
    
    52
    +            # Avoid hitting the limit too quickly.
    
    53
    +            limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    
    54
    +            if limits[0] != limits[1]:
    
    55
    +                resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))

  • buildstream/_project.py
    ... ... @@ -19,6 +19,7 @@
    19 19
     #        Tiago Gomes <tiago gomes codethink co uk>
    
    20 20
     
    
    21 21
     import os
    
    22
    +import sys
    
    22 23
     from collections import Mapping, OrderedDict
    
    23 24
     from pluginbase import PluginBase
    
    24 25
     from . import utils
    
    ... ... @@ -594,7 +595,7 @@ class Project():
    594 595
             # Based on some testing (mainly on AWS), maximum effective
    
    595 596
             # max-jobs value seems to be around 8-10 if we have enough cores
    
    596 597
             # users should set values based on workload and build infrastructure
    
    597
    -        output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
    
    598
    +        output.base_variables['max-jobs'] = str(min(os.cpu_count(), 8))
    
    598 599
     
    
    599 600
             # Export options into variables, if that was requested
    
    600 601
             output.options.export_variables(output.base_variables)
    

  • tests/testutils/repo/bzr.py
    ... ... @@ -2,6 +2,7 @@ import os
    2 2
     import subprocess
    
    3 3
     import pytest
    
    4 4
     
    
    5
    +from buildstream import utils
    
    5 6
     from .repo import Repo
    
    6 7
     from ..site import HAVE_BZR
    
    7 8
     
    
    ... ... @@ -20,11 +21,11 @@ class Bzr(Repo):
    20 21
         def create(self, directory):
    
    21 22
             branch_dir = os.path.join(self.repo, 'trunk')
    
    22 23
     
    
    23
    -        subprocess.call(['bzr', 'init-repo', self.repo], env=BZR_ENV)
    
    24
    -        subprocess.call(['bzr', 'init', branch_dir], env=BZR_ENV)
    
    24
    +        subprocess.call([utils.get_host_tool('bzr'), 'init-repo', self.repo], env=BZR_ENV)
    
    25
    +        subprocess.call([utils.get_host_tool('bzr'), 'init', branch_dir], env=BZR_ENV)
    
    25 26
             self.copy_directory(directory, branch_dir)
    
    26
    -        subprocess.call(['bzr', 'add', '.'], env=BZR_ENV, cwd=branch_dir)
    
    27
    -        subprocess.call(['bzr', 'commit', '--message="Initial commit"'],
    
    27
    +        subprocess.call([utils.get_host_tool('bzr'), 'add', '.'], env=BZR_ENV, cwd=branch_dir)
    
    28
    +        subprocess.call([utils.get_host_tool('bzr'), 'commit', '--message="Initial commit"'],
    
    28 29
                             env=BZR_ENV, cwd=branch_dir)
    
    29 30
     
    
    30 31
             return self.latest_commit()
    
    ... ... @@ -42,7 +43,7 @@ class Bzr(Repo):
    42 43
     
    
    43 44
         def latest_commit(self):
    
    44 45
             output = subprocess.check_output([
    
    45
    -            'bzr', 'version-info',
    
    46
    +            utils.get_host_tool('bzr'), 'version-info',
    
    46 47
                 '--custom', '--template={revno}',
    
    47 48
                 os.path.join(self.repo, 'trunk')
    
    48 49
             ], env=BZR_ENV)
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]