[Notes] [Git][BuildStream/buildstream][mac_fixes] 2 commits: Adding darw

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

47c899cf

by knownexus at 2018-09-07T10:03:02Z

Adding darwin.py platform

Adding functionality to recognise Darwin as a platform in plaform.py

05a12b38

by knownexus at 2018-09-07T10:03:09Z

Stopped safe_remove attempting to unlink dirs

Previously safe_remove would attempt to unlink a path
Before attempting to remove it if it was a dir

Now it checks for a dir before that step

6 changed files:

+ buildstream/_platform/darwin.py
buildstream/_platform/linux.py
buildstream/_platform/platform.py
buildstream/_platform/unix.py
buildstream/_project.py
buildstream/utils.py

Changes:

buildstream/_platform/darwin.py

 +#
 +#  Copyright (C) 2017 Codethink Limited
 +#  Copyright (C) 2018 Bloomberg Finance LP
 +#
 +#  This program is free software; you can redistribute it and/or
 +#  modify it under the terms of the GNU Lesser General Public
 +#  License as published by the Free Software Foundation; either
 +#  version 2 of the License, or (at your option) any later version.
 +#
 +#  This library is distributed in the hope that it will be useful,
 +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
 +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
 +#  Lesser General Public License for more details.
 +#
 +#  You should have received a copy of the GNU Lesser General Public
 +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
++
 +import os
 +import resource
++
 +from .._exceptions import PlatformError
 +from ..sandbox import SandboxChroot
++
 +from . import Platform
++
++
 +class Darwin(Platform):
++
 +    def __init__(self, context):
++
 +        # This value comes from OPEN_MAX in syslimits.h
 +        OPEN_MAX = 10240
 +        super().__init__(context)
++
 +    @property
 +    def artifactcache(self):
 +        return self._artifact_cache
++
 +    def create_sandbox(self, *args, **kwargs):
 +        return SandboxChroot(*args, **kwargs)
++
 +    def get_cpu_count(self, cap=None):
 +        if cap < os.cpu_count():
 +            return cap
 +        else:
 +            return os.cpu_count()
++
 +    def set_resources(self, soft_limit=OPEN_MAX, hard_limit=None):
 +        super().set_resources(soft_limit)

buildstream/_platform/linux.py

@@ -41,6 +41,7 @@ class Linux(Platform):
              self._user_ns_available = self._check_user_ns_available(context)
          else:
              self._user_ns_available = False
++
          self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
      @property

buildstream/_platform/platform.py

@@ -22,6 +22,7 @@ import sys
  import resource
  from .._exceptions import PlatformError, ImplError
 +from .._artifactcache.cascache import CASCache
  class Platform():
@@ -39,22 +40,27 @@ class Platform():
      def __init__(self, context):
          self.context = context
          self.set_resources()
 +        self._artifact_cache = CASCache(context)
      @classmethod
      def create_instance(cls, *args, **kwargs):
 -        if sys.platform.startswith('linux'):
 -            backend = 'linux'
 -        else:
 -            backend = 'unix'
          # Meant for testing purposes and therefore hidden in the
          # deepest corners of the source code. Try not to abuse this,
          # please?
          if os.getenv('BST_FORCE_BACKEND'):
              backend = os.getenv('BST_FORCE_BACKEND')
 +        elif sys.platform.startswith('linux'):
 +            backend = 'linux'
 +        elif sys.platform.startswith('darwin'):
 +            backend = 'darwin'
 +        else:
 +            backend = 'unix'
          if backend == 'linux':
              from .linux import Linux as PlatformImpl
 +        elif backend == 'darwin':
 +            from .darwin import Darwin as PlatformImpl
          elif backend == 'unix':
              from .unix import Unix as PlatformImpl
          else:

buildstream/_platform/unix.py

@@ -20,7 +20,6 @@
  import os
  import resource
 -from .._artifactcache.cascache import CASCache
  from .._exceptions import PlatformError
  from ..sandbox import SandboxChroot
@@ -32,7 +31,6 @@ class Unix(Platform):
      def __init__(self, context):
          super().__init__(context)
 -        self._artifact_cache = CASCache(context)
          # Not necessarily 100% reliable, but we want to fail early.
          if os.geteuid() != 0:

buildstream/_project.py

@@ -19,6 +19,7 @@
  #        Tiago Gomes <tiago gomes codethink co uk>
  import os
 +import sys
  from collections import Mapping, OrderedDict
  from pluginbase import PluginBase
  from . import utils
@@ -38,6 +39,7 @@ from ._loader import Loader
  from .element import Element
  from ._message import Message, MessageType
  from ._includes import Includes
 +from ._platform import Platform
  # Project Configuration file
@@ -605,7 +607,8 @@ class Project():
          # Based on some testing (mainly on AWS), maximum effective
          # max-jobs value seems to be around 8-10 if we have enough cores
          # users should set values based on workload and build infrastructure
 -        output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
 +        platform = Platform.get_platform()
 +        output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
          # Export options into variables, if that was requested
          output.options.export_variables(output.base_variables)

buildstream/utils.py

@@ -35,6 +35,7 @@ import tempfile
  import itertools
  import functools
  from contextlib import contextmanager
 +from stat import S_ISDIR
  import psutil
@@ -328,25 +329,25 @@ def safe_remove(path):
      Raises:
         UtilError: In the case of unexpected system call failures
      """
 -    if os.path.lexists(path):
 +    if not S_ISDIR(os.lstat(path).st_mode):
          # Try to remove anything that is in the way, but issue
          # a warning instead if it removes a non empty directory
          try:
              os.unlink(path)
 +            return True
          except OSError as e:
 -            if e.errno != errno.EISDIR:
 -                raise UtilError("Failed to remove '{}': {}"
 -                                .format(path, e))
+-
 -            try:
 -                os.rmdir(path)
 -            except OSError as e:
 -                if e.errno == errno.ENOTEMPTY:
 -                    return False
 -                else:
 -                    raise UtilError("Failed to remove '{}': {}"
 -                                    .format(path, e))
 +            raise UtilError("Failed to remove '{}': {}"
 +                            .format(path, e))
++
 +    try:
 +        os.rmdir(path)
 +    except OSError as e:
 +        if e.errno == errno.ENOTEMPTY:
 +            return False
 +        else:
 +            raise UtilError("Failed to remove '{}': {}"
 +                            .format(path, e))
      return True

[Notes] [Git][BuildStream/buildstream][mac_fixes] 2 commits: Adding darwin.py platform

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

6 changed files:

Changes: