Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream
Commits:
6 changed files:
- + buildstream/_platform/darwin.py
- buildstream/_platform/linux.py
- buildstream/_platform/platform.py
- buildstream/_platform/unix.py
- buildstream/_project.py
- buildstream/utils.py
Changes:
| 1 |
+#
|
|
| 2 |
+# Copyright (C) 2017 Codethink Limited
|
|
| 3 |
+# Copyright (C) 2018 Bloomberg Finance LP
|
|
| 4 |
+#
|
|
| 5 |
+# This program is free software; you can redistribute it and/or
|
|
| 6 |
+# modify it under the terms of the GNU Lesser General Public
|
|
| 7 |
+# License as published by the Free Software Foundation; either
|
|
| 8 |
+# version 2 of the License, or (at your option) any later version.
|
|
| 9 |
+#
|
|
| 10 |
+# This library is distributed in the hope that it will be useful,
|
|
| 11 |
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
| 12 |
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
| 13 |
+# Lesser General Public License for more details.
|
|
| 14 |
+#
|
|
| 15 |
+# You should have received a copy of the GNU Lesser General Public
|
|
| 16 |
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
|
|
| 17 |
+ |
|
| 18 |
+import os
|
|
| 19 |
+import resource
|
|
| 20 |
+ |
|
| 21 |
+from .._exceptions import PlatformError
|
|
| 22 |
+from ..sandbox import SandboxChroot
|
|
| 23 |
+ |
|
| 24 |
+from . import Platform
|
|
| 25 |
+ |
|
| 26 |
+ |
|
| 27 |
+class Darwin(Platform):
|
|
| 28 |
+ |
|
| 29 |
+ def __init__(self, context):
|
|
| 30 |
+ |
|
| 31 |
+ super().__init__(context)
|
|
| 32 |
+ |
|
| 33 |
+ @property
|
|
| 34 |
+ def artifactcache(self):
|
|
| 35 |
+ return self._artifact_cache
|
|
| 36 |
+ |
|
| 37 |
+ def create_sandbox(self, *args, **kwargs):
|
|
| 38 |
+ raise ImplError("Platform {platform} does not implement create_sandbox"
|
|
| 39 |
+ .format(platform=type(self).__name__))
|
|
| 40 |
+ |
|
| 41 |
+ def get_cpu_count(self, cap=None):
|
|
| 42 |
+ if cap < os.cpu_count():
|
|
| 43 |
+ return cap
|
|
| 44 |
+ else:
|
|
| 45 |
+ return os.cpu_count()
|
|
| 46 |
+ |
|
| 47 |
+ def set_resources(self, soft_limit=49152, hard_limit=None):
|
|
| 48 |
+ super().set_resources(soft_limit)
|
| ... | ... | @@ -41,6 +41,7 @@ class Linux(Platform): |
| 41 | 41 |
self._user_ns_available = self._check_user_ns_available(context)
|
| 42 | 42 |
else:
|
| 43 | 43 |
self._user_ns_available = False
|
| 44 |
+ |
|
| 44 | 45 |
self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
|
| 45 | 46 |
|
| 46 | 47 |
@property
|
| ... | ... | @@ -21,6 +21,7 @@ import os |
| 21 | 21 |
import sys
|
| 22 | 22 |
|
| 23 | 23 |
from .._exceptions import PlatformError, ImplError
|
| 24 |
+from .._artifactcache.cascache import CASCache
|
|
| 24 | 25 |
|
| 25 | 26 |
|
| 26 | 27 |
class Platform():
|
| ... | ... | @@ -38,22 +39,27 @@ class Platform(): |
| 38 | 39 |
def __init__(self, context):
|
| 39 | 40 |
self.context = context
|
| 40 | 41 |
self.set_resources()
|
| 42 |
+ self._artifact_cache = CASCache(context)
|
|
| 41 | 43 |
|
| 42 | 44 |
@classmethod
|
| 43 | 45 |
def create_instance(cls, *args, **kwargs):
|
| 44 |
- if sys.platform.startswith('linux'):
|
|
| 45 |
- backend = 'linux'
|
|
| 46 |
- else:
|
|
| 47 |
- backend = 'unix'
|
|
| 48 | 46 |
|
| 49 | 47 |
# Meant for testing purposes and therefore hidden in the
|
| 50 | 48 |
# deepest corners of the source code. Try not to abuse this,
|
| 51 | 49 |
# please?
|
| 52 | 50 |
if os.getenv('BST_FORCE_BACKEND'):
|
| 53 | 51 |
backend = os.getenv('BST_FORCE_BACKEND')
|
| 52 |
+ elif sys.platform.startswith('linux'):
|
|
| 53 |
+ backend = 'linux'
|
|
| 54 |
+ elif sys.platform.startswith('darwin'):
|
|
| 55 |
+ backend = 'darwin'
|
|
| 56 |
+ else:
|
|
| 57 |
+ backend = 'unix'
|
|
| 54 | 58 |
|
| 55 | 59 |
if backend == 'linux':
|
| 56 | 60 |
from .linux import Linux as PlatformImpl
|
| 61 |
+ elif backend == 'darwin':
|
|
| 62 |
+ from .darwin import Darwin as PlatformImpl
|
|
| 57 | 63 |
elif backend == 'unix':
|
| 58 | 64 |
from .unix import Unix as PlatformImpl
|
| 59 | 65 |
else:
|
| ... | ... | @@ -20,7 +20,6 @@ |
| 20 | 20 |
import os
|
| 21 | 21 |
import resource
|
| 22 | 22 |
|
| 23 |
-from .._artifactcache.cascache import CASCache
|
|
| 24 | 23 |
from .._exceptions import PlatformError
|
| 25 | 24 |
from ..sandbox import SandboxChroot
|
| 26 | 25 |
|
| ... | ... | @@ -32,7 +31,6 @@ class Unix(Platform): |
| 32 | 31 |
def __init__(self, context):
|
| 33 | 32 |
|
| 34 | 33 |
super().__init__(context)
|
| 35 |
- self._artifact_cache = CASCache(context)
|
|
| 36 | 34 |
|
| 37 | 35 |
# Not necessarily 100% reliable, but we want to fail early.
|
| 38 | 36 |
if os.geteuid() != 0:
|
| ... | ... | @@ -19,6 +19,7 @@ |
| 19 | 19 |
# Tiago Gomes <tiago gomes codethink co uk>
|
| 20 | 20 |
|
| 21 | 21 |
import os
|
| 22 |
+import sys
|
|
| 22 | 23 |
from collections import Mapping, OrderedDict
|
| 23 | 24 |
from pluginbase import PluginBase
|
| 24 | 25 |
from . import utils
|
| ... | ... | @@ -38,6 +39,7 @@ from ._loader import Loader |
| 38 | 39 |
from .element import Element
|
| 39 | 40 |
from ._message import Message, MessageType
|
| 40 | 41 |
from ._includes import Includes
|
| 42 |
+from ._platform import Platform
|
|
| 41 | 43 |
|
| 42 | 44 |
|
| 43 | 45 |
# Project Configuration file
|
| ... | ... | @@ -605,7 +607,8 @@ class Project(): |
| 605 | 607 |
# Based on some testing (mainly on AWS), maximum effective
|
| 606 | 608 |
# max-jobs value seems to be around 8-10 if we have enough cores
|
| 607 | 609 |
# users should set values based on workload and build infrastructure
|
| 608 |
- output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
|
|
| 610 |
+ platform = Platform.get_platform()
|
|
| 611 |
+ output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
|
|
| 609 | 612 |
|
| 610 | 613 |
# Export options into variables, if that was requested
|
| 611 | 614 |
output.options.export_variables(output.base_variables)
|
| ... | ... | @@ -335,7 +335,7 @@ def safe_remove(path): |
| 335 | 335 |
try:
|
| 336 | 336 |
os.unlink(path)
|
| 337 | 337 |
except OSError as e:
|
| 338 |
- if e.errno != errno.EISDIR:
|
|
| 338 |
+ if e.errno != errno.EISDIR and (e.errno != errno.EPERM or not os.path.isdir(path)):
|
|
| 339 | 339 |
raise UtilError("Failed to remove '{}': {}"
|
| 340 | 340 |
.format(path, e))
|
| 341 | 341 |
|