Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream
Commits:
-
b4146a0f
by knownexus at 2018-09-19T10:14:07Z
-
2f79a947
by knownexus at 2018-09-19T10:16:40Z
-
182f48a9
by knownexus at 2018-09-19T10:17:45Z
-
9eb4b418
by knownexus at 2018-09-19T10:19:06Z
-
c6b830c8
by knownexus at 2018-09-19T10:47:14Z
-
50b21955
by knownexus at 2018-09-19T10:48:03Z
-
3abbfb21
by knownexus at 2018-09-19T10:48:21Z
9 changed files:
- buildstream/_frontend/app.py
- + buildstream/_platform/darwin.py
- buildstream/_platform/linux.py
- buildstream/_platform/platform.py
- buildstream/_platform/unix.py
- buildstream/_project.py
- buildstream/sandbox/__init__.py
- + buildstream/sandbox/_sandboxdummy.py
- buildstream/utils.py
Changes:
... | ... | @@ -116,14 +116,6 @@ class App(): |
116 | 116 |
else:
|
117 | 117 |
self.colors = False
|
118 | 118 |
|
119 |
- # Increase the soft limit for open file descriptors to the maximum.
|
|
120 |
- # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
|
|
121 |
- # Avoid hitting the limit too quickly.
|
|
122 |
- limits = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
123 |
- if limits[0] != limits[1]:
|
|
124 |
- # Set soft limit to hard limit
|
|
125 |
- resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
|
|
126 |
- |
|
127 | 119 |
# create()
|
128 | 120 |
#
|
129 | 121 |
# Should be used instead of the regular constructor.
|
1 |
+#
|
|
2 |
+# Copyright (C) 2017 Codethink Limited
|
|
3 |
+# Copyright (C) 2018 Bloomberg Finance LP
|
|
4 |
+#
|
|
5 |
+# This program is free software; you can redistribute it and/or
|
|
6 |
+# modify it under the terms of the GNU Lesser General Public
|
|
7 |
+# License as published by the Free Software Foundation; either
|
|
8 |
+# version 2 of the License, or (at your option) any later version.
|
|
9 |
+#
|
|
10 |
+# This library is distributed in the hope that it will be useful,
|
|
11 |
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12 |
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
13 |
+# Lesser General Public License for more details.
|
|
14 |
+#
|
|
15 |
+# You should have received a copy of the GNU Lesser General Public
|
|
16 |
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
|
|
17 |
+ |
|
18 |
+import os
|
|
19 |
+import resource
|
|
20 |
+ |
|
21 |
+from .._exceptions import PlatformError
|
|
22 |
+from ..sandbox import SandboxChroot, SandboxDummy
|
|
23 |
+ |
|
24 |
+from . import Platform
|
|
25 |
+ |
|
26 |
+ |
|
27 |
+class Darwin(Platform):
|
|
28 |
+ |
|
29 |
+ # This value comes from OPEN_MAX in syslimits.h
|
|
30 |
+ OPEN_MAX = 10240
|
|
31 |
+ |
|
32 |
+ def __init__(self, context):
|
|
33 |
+ |
|
34 |
+ super().__init__(context)
|
|
35 |
+ |
|
36 |
+ @property
|
|
37 |
+ def artifactcache(self):
|
|
38 |
+ return self._artifact_cache
|
|
39 |
+ |
|
40 |
+ def create_sandbox(self, *args, **kwargs):
|
|
41 |
+ return SandboxDummy(*args, **kwargs)
|
|
42 |
+ |
|
43 |
+ def get_cpu_count(self, cap=None):
|
|
44 |
+ if cap < os.cpu_count():
|
|
45 |
+ return cap
|
|
46 |
+ else:
|
|
47 |
+ return os.cpu_count()
|
|
48 |
+ |
|
49 |
+ def set_resource_limits(self, soft_limit=OPEN_MAX, hard_limit=None):
|
|
50 |
+ super().set_resource_limits(soft_limit)
|
... | ... | @@ -17,13 +17,14 @@ |
17 | 17 |
# Authors:
|
18 | 18 |
# Tristan Maat <tristan maat codethink co uk>
|
19 | 19 |
|
20 |
+import os
|
|
20 | 21 |
import subprocess
|
21 | 22 |
|
22 | 23 |
from .. import _site
|
23 | 24 |
from .. import utils
|
24 | 25 |
from .._artifactcache.cascache import CASCache
|
25 | 26 |
from .._message import Message, MessageType
|
26 |
-from ..sandbox import SandboxBwrap
|
|
27 |
+from ..sandbox import SandboxBwrap, SandboxDummy
|
|
27 | 28 |
|
28 | 29 |
from . import Platform
|
29 | 30 |
|
... | ... | @@ -32,27 +33,40 @@ class Linux(Platform): |
32 | 33 |
|
33 | 34 |
def __init__(self, context):
|
34 | 35 |
|
35 |
- super().__init__(context)
|
|
36 |
- |
|
37 | 36 |
self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
|
38 |
- self._user_ns_available = self._check_user_ns_available(context)
|
|
39 |
- self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
|
|
37 |
+ |
|
38 |
+ if self._local_sandbox_available():
|
|
39 |
+ self._user_ns_available = self._check_user_ns_available(context)
|
|
40 |
+ else:
|
|
41 |
+ self._user_ns_available = False
|
|
42 |
+ |
|
43 |
+ # _user_ns_available needs to be set before chaining up to the super class
|
|
44 |
+ # This is because it will call create_artifact_cache()
|
|
45 |
+ super().__init__(context)
|
|
40 | 46 |
|
41 | 47 |
@property
|
42 | 48 |
def artifactcache(self):
|
43 | 49 |
return self._artifact_cache
|
44 | 50 |
|
45 | 51 |
def create_sandbox(self, *args, **kwargs):
|
46 |
- # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
|
|
47 |
- kwargs['user_ns_available'] = self._user_ns_available
|
|
48 |
- kwargs['die_with_parent_available'] = self._die_with_parent_available
|
|
49 |
- return SandboxBwrap(*args, **kwargs)
|
|
52 |
+ if not self._local_sandbox_available():
|
|
53 |
+ return SandboxDummy(*args, **kwargs)
|
|
54 |
+ else:
|
|
55 |
+ # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
|
|
56 |
+ kwargs['user_ns_available'] = self._user_ns_available
|
|
57 |
+ kwargs['die_with_parent_available'] = self._die_with_parent_available
|
|
58 |
+ return SandboxBwrap(*args, **kwargs)
|
|
59 |
+ |
|
60 |
+ def create_artifact_cache(self, context, *, enable_push):
|
|
61 |
+ return super().create_artifact_cache(context, *, self._user_ns_available)
|
|
50 | 62 |
|
51 | 63 |
################################################
|
52 | 64 |
# Private Methods #
|
53 | 65 |
################################################
|
54 |
- def _check_user_ns_available(self, context):
|
|
66 |
+ def _local_sandbox_available(self):
|
|
67 |
+ return os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')
|
|
55 | 68 |
|
69 |
+ def _check_user_ns_available(self, context):
|
|
56 | 70 |
# Here, lets check if bwrap is able to create user namespaces,
|
57 | 71 |
# issue a warning if it's not available, and save the state
|
58 | 72 |
# locally so that we can inform the sandbox to not try it
|
... | ... | @@ -19,8 +19,10 @@ |
19 | 19 |
|
20 | 20 |
import os
|
21 | 21 |
import sys
|
22 |
+import resource
|
|
22 | 23 |
|
23 | 24 |
from .._exceptions import PlatformError, ImplError
|
25 |
+from .._artifactcache.cascache import CASCache
|
|
24 | 26 |
|
25 | 27 |
|
26 | 28 |
class Platform():
|
... | ... | @@ -37,22 +39,28 @@ class Platform(): |
37 | 39 |
#
|
38 | 40 |
def __init__(self, context):
|
39 | 41 |
self.context = context
|
42 |
+ self.set_resource_limits()
|
|
43 |
+ self._artifact_cache = self.create_artifact_cache(context, True)
|
|
40 | 44 |
|
41 | 45 |
@classmethod
|
42 | 46 |
def create_instance(cls, *args, **kwargs):
|
43 |
- if sys.platform.startswith('linux'):
|
|
44 |
- backend = 'linux'
|
|
45 |
- else:
|
|
46 |
- backend = 'unix'
|
|
47 | 47 |
|
48 | 48 |
# Meant for testing purposes and therefore hidden in the
|
49 | 49 |
# deepest corners of the source code. Try not to abuse this,
|
50 | 50 |
# please?
|
51 | 51 |
if os.getenv('BST_FORCE_BACKEND'):
|
52 | 52 |
backend = os.getenv('BST_FORCE_BACKEND')
|
53 |
+ elif sys.platform.startswith('linux'):
|
|
54 |
+ backend = 'linux'
|
|
55 |
+ elif sys.platform.startswith('darwin'):
|
|
56 |
+ backend = 'darwin'
|
|
57 |
+ else:
|
|
58 |
+ backend = 'unix'
|
|
53 | 59 |
|
54 | 60 |
if backend == 'linux':
|
55 | 61 |
from .linux import Linux as PlatformImpl
|
62 |
+ elif backend == 'darwin':
|
|
63 |
+ from .darwin import Darwin as PlatformImpl
|
|
56 | 64 |
elif backend == 'unix':
|
57 | 65 |
from .unix import Unix as PlatformImpl
|
58 | 66 |
else:
|
... | ... | @@ -66,6 +74,9 @@ class Platform(): |
66 | 74 |
raise PlatformError("Platform needs to be initialized first")
|
67 | 75 |
return cls._instance
|
68 | 76 |
|
77 |
+ def get_cpu_count(self, cap=None):
|
|
78 |
+ return min(len(os.sched_getaffinity(0)), cap)
|
|
79 |
+ |
|
69 | 80 |
##################################################################
|
70 | 81 |
# Platform properties #
|
71 | 82 |
##################################################################
|
... | ... | @@ -92,3 +103,18 @@ class Platform(): |
92 | 103 |
def create_sandbox(self, *args, **kwargs):
|
93 | 104 |
raise ImplError("Platform {platform} does not implement create_sandbox()"
|
94 | 105 |
.format(platform=type(self).__name__))
|
106 |
+ |
|
107 |
+ def set_resource_limits(self, soft_limit=None, hard_limit=None):
|
|
108 |
+ # Need to set resources for _frontend/app.py as this is dependent on the platform
|
|
109 |
+ # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
|
|
110 |
+ # Avoid hitting the limit too quickly.
|
|
111 |
+ limits = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
112 |
+ if limits[0] != limits[1]:
|
|
113 |
+ if soft_limit is None:
|
|
114 |
+ soft_limit = limits[1]
|
|
115 |
+ if hard_limit is None:
|
|
116 |
+ hard_limit = limits[1]
|
|
117 |
+ resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
|
|
118 |
+ |
|
119 |
+ def create_artifact_cache(self, context, *, enable_push=True):
|
|
120 |
+ return CASCache(context=context, *, enable_push=enable_push)
|
... | ... | @@ -31,7 +31,6 @@ class Unix(Platform): |
31 | 31 |
def __init__(self, context):
|
32 | 32 |
|
33 | 33 |
super().__init__(context)
|
34 |
- self._artifact_cache = CASCache(context)
|
|
35 | 34 |
|
36 | 35 |
# Not necessarily 100% reliable, but we want to fail early.
|
37 | 36 |
if os.geteuid() != 0:
|
... | ... | @@ -38,6 +38,7 @@ from ._loader import Loader |
38 | 38 |
from .element import Element
|
39 | 39 |
from ._message import Message, MessageType
|
40 | 40 |
from ._includes import Includes
|
41 |
+from ._platform import Platform
|
|
41 | 42 |
|
42 | 43 |
|
43 | 44 |
# Project Configuration file
|
... | ... | @@ -617,7 +618,8 @@ class Project(): |
617 | 618 |
# Based on some testing (mainly on AWS), maximum effective
|
618 | 619 |
# max-jobs value seems to be around 8-10 if we have enough cores
|
619 | 620 |
# users should set values based on workload and build infrastructure
|
620 |
- output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
|
|
621 |
+ platform = Platform.get_platform()
|
|
622 |
+ output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
|
|
621 | 623 |
|
622 | 624 |
# Export options into variables, if that was requested
|
623 | 625 |
output.options.export_variables(output.base_variables)
|
... | ... | @@ -21,3 +21,4 @@ from .sandbox import Sandbox, SandboxFlags |
21 | 21 |
from ._sandboxchroot import SandboxChroot
|
22 | 22 |
from ._sandboxbwrap import SandboxBwrap
|
23 | 23 |
from ._sandboxremote import SandboxRemote
|
24 |
+from ._sandboxdummy import SandboxDummy
|
1 |
+#
|
|
2 |
+# Copyright (C) 2017 Codethink Limited
|
|
3 |
+#
|
|
4 |
+# This program is free software; you can redistribute it and/or
|
|
5 |
+# modify it under the terms of the GNU Lesser General Public
|
|
6 |
+# License as published by the Free Software Foundation; either
|
|
7 |
+# version 2 of the License, or (at your option) any later version.
|
|
8 |
+#
|
|
9 |
+# This library is distributed in the hope that it will be useful,
|
|
10 |
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11 |
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
12 |
+# Lesser General Public License for more details.
|
|
13 |
+#
|
|
14 |
+# You should have received a copy of the GNU Lesser General Public
|
|
15 |
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
|
|
16 |
+#
|
|
17 |
+# Authors:
|
|
18 |
+ |
|
19 |
+from .._exceptions import SandboxError
|
|
20 |
+from . import Sandbox
|
|
21 |
+ |
|
22 |
+ |
|
23 |
+class SandboxDummy(Sandbox):
|
|
24 |
+ def __init__(self, *args, **kwargs):
|
|
25 |
+ super().__init__(*args, **kwargs)
|
|
26 |
+ |
|
27 |
+ def run(self, command, flags, *, cwd=None, env=None):
|
|
28 |
+ |
|
29 |
+ # Fallback to the sandbox default settings for
|
|
30 |
+ # the cwd and env.
|
|
31 |
+ #
|
|
32 |
+ cwd = self._get_work_directory(cwd=cwd)
|
|
33 |
+ env = self._get_environment(cwd=cwd, env=env)
|
|
34 |
+ |
|
35 |
+ if not self._has_command(command[0], env):
|
|
36 |
+ raise SandboxError("Staged artifacts do not provide command "
|
|
37 |
+ "'{}'".format(command[0]),
|
|
38 |
+ reason='missing-command')
|
|
39 |
+ |
|
40 |
+ raise SandboxError("This platform does not support local builds")
|
... | ... | @@ -35,6 +35,7 @@ import tempfile |
35 | 35 |
import itertools
|
36 | 36 |
import functools
|
37 | 37 |
from contextlib import contextmanager
|
38 |
+from stat import S_ISDIR
|
|
38 | 39 |
|
39 | 40 |
import psutil
|
40 | 41 |
|
... | ... | @@ -328,27 +329,25 @@ def safe_remove(path): |
328 | 329 |
Raises:
|
329 | 330 |
UtilError: In the case of unexpected system call failures
|
330 | 331 |
"""
|
331 |
- if os.path.lexists(path):
|
|
332 |
- |
|
333 |
- # Try to remove anything that is in the way, but issue
|
|
334 |
- # a warning instead if it removes a non empty directory
|
|
335 |
- try:
|
|
332 |
+ try:
|
|
333 |
+ if S_ISDIR(os.lstat(path).st_mode):
|
|
334 |
+ os.rmdir(path)
|
|
335 |
+ else:
|
|
336 | 336 |
os.unlink(path)
|
337 |
- except OSError as e:
|
|
338 |
- if e.errno != errno.EISDIR:
|
|
339 |
- raise UtilError("Failed to remove '{}': {}"
|
|
340 |
- .format(path, e))
|
|
341 |
- |
|
342 |
- try:
|
|
343 |
- os.rmdir(path)
|
|
344 |
- except OSError as e:
|
|
345 |
- if e.errno == errno.ENOTEMPTY:
|
|
346 |
- return False
|
|
347 |
- else:
|
|
348 |
- raise UtilError("Failed to remove '{}': {}"
|
|
349 |
- .format(path, e))
|
|
350 | 337 |
|
351 |
- return True
|
|
338 |
+ # File removed/unlinked successfully
|
|
339 |
+ return True
|
|
340 |
+ |
|
341 |
+ except OSError as e:
|
|
342 |
+ if e.errno == errno.ENOTEMPTY:
|
|
343 |
+ # Path is non-empty directory
|
|
344 |
+ return False
|
|
345 |
+ elif e.errno == errno.ENOENT:
|
|
346 |
+ # Path does not exist
|
|
347 |
+ return True
|
|
348 |
+ |
|
349 |
+ raise UtilError("Failed to remove '{}': {}"
|
|
350 |
+ .format(path, e))
|
|
352 | 351 |
|
353 | 352 |
|
354 | 353 |
def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=False):
|