Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream
Commits:
-
5d155705
by knownexus at 2018-08-29T12:45:15Z
7 changed files:
- buildstream/_frontend/app.py
- + buildstream/_platform/darwin.py
- buildstream/_platform/linux.py
- buildstream/_platform/platform.py
- buildstream/_platform/unix.py
- buildstream/_project.py
- tests/testutils/repo/bzr.py
Changes:
... | ... | @@ -116,14 +116,6 @@ class App(): |
116 | 116 |
else:
|
117 | 117 |
self.colors = False
|
118 | 118 |
|
119 |
- # Increase the soft limit for open file descriptors to the maximum.
|
|
120 |
- # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
|
|
121 |
- # Avoid hitting the limit too quickly.
|
|
122 |
- limits = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
123 |
- if limits[0] != limits[1]:
|
|
124 |
- # Set soft limit to hard limit
|
|
125 |
- resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
|
|
126 |
- |
|
127 | 119 |
# create()
|
128 | 120 |
#
|
129 | 121 |
# Should be used instead of the regular constructor.
|
1 |
+#
|
|
2 |
+# Copyright (C) 2017 Codethink Limited
|
|
3 |
+#
|
|
4 |
+# This program is free software; you can redistribute it and/or
|
|
5 |
+# modify it under the terms of the GNU Lesser General Public
|
|
6 |
+# License as published by the Free Software Foundation; either
|
|
7 |
+# version 2 of the License, or (at your option) any later version.
|
|
8 |
+#
|
|
9 |
+# This library is distributed in the hope that it will be useful,
|
|
10 |
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11 |
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
12 |
+# Lesser General Public License for more details.
|
|
13 |
+#
|
|
14 |
+# You should have received a copy of the GNU Lesser General Public
|
|
15 |
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
|
|
16 |
+#
|
|
17 |
+# Authors:
|
|
18 |
+# Phillip Smyth <Phillip Smyth codethink co uk>
|
|
19 |
+ |
|
20 |
+import os
|
|
21 |
+import resource
|
|
22 |
+ |
|
23 |
+from .._artifactcache.cascache import CASCache
|
|
24 |
+from .._exceptions import PlatformError
|
|
25 |
+from ..sandbox import SandboxChroot
|
|
26 |
+ |
|
27 |
+from . import Platform
|
|
28 |
+ |
|
29 |
+ |
|
30 |
+class Darwin(Platform):
|
|
31 |
+ |
|
32 |
+ def __init__(self, context):
|
|
33 |
+ |
|
34 |
+ super().__init__(context)
|
|
35 |
+ self._artifact_cache = CASCache(context)
|
|
36 |
+ |
|
37 |
+ self.set_resources_limit()
|
|
38 |
+ # Not necessarily 100% reliable, but we want to fail early.
|
|
39 |
+ if os.geteuid() != 0:
|
|
40 |
+ raise PlatformError("Root privileges are required to run without bubblewrap.")
|
|
41 |
+ |
|
42 |
+ @property
|
|
43 |
+ def artifactcache(self):
|
|
44 |
+ return self._artifact_cache
|
|
45 |
+ |
|
46 |
+ def create_sandbox(self, *args, **kwargs):
|
|
47 |
+ return SandboxChroot(*args, **kwargs)
|
|
48 |
+ |
|
49 |
+ def set_resources_limit(self):
|
|
50 |
+ # Need to set resources for _frontend/app.py as this is dependent on the platform
|
|
51 |
+ # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
|
|
52 |
+ # Avoid hitting the limit too quickly.
|
|
53 |
+ limits = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
54 |
+ if limits[0] != limits[1]:
|
|
55 |
+ resource.setrlimit(resource.RLIMIT_NOFILE, (49152, limits[1]))
|
... | ... | @@ -21,6 +21,7 @@ import subprocess |
21 | 21 |
|
22 | 22 |
from .. import _site
|
23 | 23 |
from .. import utils
|
24 |
+import resource
|
|
24 | 25 |
from .._artifactcache.cascache import CASCache
|
25 | 26 |
from .._message import Message, MessageType
|
26 | 27 |
from ..sandbox import SandboxBwrap
|
... | ... | @@ -37,6 +38,7 @@ class Linux(Platform): |
37 | 38 |
self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
|
38 | 39 |
self._user_ns_available = self._check_user_ns_available(context)
|
39 | 40 |
self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
|
41 |
+ self.set_resources_limit()
|
|
40 | 42 |
|
41 | 43 |
@property
|
42 | 44 |
def artifactcache(self):
|
... | ... | @@ -48,10 +50,21 @@ class Linux(Platform): |
48 | 50 |
kwargs['die_with_parent_available'] = self._die_with_parent_available
|
49 | 51 |
return SandboxBwrap(*args, **kwargs)
|
50 | 52 |
|
53 |
+ def set_resources_limit(self):
|
|
54 |
+ # Need to set resources for _frontend/app.py as this is dependent on the platform
|
|
55 |
+ # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
|
|
56 |
+ # Avoid hitting the limit too quickly.
|
|
57 |
+ limits = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
58 |
+ if limits[0] != limits[1]:
|
|
59 |
+ resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
|
|
60 |
+ |
|
51 | 61 |
################################################
|
52 | 62 |
# Private Methods #
|
53 | 63 |
################################################
|
54 | 64 |
def _check_user_ns_available(self, context):
|
65 |
+ import os
|
|
66 |
+ if not os.path.exists('/dev/fuse'):
|
|
67 |
+ return False
|
|
55 | 68 |
|
56 | 69 |
# Here, lets check if bwrap is able to create user namespaces,
|
57 | 70 |
# issue a warning if it's not available, and save the state
|
... | ... | @@ -40,19 +40,22 @@ class Platform(): |
40 | 40 |
|
41 | 41 |
@classmethod
|
42 | 42 |
def create_instance(cls, *args, **kwargs):
|
43 |
- if sys.platform.startswith('linux'):
|
|
44 |
- backend = 'linux'
|
|
45 |
- else:
|
|
46 |
- backend = 'unix'
|
|
47 |
- |
|
48 | 43 |
# Meant for testing purposes and therefore hidden in the
|
49 | 44 |
# deepest corners of the source code. Try not to abuse this,
|
50 | 45 |
# please?
|
51 | 46 |
if os.getenv('BST_FORCE_BACKEND'):
|
52 | 47 |
backend = os.getenv('BST_FORCE_BACKEND')
|
48 |
+ elif sys.platform.startswith('linux'):
|
|
49 |
+ backend = 'linux'
|
|
50 |
+ elif sys.platform.startswith('darwin'):
|
|
51 |
+ backend = 'darwin'
|
|
52 |
+ else:
|
|
53 |
+ backend = 'unix'
|
|
53 | 54 |
|
54 | 55 |
if backend == 'linux':
|
55 | 56 |
from .linux import Linux as PlatformImpl
|
57 |
+ elif backend == 'darwin':
|
|
58 |
+ from .darwin import Darwin as PlatformImpl
|
|
56 | 59 |
elif backend == 'unix':
|
57 | 60 |
from .unix import Unix as PlatformImpl
|
58 | 61 |
else:
|
... | ... | @@ -92,3 +95,7 @@ class Platform(): |
92 | 95 |
def create_sandbox(self, *args, **kwargs):
|
93 | 96 |
raise ImplError("Platform {platform} does not implement create_sandbox()"
|
94 | 97 |
.format(platform=type(self).__name__))
|
98 |
+ |
|
99 |
+ def set_resources_limit(self):
|
|
100 |
+ raise ImplError("Platform {platform} does not implement set_resources_limit()"
|
|
101 |
+ .format(platform=type(self).__name__))
|
... | ... | @@ -18,6 +18,7 @@ |
18 | 18 |
# Tristan Maat <tristan maat codethink co uk>
|
19 | 19 |
|
20 | 20 |
import os
|
21 |
+import resource
|
|
21 | 22 |
|
22 | 23 |
from .._artifactcache.cascache import CASCache
|
23 | 24 |
from .._exceptions import PlatformError
|
... | ... | @@ -32,6 +33,7 @@ class Unix(Platform): |
32 | 33 |
|
33 | 34 |
super().__init__(context)
|
34 | 35 |
self._artifact_cache = CASCache(context)
|
36 |
+ self.set_resources_limit()
|
|
35 | 37 |
|
36 | 38 |
# Not necessarily 100% reliable, but we want to fail early.
|
37 | 39 |
if os.geteuid() != 0:
|
... | ... | @@ -43,3 +45,11 @@ class Unix(Platform): |
43 | 45 |
|
44 | 46 |
def create_sandbox(self, *args, **kwargs):
|
45 | 47 |
return SandboxChroot(*args, **kwargs)
|
48 |
+ |
|
49 |
+ def set_resources_limit(self):
|
|
50 |
+ # Need to set resources for _frontend/app.py as this is dependent on the platform
|
|
51 |
+ # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
|
|
52 |
+ # Avoid hitting the limit too quickly.
|
|
53 |
+ limits = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
54 |
+ if limits[0] != limits[1]:
|
|
55 |
+ resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
|
... | ... | @@ -19,6 +19,7 @@ |
19 | 19 |
# Tiago Gomes <tiago gomes codethink co uk>
|
20 | 20 |
|
21 | 21 |
import os
|
22 |
+import sys
|
|
22 | 23 |
from collections import Mapping, OrderedDict
|
23 | 24 |
from pluginbase import PluginBase
|
24 | 25 |
from . import utils
|
... | ... | @@ -594,7 +595,7 @@ class Project(): |
594 | 595 |
# Based on some testing (mainly on AWS), maximum effective
|
595 | 596 |
# max-jobs value seems to be around 8-10 if we have enough cores
|
596 | 597 |
# users should set values based on workload and build infrastructure
|
597 |
- output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
|
|
598 |
+ output.base_variables['max-jobs'] = str(min(os.cpu_count(), 8))
|
|
598 | 599 |
|
599 | 600 |
# Export options into variables, if that was requested
|
600 | 601 |
output.options.export_variables(output.base_variables)
|
... | ... | @@ -2,6 +2,7 @@ import os |
2 | 2 |
import subprocess
|
3 | 3 |
import pytest
|
4 | 4 |
|
5 |
+from buildstream import utils
|
|
5 | 6 |
from .repo import Repo
|
6 | 7 |
from ..site import HAVE_BZR
|
7 | 8 |
|
... | ... | @@ -20,11 +21,11 @@ class Bzr(Repo): |
20 | 21 |
def create(self, directory):
|
21 | 22 |
branch_dir = os.path.join(self.repo, 'trunk')
|
22 | 23 |
|
23 |
- subprocess.call(['bzr', 'init-repo', self.repo], env=BZR_ENV)
|
|
24 |
- subprocess.call(['bzr', 'init', branch_dir], env=BZR_ENV)
|
|
24 |
+ subprocess.call([utils.get_host_tool('bzr'), 'init-repo', self.repo], env=BZR_ENV)
|
|
25 |
+ subprocess.call([utils.get_host_tool('bzr'), 'init', branch_dir], env=BZR_ENV)
|
|
25 | 26 |
self.copy_directory(directory, branch_dir)
|
26 |
- subprocess.call(['bzr', 'add', '.'], env=BZR_ENV, cwd=branch_dir)
|
|
27 |
- subprocess.call(['bzr', 'commit', '--message="Initial commit"'],
|
|
27 |
+ subprocess.call([utils.get_host_tool('bzr'), 'add', '.'], env=BZR_ENV, cwd=branch_dir)
|
|
28 |
+ subprocess.call([utils.get_host_tool('bzr'), 'commit', '--message="Initial commit"'],
|
|
28 | 29 |
env=BZR_ENV, cwd=branch_dir)
|
29 | 30 |
|
30 | 31 |
return self.latest_commit()
|
... | ... | @@ -42,7 +43,7 @@ class Bzr(Repo): |
42 | 43 |
|
43 | 44 |
def latest_commit(self):
|
44 | 45 |
output = subprocess.check_output([
|
45 |
- 'bzr', 'version-info',
|
|
46 |
+ utils.get_host_tool('bzr'), 'version-info',
|
|
46 | 47 |
'--custom', '--template={revno}',
|
47 | 48 |
os.path.join(self.repo, 'trunk')
|
48 | 49 |
], env=BZR_ENV)
|