[Notes] [Git][BuildStream/buildstream][mac_fixes] 6 commits: Replacing string 'bzr' with value from host tools

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

• 680e4fe1
  by knownexus at 2018-08-30T13:37:55Z

  Replacing string 'bzr' with value from host tools
  

• 8cd719eb
  by Phillip Smyth at 2018-08-30T15:47:06Z

  Merge branch 'bzr_fix' into 'master'
  
  Replacing string 'bzr' with value from host tools
  
  See merge request BuildStream/buildstream!764

• d739814f
  by knownexus at 2018-08-31T09:25:01Z

  Moved `resource.getrlimit()`
  
  Removed `resource.getrlimit()` functionality from app.py
  Added `resource.getrlimit()` functionality to linux.py
  Added `resource.getrlimit()` functionality to unix.py
  

• 0524f9f0
  by knownexus at 2018-08-31T09:25:01Z

  Add FUSE check to linux.py
  
  This needed to be implemented as Windows WSL
  Does not currently support FUSE
  

• 60409955
  by knownexus at 2018-08-31T09:25:13Z

  Adding darwin.py platform
  
  Adding functionality to recognise Darwin as a platform in plaform.py
  

• 91081b70
  by knownexus at 2018-08-31T09:29:17Z

  Moving `os.sched_getaffinity(0)` logic
  
  Removed `os.sched_getaffinity(0)` from _project.py
  Added `os.sched_getaffinity(0)` to platform.py
  Added `os.cpu_count()` to darwin.py
  

7 changed files:

• buildstream/_frontend/app.py
• + buildstream/_platform/darwin.py
• buildstream/_platform/linux.py
• buildstream/_platform/platform.py
• buildstream/_platform/unix.py
• buildstream/_project.py
• tests/testutils/repo/bzr.py

Changes:

buildstream/_frontend/app.py

---

...	...	@@ -116,14 +116,6 @@ class App():
116	116	else:
117	117	self.colors = False
118	118
119		- # Increase the soft limit for open file descriptors to the maximum.
120		- # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
121		- # Avoid hitting the limit too quickly.
122		- limits = resource.getrlimit(resource.RLIMIT_NOFILE)
123		- if limits[0] != limits[1]:
124		- # Set soft limit to hard limit
125		- resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
126		-
127	119	# create()
128	120	#
129	121	# Should be used instead of the regular constructor.

buildstream/_platform/darwin.py

---

	1	+#
	2	+# Copyright (C) 2017 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
	4	+#
	5	+# This program is free software; you can redistribute it and/or
	6	+# modify it under the terms of the GNU Lesser General Public
	7	+# License as published by the Free Software Foundation; either
	8	+# version 2 of the License, or (at your option) any later version.
	9	+#
	10	+# This library is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	+# Lesser General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU Lesser General Public
	16	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	17	+
	18	+import os
	19	+import resource
	20	+
	21	+from .._artifactcache.cascache import CASCache
	22	+from .._exceptions import PlatformError
	23	+from ..sandbox import SandboxChroot
	24	+
	25	+from . import Platform
	26	+
	27	+
	28	+class Darwin(Platform):
	29	+
	30	+ def __init__(self, context):
	31	+
	32	+ super().__init__(context)
	33	+
	34	+ # Need to set resources for _frontend/app.py as this is dependent on the platform
	35	+ # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
	36	+ # Avoid hitting the limit too quickly.
	37	+ limits = resource.getrlimit(resource.RLIMIT_NOFILE)
	38	+ if limits[0] != limits[1]:
	39	+ resource.setrlimit(resource.RLIMIT_NOFILE, (49152, limits[1]))
	40	+
	41	+ @property
	42	+ def artifactcache(self):
	43	+ return self._artifact_cache
	44	+
	45	+ def create_sandbox(self, *args, **kwargs):
	46	+ return SandboxChroot(*args, **kwargs)
	47	+
	48	+ def get_cpu_count(self):
	49	+ return str(os.cpu_count())

buildstream/_platform/linux.py

---

...	...	@@ -17,10 +17,12 @@
17	17	# Authors:
18	18	# Tristan Maat <tristan maat codethink co uk>
19	19
	20	+import os
20	21	import subprocess
21	22
22	23	from .. import _site
23	24	from .. import utils
	25	+import resource
24	26	from .._artifactcache.cascache import CASCache
25	27	from .._message import Message, MessageType
26	28	from ..sandbox import SandboxBwrap
...	...	@@ -35,9 +37,18 @@ class Linux(Platform):
35	37	super().__init__(context)
36	38
37	39	self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
38		- self._user_ns_available = self._check_user_ns_available(context)
	40	+
	41	+ self._user_ns_available = self._check_fuse_available() and self._check_user_ns_available(context)
	42	+
39	43	self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
40	44
	45	+ # Need to set resources for _frontend/app.py as this is dependent on the platform
	46	+ # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
	47	+ # Avoid hitting the limit too quickly.
	48	+ limits = resource.getrlimit(resource.RLIMIT_NOFILE)
	49	+ if limits[0] != limits[1]:
	50	+ resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
	51	+
41	52	@property
42	53	def artifactcache(self):
43	54	return self._artifact_cache
...	...	@@ -51,6 +62,9 @@ class Linux(Platform):
51	62	################################################
52	63	# Private Methods #
53	64	################################################
	65	+ def _check_fuse_available(self):
	66	+ return os.path.exists('/dev/fuse')
	67	+
54	68	def _check_user_ns_available(self, context):
55	69
56	70	# Here, lets check if bwrap is able to create user namespaces,

buildstream/_platform/platform.py

---

...	...	@@ -37,22 +37,27 @@ class Platform():
37	37	#
38	38	def __init__(self, context):
39	39	self.context = context
	40	+ self._artifact_cache = CASCache(context)
40	41
41	42	@classmethod
42	43	def create_instance(cls, *args, **kwargs):
43		- if sys.platform.startswith('linux'):
44		- backend = 'linux'
45		- else:
46		- backend = 'unix'
47	44
48	45	# Meant for testing purposes and therefore hidden in the
49	46	# deepest corners of the source code. Try not to abuse this,
50	47	# please?
51	48	if os.getenv('BST_FORCE_BACKEND'):
52	49	backend = os.getenv('BST_FORCE_BACKEND')
	50	+ elif sys.platform.startswith('linux'):
	51	+ backend = 'linux'
	52	+ elif sys.platform.startswith('darwin'):
	53	+ backend = 'darwin'
	54	+ else:
	55	+ backend = 'unix'
53	56
54	57	if backend == 'linux':
55	58	from .linux import Linux as PlatformImpl
	59	+ elif backend == 'darwin':
	60	+ from .darwin import Darwin as PlatformImpl
56	61	elif backend == 'unix':
57	62	from .unix import Unix as PlatformImpl
58	63	else:
...	...	@@ -92,3 +97,9 @@ class Platform():
92	97	def create_sandbox(self, *args, **kwargs):
93	98	raise ImplError("Platform {platform} does not implement create_sandbox()"
94	99	.format(platform=type(self).__name__))
	100	+
	101	+ def get_cpu_count(self, cap=None):
	102	+ if cap is None:
	103	+ return len(os.sched_getaffinity(0))
	104	+ else:
	105	+ return min(len(os.sched_getaffinity(0)), cap)

buildstream/_platform/unix.py

---

...	...	@@ -18,6 +18,7 @@
18	18	# Tristan Maat <tristan maat codethink co uk>
19	19
20	20	import os
	21	+import resource
21	22
22	23	from .._artifactcache.cascache import CASCache
23	24	from .._exceptions import PlatformError
...	...	@@ -31,7 +32,13 @@ class Unix(Platform):
31	32	def __init__(self, context):
32	33
33	34	super().__init__(context)
34		- self._artifact_cache = CASCache(context)
	35	+
	36	+ # Need to set resources for _frontend/app.py as this is dependent on the platform
	37	+ # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
	38	+ # Avoid hitting the limit too quickly.
	39	+ limits = resource.getrlimit(resource.RLIMIT_NOFILE)
	40	+ if limits[0] != limits[1]:
	41	+ resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
35	42
36	43	# Not necessarily 100% reliable, but we want to fail early.
37	44	if os.geteuid() != 0:

buildstream/_project.py

---

...	...	@@ -19,6 +19,7 @@
19	19	# Tiago Gomes <tiago gomes codethink co uk>
20	20
21	21	import os
	22	+import sys
22	23	from collections import Mapping, OrderedDict
23	24	from pluginbase import PluginBase
24	25	from . import utils
...	...	@@ -38,6 +39,7 @@ from ._loader import Loader
38	39	from .element import Element
39	40	from ._message import Message, MessageType
40	41	from ._includes import Includes
	42	+from ._platform import Platform
41	43
42	44
43	45	# Project Configuration file
...	...	@@ -594,7 +596,8 @@ class Project():
594	596	# Based on some testing (mainly on AWS), maximum effective
595	597	# max-jobs value seems to be around 8-10 if we have enough cores
596	598	# users should set values based on workload and build infrastructure
597		- output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
	599	+ platform = Platform.get_platform()
	600	+ output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
598	601
599	602	# Export options into variables, if that was requested
600	603	output.options.export_variables(output.base_variables)

tests/testutils/repo/bzr.py

---

...	...	@@ -2,6 +2,7 @@ import os
2	2	import subprocess
3	3	import pytest
4	4
	5	+from buildstream import utils
5	6	from .repo import Repo
6	7	from ..site import HAVE_BZR
7	8
...	...	@@ -16,15 +17,16 @@ class Bzr(Repo):
16	17	if not HAVE_BZR:
17	18	pytest.skip("bzr is not available")
18	19	super(Bzr, self).__init__(directory, subdir)
	20	+ self.bzr = utils.get_host_tool('bzr')
19	21
20	22	def create(self, directory):
21	23	branch_dir = os.path.join(self.repo, 'trunk')
22	24
23		- subprocess.call(['bzr', 'init-repo', self.repo], env=BZR_ENV)
24		- subprocess.call(['bzr', 'init', branch_dir], env=BZR_ENV)
	25	+ subprocess.call([self.bzr, 'init-repo', self.repo], env=BZR_ENV)
	26	+ subprocess.call([self.bzr, 'init', branch_dir], env=BZR_ENV)
25	27	self.copy_directory(directory, branch_dir)
26		- subprocess.call(['bzr', 'add', '.'], env=BZR_ENV, cwd=branch_dir)
27		- subprocess.call(['bzr', 'commit', '--message="Initial commit"'],
	28	+ subprocess.call([self.bzr, 'add', '.'], env=BZR_ENV, cwd=branch_dir)
	29	+ subprocess.call([self.bzr, 'commit', '--message="Initial commit"'],
28	30	env=BZR_ENV, cwd=branch_dir)
29	31
30	32	return self.latest_commit()
...	...	@@ -42,7 +44,7 @@ class Bzr(Repo):
42	44
43	45	def latest_commit(self):
44	46	output = subprocess.check_output([
45		- 'bzr', 'version-info',
	47	+ self.bzr, 'version-info',
46	48	'--custom', '--template={revno}',
47	49	os.path.join(self.repo, 'trunk')
48	50	], env=BZR_ENV)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.