[Notes] [Git][BuildStream/buildstream][mac_fixes] 6 commits: Moved `resource.getrlimit()`

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

• daddcd17
  by knownexus at 2018-09-07T11:20:37Z

  Moved `resource.getrlimit()`
  
  Removed `resource.getrlimit()` functionality from app.py
  Added `resource.getrlimit()` functionality to platform.py as function
  Called new function from __init__.py of linux.py and unix.py
  

• 3b03b9e7
  by knownexus at 2018-09-07T11:20:42Z

  Add FUSE check to linux.py
  
  This needed to be implemented as Windows WSL
  Does not currently support FUSE
  

• ae75f299
  by knownexus at 2018-09-07T11:20:42Z

  Moving max-jobs call to Platform
  

• 242ed5e3
  by knownexus at 2018-09-07T11:20:42Z

  Move Cas call to Platform
  

• 54a450e1
  by knownexus at 2018-09-07T11:20:42Z

  Adding darwin.py platform
  
  Adding functionality to recognise Darwin as a platform in plaform.py
  

• 30619cad
  by knownexus at 2018-09-07T11:20:42Z

  Stopped safe_remove attempting to unlink dirs
  
  Previously safe_remove would attempt to unlink a path
  Before attempting to remove it if it was a dir
  
  Now it checks for a dir before that step
  

7 changed files:

• buildstream/_frontend/app.py
• + buildstream/_platform/darwin.py
• buildstream/_platform/linux.py
• buildstream/_platform/platform.py
• buildstream/_platform/unix.py
• buildstream/_project.py
• buildstream/utils.py

Changes:

buildstream/_frontend/app.py

---

...	...	@@ -116,14 +116,6 @@ class App():
116	116	else:
117	117	self.colors = False
118	118
119		- # Increase the soft limit for open file descriptors to the maximum.
120		- # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
121		- # Avoid hitting the limit too quickly.
122		- limits = resource.getrlimit(resource.RLIMIT_NOFILE)
123		- if limits[0] != limits[1]:
124		- # Set soft limit to hard limit
125		- resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
126		-
127	119	# create()
128	120	#
129	121	# Should be used instead of the regular constructor.

buildstream/_platform/darwin.py

---

	1	+#
	2	+# Copyright (C) 2017 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
	4	+#
	5	+# This program is free software; you can redistribute it and/or
	6	+# modify it under the terms of the GNU Lesser General Public
	7	+# License as published by the Free Software Foundation; either
	8	+# version 2 of the License, or (at your option) any later version.
	9	+#
	10	+# This library is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	+# Lesser General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU Lesser General Public
	16	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	17	+
	18	+import os
	19	+import resource
	20	+
	21	+from .._exceptions import PlatformError
	22	+from ..sandbox import SandboxChroot
	23	+
	24	+from . import Platform
	25	+
	26	+
	27	+class Darwin(Platform):
	28	+
	29	+ def __init__(self, context):
	30	+
	31	+ # This value comes from OPEN_MAX in syslimits.h
	32	+ OPEN_MAX = 10240
	33	+ super().__init__(context)
	34	+
	35	+ @property
	36	+ def artifactcache(self):
	37	+ return self._artifact_cache
	38	+
	39	+ def create_sandbox(self, *args, **kwargs):
	40	+ return SandboxChroot(*args, **kwargs)
	41	+
	42	+ def get_cpu_count(self, cap=None):
	43	+ if cap < os.cpu_count():
	44	+ return cap
	45	+ else:
	46	+ return os.cpu_count()
	47	+
	48	+ def set_resources(self, soft_limit=OPEN_MAX, hard_limit=None):
	49	+ super().set_resources(soft_limit)

buildstream/_platform/linux.py

---

...	...	@@ -17,6 +17,7 @@
17	17	# Authors:
18	18	# Tristan Maat <tristan maat codethink co uk>
19	19
	20	+import os
20	21	import subprocess
21	22
22	23	from .. import _site
...	...	@@ -35,7 +36,11 @@ class Linux(Platform):
35	36	super().__init__(context)
36	37
37	38	self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
38		- self._user_ns_available = self._check_user_ns_available(context)
	39	+
	40	+ if self._local_sandbox_available():
	41	+ self._user_ns_available = self._check_user_ns_available(context)
	42	+ else:
	43	+ self._user_ns_available = False
39	44	self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
40	45
41	46	@property
...	...	@@ -51,8 +56,10 @@ class Linux(Platform):
51	56	################################################
52	57	# Private Methods #
53	58	################################################
54		- def _check_user_ns_available(self, context):
	59	+ def _local_sandbox_available(self):
	60	+ return os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')
55	61
	62	+ def _check_user_ns_available(self, context):
56	63	# Here, lets check if bwrap is able to create user namespaces,
57	64	# issue a warning if it's not available, and save the state
58	65	# locally so that we can inform the sandbox to not try it

buildstream/_platform/platform.py

---

...	...	@@ -19,8 +19,11 @@
19	19
20	20	import os
21	21	import sys
	22	+import resource
22	23
23	24	from .._exceptions import PlatformError, ImplError
	25	+from .._artifactcache.cascache import CASCache
	26	+
24	27
25	28
26	29	class Platform():
...	...	@@ -37,22 +40,28 @@ class Platform():
37	40	#
38	41	def __init__(self, context):
39	42	self.context = context
	43	+ self.set_resources()
	44	+ self._artifact_cache = CASCache(context)
40	45
41	46	@classmethod
42	47	def create_instance(cls, *args, **kwargs):
43		- if sys.platform.startswith('linux'):
44		- backend = 'linux'
45		- else:
46		- backend = 'unix'
47	48
48	49	# Meant for testing purposes and therefore hidden in the
49	50	# deepest corners of the source code. Try not to abuse this,
50	51	# please?
51	52	if os.getenv('BST_FORCE_BACKEND'):
52	53	backend = os.getenv('BST_FORCE_BACKEND')
	54	+ elif sys.platform.startswith('linux'):
	55	+ backend = 'linux'
	56	+ elif sys.platform.startswith('darwin'):
	57	+ backend = 'darwin'
	58	+ else:
	59	+ backend = 'unix'
53	60
54	61	if backend == 'linux':
55	62	from .linux import Linux as PlatformImpl
	63	+ elif backend == 'darwin':
	64	+ from .darwin import Darwin as PlatformImpl
56	65	elif backend == 'unix':
57	66	from .unix import Unix as PlatformImpl
58	67	else:
...	...	@@ -66,6 +75,9 @@ class Platform():
66	75	raise PlatformError("Platform needs to be initialized first")
67	76	return cls._instance
68	77
	78	+ def get_cpu_count(self, cap=None):
	79	+ return min(len(os.sched_getaffinity(0)), cap)
	80	+
69	81	##################################################################
70	82	# Platform properties #
71	83	##################################################################
...	...	@@ -92,3 +104,15 @@ class Platform():
92	104	def create_sandbox(self, *args, **kwargs):
93	105	raise ImplError("Platform {platform} does not implement create_sandbox()"
94	106	.format(platform=type(self).__name__))
	107	+
	108	+ def set_resources(self, soft_limit=None, hard_limit=None):
	109	+ # Need to set resources for _frontend/app.py as this is dependent on the platform
	110	+ # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
	111	+ # Avoid hitting the limit too quickly.
	112	+ limits = resource.getrlimit(resource.RLIMIT_NOFILE)
	113	+ if limits[0] != limits[1]:
	114	+ if soft_limit is None:
	115	+ soft_limit = limits[1]
	116	+ if hard_limit is None:
	117	+ hard_limit = limits[1]
	118	+ resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))

buildstream/_platform/unix.py

---

...	...	@@ -31,7 +31,6 @@ class Unix(Platform):
31	31	def __init__(self, context):
32	32
33	33	super().__init__(context)
34		- self._artifact_cache = CASCache(context)
35	34
36	35	# Not necessarily 100% reliable, but we want to fail early.
37	36	if os.geteuid() != 0:

buildstream/_project.py

---

...	...	@@ -38,6 +38,7 @@ from ._loader import Loader
38	38	from .element import Element
39	39	from ._message import Message, MessageType
40	40	from ._includes import Includes
	41	+from ._platform import Platform
41	42
42	43
43	44	# Project Configuration file
...	...	@@ -605,7 +606,8 @@ class Project():
605	606	# Based on some testing (mainly on AWS), maximum effective
606	607	# max-jobs value seems to be around 8-10 if we have enough cores
607	608	# users should set values based on workload and build infrastructure
608		- output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
	609	+ platform = Platform.get_platform()
	610	+ output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
609	611
610	612	# Export options into variables, if that was requested
611	613	output.options.export_variables(output.base_variables)

buildstream/utils.py

---

...	...	@@ -35,6 +35,7 @@ import tempfile
35	35	import itertools
36	36	import functools
37	37	from contextlib import contextmanager
	38	+from stat import S_ISDIR
38	39
39	40	import psutil
40	41
...	...	@@ -328,25 +329,25 @@ def safe_remove(path):
328	329	Raises:
329	330	UtilError: In the case of unexpected system call failures
330	331	"""
331		- if os.path.lexists(path):
	332	+ if not S_ISDIR(os.lstat(path).st_mode):
332	333
333	334	# Try to remove anything that is in the way, but issue
334	335	# a warning instead if it removes a non empty directory
335	336	try:
336	337	os.unlink(path)
	338	+ return True
337	339	except OSError as e:
338		- if e.errno != errno.EISDIR:
339		- raise UtilError("Failed to remove '{}': {}"
340		- .format(path, e))
341		-
342		- try:
343		- os.rmdir(path)
344		- except OSError as e:
345		- if e.errno == errno.ENOTEMPTY:
346		- return False
347		- else:
348		- raise UtilError("Failed to remove '{}': {}"
349		- .format(path, e))
	340	+ raise UtilError("Failed to remove '{}': {}"
	341	+ .format(path, e))
	342	+
	343	+ try:
	344	+ os.rmdir(path)
	345	+ except OSError as e:
	346	+ if e.errno == errno.ENOTEMPTY:
	347	+ return False
	348	+ else:
	349	+ raise UtilError("Failed to remove '{}': {}"
	350	+ .format(path, e))
350	351
351	352	return True
352	353

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.