[Notes] [Git][BuildStream/buildstream][mac_fixes] 8 commits: Added FUSE check, as WSL does not support FUSE yet

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

• bb17975b
  by knownexus at 2018-09-25T15:12:13Z

  Added FUSE check, as WSL does not support FUSE yet
  
  _platform/linux.py: Add FUSE check
  And Ensure exception is not raised when performing sandbox availability check
  Without this check, WSL would error upon finding FUSE missing
  This must be bypassed to allow remote execution
  

• 454b1aaf
  by knownexus at 2018-09-25T15:12:54Z

  Max-jobs is now defined on a platform basis
  
  This is due to Darwin (MacOS) having a different Max-Jobs limit
  _platform/platform.py: Adding max-jobs call
  
  _project.py: redirected 'max-jobs' to point at the platform
  

• 49020b32
  by knownexus at 2018-09-25T15:12:54Z

  Artifact caches are now defined in platform.py
  
  This was done so a default exists, but allows platforms to override as needed
  
  _platform/platform.py: Added CAS call function
  
  _platform/linux.py: Added override to CAS call
  _platform/unix.py: Remove CAS call
  

• e20f71c5
  by knownexus at 2018-09-25T15:12:54Z

  Adding Dummy sandbox
  
  This is to allow platforms that do not support sandboxing
  To error is a controlled/known way
  

• 085c966f
  by knownexus at 2018-09-25T15:12:54Z

  Adding darwin.py (MacOS) platform
  
  Adding functionality to recognise Darwin as a platform in plaform.py
  

• f49f9aab
  by knownexus at 2018-09-25T15:12:54Z

  utils.py: reworked safe_remove
  
  non-Linux platforms don't return EISDIR when attempting to unlink a directory
  
  Stopped safe_remove attempting to unlink dirs
  Previously safe_remove would attempt to unlink a path
  Before attempting to remove it if it was a dir
  
  Now it checks for a dir before that step
  

• 9c4d1f5f
  by James Ennis at 2018-09-25T15:12:54Z

  Sandbox: Ensure that we only import the sandbox subclasses when instantiate a sandbox
  
    - This patch was required due to not being able to execute `bst` on
    a Linux machine without Fuse or Bubblewrap available
  

• 002cc94d
  by Phillip Smyth at 2018-09-25T15:12:54Z

  Fixed workspace list error message
  
  If no workspaces were found, bst workspace would return `[]`
  This has been changed to a nicer message
  

10 changed files:

• + buildstream/_platform/darwin.py
• buildstream/_platform/linux.py
• buildstream/_platform/platform.py
• buildstream/_platform/unix.py
• buildstream/_project.py
• buildstream/_stream.py
• buildstream/sandbox/__init__.py
• + buildstream/sandbox/_sandboxdummy.py
• buildstream/utils.py
• tests/frontend/cross_junction_workspace.py

Changes:

buildstream/_platform/darwin.py

---

	1	+#
	2	+# Copyright (C) 2017 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
	4	+#
	5	+# This program is free software; you can redistribute it and/or
	6	+# modify it under the terms of the GNU Lesser General Public
	7	+# License as published by the Free Software Foundation; either
	8	+# version 2 of the License, or (at your option) any later version.
	9	+#
	10	+# This library is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	+# Lesser General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU Lesser General Public
	16	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	17	+
	18	+import os
	19	+import resource
	20	+
	21	+from .._exceptions import PlatformError
	22	+from ..sandbox import SandboxDummy
	23	+
	24	+from . import Platform
	25	+
	26	+
	27	+class Darwin(Platform):
	28	+
	29	+ # This value comes from OPEN_MAX in syslimits.h
	30	+ OPEN_MAX = 10240
	31	+
	32	+ def __init__(self, context):
	33	+
	34	+ super().__init__(context)
	35	+
	36	+ @property
	37	+ def artifactcache(self):
	38	+ return self._artifact_cache
	39	+
	40	+ def create_sandbox(self, *args, **kwargs):
	41	+ return SandboxDummy(*args, **kwargs)
	42	+
	43	+ def get_cpu_count(self, cap=None):
	44	+ if cap < os.cpu_count():
	45	+ return cap
	46	+ else:
	47	+ return os.cpu_count()
	48	+
	49	+ def set_resource_limits(self, soft_limit=OPEN_MAX, hard_limit=None):
	50	+ super().set_resource_limits(soft_limit)

buildstream/_platform/linux.py

---

...	...	@@ -17,13 +17,14 @@
17	17	# Authors:
18	18	# Tristan Maat <tristan maat codethink co uk>
19	19
	20	+import os
20	21	import subprocess
21	22
22	23	from .. import _site
23	24	from .. import utils
24	25	from .._artifactcache.cascache import CASCache
25	26	from .._message import Message, MessageType
26		-from ..sandbox import SandboxBwrap
	27	+from ..sandbox import SandboxDummy
27	28
28	29	from . import Platform
29	30
...	...	@@ -32,25 +33,43 @@ class Linux(Platform):
32	33
33	34	def __init__(self, context):
34	35
35		- super().__init__(context)
36		-
37	36	self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
38		- self._user_ns_available = self._check_user_ns_available(context)
39		- self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
	37	+
	38	+ if self._local_sandbox_available():
	39	+ self._user_ns_available = self._check_user_ns_available(context)
	40	+ else:
	41	+ self._user_ns_available = False
	42	+
	43	+ # _user_ns_available needs to be set before chaining up to the super class
	44	+ # This is because it will call create_artifact_cache()
	45	+ super().__init__(context)
40	46
41	47	@property
42	48	def artifactcache(self):
43	49	return self._artifact_cache
44	50
45	51	def create_sandbox(self, *args, **kwargs):
46		- # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
47		- kwargs['user_ns_available'] = self._user_ns_available
48		- kwargs['die_with_parent_available'] = self._die_with_parent_available
49		- return SandboxBwrap(*args, **kwargs)
	52	+ if not self._local_sandbox_available():
	53	+ return SandboxDummy(*args, **kwargs)
	54	+ else:
	55	+ from ..sandbox._sandboxbwrap import SandboxBwrap
	56	+ # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
	57	+ kwargs['user_ns_available'] = self._user_ns_available
	58	+ kwargs['die_with_parent_available'] = self._die_with_parent_available
	59	+ return SandboxBwrap(*args, **kwargs)
	60	+
	61	+ def create_artifact_cache(self, context, *, enable_push):
	62	+ return super().create_artifact_cache(context=context, enable_push=self._user_ns_available)
50	63
51	64	################################################
52	65	# Private Methods #
53	66	################################################
	67	+ def _local_sandbox_available(self):
	68	+ try:
	69	+ return os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')
	70	+ except utils.ProgramNotFoundError:
	71	+ return False
	72	+
54	73	def _check_user_ns_available(self, context):
55	74	# Here, lets check if bwrap is able to create user namespaces,
56	75	# issue a warning if it's not available, and save the state

buildstream/_platform/platform.py

---

...	...	@@ -22,6 +22,7 @@ import sys
22	22	import resource
23	23
24	24	from .._exceptions import PlatformError, ImplError
	25	+from .._artifactcache.cascache import CASCache
25	26
26	27
27	28	class Platform():
...	...	@@ -39,22 +40,27 @@ class Platform():
39	40	def __init__(self, context):
40	41	self.context = context
41	42	self.set_resource_limits()
	43	+ self._artifact_cache = self.create_artifact_cache(context, enable_push=True)
42	44
43	45	@classmethod
44	46	def create_instance(cls, *args, **kwargs):
45		- if sys.platform.startswith('linux'):
46		- backend = 'linux'
47		- else:
48		- backend = 'unix'
49	47
50	48	# Meant for testing purposes and therefore hidden in the
51	49	# deepest corners of the source code. Try not to abuse this,
52	50	# please?
53	51	if os.getenv('BST_FORCE_BACKEND'):
54	52	backend = os.getenv('BST_FORCE_BACKEND')
	53	+ elif sys.platform.startswith('linux'):
	54	+ backend = 'linux'
	55	+ elif sys.platform.startswith('darwin'):
	56	+ backend = 'darwin'
	57	+ else:
	58	+ backend = 'unix'
55	59
56	60	if backend == 'linux':
57	61	from .linux import Linux as PlatformImpl
	62	+ elif backend == 'darwin':
	63	+ from .darwin import Darwin as PlatformImpl
58	64	elif backend == 'unix':
59	65	from .unix import Unix as PlatformImpl
60	66	else:
...	...	@@ -68,6 +74,9 @@ class Platform():
68	74	raise PlatformError("Platform needs to be initialized first")
69	75	return cls._instance
70	76
	77	+ def get_cpu_count(self, cap=None):
	78	+ return min(len(os.sched_getaffinity(0)), cap)
	79	+
71	80	##################################################################
72	81	# Platform properties #
73	82	##################################################################
...	...	@@ -106,3 +115,6 @@ class Platform():
106	115	if hard_limit is None:
107	116	hard_limit = limits[1]
108	117	resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
	118	+
	119	+ def create_artifact_cache(self, context, *, enable_push=True):
	120	+ return CASCache(context=context, enable_push=enable_push)

buildstream/_platform/unix.py

---

...	...	@@ -21,7 +21,6 @@ import os
21	21
22	22	from .._artifactcache.cascache import CASCache
23	23	from .._exceptions import PlatformError
24		-from ..sandbox import SandboxChroot
25	24
26	25	from . import Platform
27	26
...	...	@@ -31,7 +30,6 @@ class Unix(Platform):
31	30	def __init__(self, context):
32	31
33	32	super().__init__(context)
34		- self._artifact_cache = CASCache(context)
35	33
36	34	# Not necessarily 100% reliable, but we want to fail early.
37	35	if os.geteuid() != 0:
...	...	@@ -42,4 +40,5 @@ class Unix(Platform):
42	40	return self._artifact_cache
43	41
44	42	def create_sandbox(self, *args, **kwargs):
	43	+ from ..sandbox._sandboxchroot import SandboxChroot
45	44	return SandboxChroot(*args, **kwargs)

buildstream/_project.py

---

...	...	@@ -38,6 +38,7 @@ from ._loader import Loader
38	38	from .element import Element
39	39	from ._message import Message, MessageType
40	40	from ._includes import Includes
	41	+from ._platform import Platform
41	42
42	43
43	44	# Project Configuration file
...	...	@@ -617,7 +618,8 @@ class Project():
617	618	# Based on some testing (mainly on AWS), maximum effective
618	619	# max-jobs value seems to be around 8-10 if we have enough cores
619	620	# users should set values based on workload and build infrastructure
620		- output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
	621	+ platform = Platform.get_platform()
	622	+ output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
621	623
622	624	# Export options into variables, if that was requested
623	625	output.options.export_variables(output.base_variables)

buildstream/_stream.py

---

...	...	@@ -641,6 +641,9 @@ class Stream():
641	641	}
642	642	workspaces.append(workspace_detail)
643	643
	644	+ if not workspaces:
	645	+ workspaces = "No workspaces found"
	646	+
644	647	_yaml.dump({
645	648	'workspaces': workspaces
646	649	})

buildstream/sandbox/__init__.py

---

...	...	@@ -18,6 +18,5 @@
18	18	# Tristan Maat <tristan maat codethink co uk>
19	19
20	20	from .sandbox import Sandbox, SandboxFlags
21		-from ._sandboxchroot import SandboxChroot
22		-from ._sandboxbwrap import SandboxBwrap
23	21	from ._sandboxremote import SandboxRemote
	22	+from ._sandboxdummy import SandboxDummy

buildstream/sandbox/_sandboxdummy.py

---

	1	+#
	2	+# Copyright (C) 2017 Codethink Limited
	3	+#
	4	+# This program is free software; you can redistribute it and/or
	5	+# modify it under the terms of the GNU Lesser General Public
	6	+# License as published by the Free Software Foundation; either
	7	+# version 2 of the License, or (at your option) any later version.
	8	+#
	9	+# This library is distributed in the hope that it will be useful,
	10	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	+# Lesser General Public License for more details.
	13	+#
	14	+# You should have received a copy of the GNU Lesser General Public
	15	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	16	+#
	17	+# Authors:
	18	+
	19	+from .._exceptions import SandboxError
	20	+from . import Sandbox
	21	+
	22	+
	23	+class SandboxDummy(Sandbox):
	24	+ def __init__(self, *args, **kwargs):
	25	+ super().__init__(*args, **kwargs)
	26	+
	27	+ def run(self, command, flags, *, cwd=None, env=None):
	28	+
	29	+ # Fallback to the sandbox default settings for
	30	+ # the cwd and env.
	31	+ #
	32	+ cwd = self._get_work_directory(cwd=cwd)
	33	+ env = self._get_environment(cwd=cwd, env=env)
	34	+
	35	+ if not self._has_command(command[0], env):
	36	+ raise SandboxError("Staged artifacts do not provide command "
	37	+ "'{}'".format(command[0]),
	38	+ reason='missing-command')
	39	+
	40	+ raise SandboxError("This platform does not support local builds")

buildstream/utils.py

---

...	...	@@ -35,6 +35,7 @@ import tempfile
35	35	import itertools
36	36	import functools
37	37	from contextlib import contextmanager
	38	+from stat import S_ISDIR
38	39
39	40	import psutil
40	41
...	...	@@ -328,27 +329,25 @@ def safe_remove(path):
328	329	Raises:
329	330	UtilError: In the case of unexpected system call failures
330	331	"""
331		- if os.path.lexists(path):
332		-
333		- # Try to remove anything that is in the way, but issue
334		- # a warning instead if it removes a non empty directory
335		- try:
	332	+ try:
	333	+ if S_ISDIR(os.lstat(path).st_mode):
	334	+ os.rmdir(path)
	335	+ else:
336	336	os.unlink(path)
337		- except OSError as e:
338		- if e.errno != errno.EISDIR:
339		- raise UtilError("Failed to remove '{}': {}"
340		- .format(path, e))
341		-
342		- try:
343		- os.rmdir(path)
344		- except OSError as e:
345		- if e.errno == errno.ENOTEMPTY:
346		- return False
347		- else:
348		- raise UtilError("Failed to remove '{}': {}"
349		- .format(path, e))
350	337
351		- return True
	338	+ # File removed/unlinked successfully
	339	+ return True
	340	+
	341	+ except OSError as e:
	342	+ if e.errno == errno.ENOTEMPTY:
	343	+ # Path is non-empty directory
	344	+ return False
	345	+ elif e.errno == errno.ENOENT:
	346	+ # Path does not exist
	347	+ return True
	348	+
	349	+ raise UtilError("Failed to remove '{}': {}"
	350	+ .format(path, e))
352	351
353	352
354	353	def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=False):

tests/frontend/cross_junction_workspace.py

---

...	...	@@ -93,9 +93,10 @@ def test_close_cross_junction(cli, tmpdir):
93	93	result.assert_success()
94	94
95	95	loaded = _yaml.load_data(result.output)
96		- assert isinstance(loaded.get('workspaces'), list)
97		- workspaces = loaded['workspaces']
98		- assert len(workspaces) == 0
	96	+ if not loaded['workspaces'] == "No workspaces found":
	97	+ assert isinstance(loaded.get('workspaces'), list)
	98	+ workspaces = loaded['workspaces']
	99	+ assert len(workspaces) == 0
99	100
100	101
101	102	def test_close_all_cross_junction(cli, tmpdir):
...	...	@@ -112,6 +113,7 @@ def test_close_all_cross_junction(cli, tmpdir):
112	113	result.assert_success()
113	114
114	115	loaded = _yaml.load_data(result.output)
115		- assert isinstance(loaded.get('workspaces'), list)
116		- workspaces = loaded['workspaces']
117		- assert len(workspaces) == 0
	116	+ if not loaded['workspaces'] == "No workspaces found":
	117	+ assert isinstance(loaded.get('workspaces'), list)
	118	+ workspaces = loaded['workspaces']
	119	+ assert len(workspaces) == 0

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.