[Notes] [Git][BuildStream/buildstream][mac_fixes] 5 commits: Artifact caches are now defined in platform.py

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch mac_fixes at BuildStream / buildstream

Commits:

• 2aa83b52
  by knownexus at 2018-09-25T15:37:51Z

  Artifact caches are now defined in platform.py
  
  This was done so a default exists, but allows platforms to override as needed
  
  _platform/platform.py: Added CAS call function
  
  _platform/linux.py: Added override to CAS call
  _platform/unix.py: Remove CAS call
  

• 5850e128
  by knownexus at 2018-09-25T15:37:55Z

  Adding Dummy sandbox
  
  This is to allow platforms that do not support sandboxing
  To error is a controlled/known way
  

• 3acb6476
  by knownexus at 2018-09-25T15:37:55Z

  Adding darwin.py (MacOS) platform
  
  Adding functionality to recognise Darwin as a platform in plaform.py
  

• 0902cc90
  by knownexus at 2018-09-25T15:37:55Z

  utils.py: reworked safe_remove
  
  non-Linux platforms don't return EISDIR when attempting to unlink a directory
  
  Stopped safe_remove attempting to unlink dirs
  Previously safe_remove would attempt to unlink a path
  Before attempting to remove it if it was a dir
  
  Now it checks for a dir before that step
  

• 18dc5af3
  by James Ennis at 2018-09-25T15:37:55Z

  Sandbox: Ensure that we only import the sandbox subclasses when instantiate a sandbox
  
    - This patch was required due to not being able to execute `bst` on
    a Linux machine without Fuse or Bubblewrap available
  

7 changed files:

• + buildstream/_platform/darwin.py
• buildstream/_platform/linux.py
• buildstream/_platform/platform.py
• buildstream/_platform/unix.py
• buildstream/sandbox/__init__.py
• + buildstream/sandbox/_sandboxdummy.py
• buildstream/utils.py

Changes:

buildstream/_platform/darwin.py

---

	1	+#
	2	+# Copyright (C) 2017 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
	4	+#
	5	+# This program is free software; you can redistribute it and/or
	6	+# modify it under the terms of the GNU Lesser General Public
	7	+# License as published by the Free Software Foundation; either
	8	+# version 2 of the License, or (at your option) any later version.
	9	+#
	10	+# This library is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	+# Lesser General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU Lesser General Public
	16	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	17	+
	18	+import os
	19	+import resource
	20	+
	21	+from .._exceptions import PlatformError
	22	+from ..sandbox import SandboxDummy
	23	+
	24	+from . import Platform
	25	+
	26	+
	27	+class Darwin(Platform):
	28	+
	29	+ # This value comes from OPEN_MAX in syslimits.h
	30	+ OPEN_MAX = 10240
	31	+
	32	+ def __init__(self, context):
	33	+
	34	+ super().__init__(context)
	35	+
	36	+ @property
	37	+ def artifactcache(self):
	38	+ return self._artifact_cache
	39	+
	40	+ def create_sandbox(self, *args, **kwargs):
	41	+ return SandboxDummy(*args, **kwargs)
	42	+
	43	+ def get_cpu_count(self, cap=None):
	44	+ if cap < os.cpu_count():
	45	+ return cap
	46	+ else:
	47	+ return os.cpu_count()
	48	+
	49	+ def set_resource_limits(self, soft_limit=OPEN_MAX, hard_limit=None):
	50	+ super().set_resource_limits(soft_limit)

buildstream/_platform/linux.py

---

...	...	@@ -22,9 +22,8 @@ import subprocess
22	22
23	23	from .. import _site
24	24	from .. import utils
25		-from .._artifactcache.cascache import CASCache
26	25	from .._message import Message, MessageType
27		-from ..sandbox import SandboxBwrap
	26	+from ..sandbox import SandboxDummy
28	27
29	28	from . import Platform
30	29
...	...	@@ -33,25 +32,33 @@ class Linux(Platform):
33	32
34	33	def __init__(self, context):
35	34
36		- super().__init__(context)
37		-
38	35	self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
39	36
40	37	if self._local_sandbox_available():
41	38	self._user_ns_available = self._check_user_ns_available(context)
42	39	else:
43	40	self._user_ns_available = False
44		- self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
	41	+
	42	+ # _user_ns_available needs to be set before chaining up to the super class
	43	+ # This is because it will call create_artifact_cache()
	44	+ super().__init__(context)
45	45
46	46	@property
47	47	def artifactcache(self):
48	48	return self._artifact_cache
49	49
50	50	def create_sandbox(self, *args, **kwargs):
51		- # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
52		- kwargs['user_ns_available'] = self._user_ns_available
53		- kwargs['die_with_parent_available'] = self._die_with_parent_available
54		- return SandboxBwrap(*args, **kwargs)
	51	+ if not self._local_sandbox_available():
	52	+ return SandboxDummy(*args, **kwargs)
	53	+ else:
	54	+ from ..sandbox._sandboxbwrap import SandboxBwrap
	55	+ # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
	56	+ kwargs['user_ns_available'] = self._user_ns_available
	57	+ kwargs['die_with_parent_available'] = self._die_with_parent_available
	58	+ return SandboxBwrap(*args, **kwargs)
	59	+
	60	+ def create_artifact_cache(self, context, *, enable_push):
	61	+ return super().create_artifact_cache(context=context, enable_push=self._user_ns_available)
55	62
56	63	################################################
57	64	# Private Methods #

buildstream/_platform/platform.py

---

...	...	@@ -22,6 +22,7 @@ import sys
22	22	import resource
23	23
24	24	from .._exceptions import PlatformError, ImplError
	25	+from .._artifactcache.cascache import CASCache
25	26
26	27
27	28	class Platform():
...	...	@@ -39,22 +40,27 @@ class Platform():
39	40	def __init__(self, context):
40	41	self.context = context
41	42	self.set_resource_limits()
	43	+ self._artifact_cache = self.create_artifact_cache(context, enable_push=True)
42	44
43	45	@classmethod
44	46	def create_instance(cls, *args, **kwargs):
45		- if sys.platform.startswith('linux'):
46		- backend = 'linux'
47		- else:
48		- backend = 'unix'
49	47
50	48	# Meant for testing purposes and therefore hidden in the
51	49	# deepest corners of the source code. Try not to abuse this,
52	50	# please?
53	51	if os.getenv('BST_FORCE_BACKEND'):
54	52	backend = os.getenv('BST_FORCE_BACKEND')
	53	+ elif sys.platform.startswith('linux'):
	54	+ backend = 'linux'
	55	+ elif sys.platform.startswith('darwin'):
	56	+ backend = 'darwin'
	57	+ else:
	58	+ backend = 'unix'
55	59
56	60	if backend == 'linux':
57	61	from .linux import Linux as PlatformImpl
	62	+ elif backend == 'darwin':
	63	+ from .darwin import Darwin as PlatformImpl
58	64	elif backend == 'unix':
59	65	from .unix import Unix as PlatformImpl
60	66	else:
...	...	@@ -109,3 +115,6 @@ class Platform():
109	115	if hard_limit is None:
110	116	hard_limit = limits[1]
111	117	resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
	118	+
	119	+ def create_artifact_cache(self, context, *, enable_push=True):
	120	+ return CASCache(context=context, enable_push=enable_push)

buildstream/_platform/unix.py

---

...	...	@@ -19,9 +19,7 @@
19	19
20	20	import os
21	21
22		-from .._artifactcache.cascache import CASCache
23	22	from .._exceptions import PlatformError
24		-from ..sandbox import SandboxChroot
25	23
26	24	from . import Platform
27	25
...	...	@@ -31,7 +29,6 @@ class Unix(Platform):
31	29	def __init__(self, context):
32	30
33	31	super().__init__(context)
34		- self._artifact_cache = CASCache(context)
35	32
36	33	# Not necessarily 100% reliable, but we want to fail early.
37	34	if os.geteuid() != 0:
...	...	@@ -42,4 +39,5 @@ class Unix(Platform):
42	39	return self._artifact_cache
43	40
44	41	def create_sandbox(self, *args, **kwargs):
	42	+ from ..sandbox._sandboxchroot import SandboxChroot
45	43	return SandboxChroot(*args, **kwargs)

buildstream/sandbox/__init__.py

---

...	...	@@ -18,6 +18,5 @@
18	18	# Tristan Maat <tristan maat codethink co uk>
19	19
20	20	from .sandbox import Sandbox, SandboxFlags
21		-from ._sandboxchroot import SandboxChroot
22		-from ._sandboxbwrap import SandboxBwrap
23	21	from ._sandboxremote import SandboxRemote
	22	+from ._sandboxdummy import SandboxDummy

buildstream/sandbox/_sandboxdummy.py

---

	1	+#
	2	+# Copyright (C) 2017 Codethink Limited
	3	+#
	4	+# This program is free software; you can redistribute it and/or
	5	+# modify it under the terms of the GNU Lesser General Public
	6	+# License as published by the Free Software Foundation; either
	7	+# version 2 of the License, or (at your option) any later version.
	8	+#
	9	+# This library is distributed in the hope that it will be useful,
	10	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	+# Lesser General Public License for more details.
	13	+#
	14	+# You should have received a copy of the GNU Lesser General Public
	15	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	16	+#
	17	+# Authors:
	18	+
	19	+from .._exceptions import SandboxError
	20	+from . import Sandbox
	21	+
	22	+
	23	+class SandboxDummy(Sandbox):
	24	+ def __init__(self, *args, **kwargs):
	25	+ super().__init__(*args, **kwargs)
	26	+
	27	+ def run(self, command, flags, *, cwd=None, env=None):
	28	+
	29	+ # Fallback to the sandbox default settings for
	30	+ # the cwd and env.
	31	+ #
	32	+ cwd = self._get_work_directory(cwd=cwd)
	33	+ env = self._get_environment(cwd=cwd, env=env)
	34	+
	35	+ if not self._has_command(command[0], env):
	36	+ raise SandboxError("Staged artifacts do not provide command "
	37	+ "'{}'".format(command[0]),
	38	+ reason='missing-command')
	39	+
	40	+ raise SandboxError("This platform does not support local builds")

buildstream/utils.py

---

...	...	@@ -35,6 +35,7 @@ import tempfile
35	35	import itertools
36	36	import functools
37	37	from contextlib import contextmanager
	38	+from stat import S_ISDIR
38	39
39	40	import psutil
40	41
...	...	@@ -328,27 +329,25 @@ def safe_remove(path):
328	329	Raises:
329	330	UtilError: In the case of unexpected system call failures
330	331	"""
331		- if os.path.lexists(path):
332		-
333		- # Try to remove anything that is in the way, but issue
334		- # a warning instead if it removes a non empty directory
335		- try:
	332	+ try:
	333	+ if S_ISDIR(os.lstat(path).st_mode):
	334	+ os.rmdir(path)
	335	+ else:
336	336	os.unlink(path)
337		- except OSError as e:
338		- if e.errno != errno.EISDIR:
339		- raise UtilError("Failed to remove '{}': {}"
340		- .format(path, e))
341		-
342		- try:
343		- os.rmdir(path)
344		- except OSError as e:
345		- if e.errno == errno.ENOTEMPTY:
346		- return False
347		- else:
348		- raise UtilError("Failed to remove '{}': {}"
349		- .format(path, e))
350	337
351		- return True
	338	+ # File removed/unlinked successfully
	339	+ return True
	340	+
	341	+ except OSError as e:
	342	+ if e.errno == errno.ENOTEMPTY:
	343	+ # Path is non-empty directory
	344	+ return False
	345	+ elif e.errno == errno.ENOENT:
	346	+ # Path does not exist
	347	+ return True
	348	+
	349	+ raise UtilError("Failed to remove '{}': {}"
	350	+ .format(path, e))
352	351
353	352
354	353	def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=False):

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.