[Notes] [Git][BuildStream/buildstream][tristan/dont-batch-prepare-assemble-by-default] 8 commits: Mock storage space checks for tests.

[Tristan Van Berkom <gitlab mg gitlab com>]

Title: GitLab

Tristan Van Berkom pushed to branch tristan/dont-batch-prepare-assemble-by-default at BuildStream / buildstream

Commits:

• d03bf316
  by Benjamin Schubert at 2018-12-13T10:24:11Z

  Mock storage space checks for tests.
  
  Fix #530
  
  - Extract free space computation in a function for easier
  mocking
  - Mock space computation during cache quota tests
  - Mock cache size during cachque quota tests
  - Add two more tests when the configuration would require
  to much storage space
  

• a116f576
  by Tristan Van Berkom at 2018-12-13T10:58:46Z

  Merge branch 'BenjaminSchubert/fix-quota-tests' into 'master'
  
  Mock storage space checks for tests.
  
  Closes #530
  
  See merge request BuildStream/buildstream!702

• e133dbd9
  by Tristan Van Berkom at 2018-12-13T11:06:59Z

  manual plugin: Support virtual directories
  

• 6c64fc35
  by Tristan Van Berkom at 2018-12-13T11:06:59Z

  makemaker plugin: Support virtual directories
  

• 51ffa706
  by Tristan Van Berkom at 2018-12-13T11:06:59Z

  pip element plugin: Support virtual directories
  

• f2e2a891
  by Tristan Van Berkom at 2018-12-13T11:06:59Z

  modulebuild plugin: Support virtual directories
  

• 741aa42c
  by Tristan Van Berkom at 2018-12-13T11:06:59Z

  distutils plugin: Support virtual directories
  

• 66c111a8
  by Tristan Van Berkom at 2018-12-13T11:06:59Z

  BuildElement: Don't enable batching of prepare and assemble by default
  
  Some external plugins depend derive from BuildElement and are broken
  by BuildElement enabling this batching by default.
  
  Instead, enable it in all of the individual build element plugin
  implementations.
  
  This fixes issue #800
  

15 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/buildelement.py
• buildstream/plugins/elements/autotools.py
• buildstream/plugins/elements/cmake.py
• buildstream/plugins/elements/distutils.py
• buildstream/plugins/elements/make.py
• buildstream/plugins/elements/makemaker.py
• buildstream/plugins/elements/manual.py
• buildstream/plugins/elements/meson.py
• buildstream/plugins/elements/modulebuild.py
• buildstream/plugins/elements/pip.py
• buildstream/plugins/elements/qmake.py
• tests/artifactcache/expiry.py
• − tests/testutils/mock_os.py
• tests/utils/misc.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -874,9 +874,7 @@ class ArtifactCache():
874	874	"\nValid values are, for example: 800M 10G 1T 50%\n"
875	875	.format(str(e))) from e
876	876
877		- stat = os.statvfs(artifactdir_volume)
878		- available_space = (stat.f_bsize * stat.f_bavail)
879		-
	877	+ available_space, total_size = self._get_volume_space_info_for(artifactdir_volume)
880	878	cache_size = self.get_cache_size()
881	879
882	880	# Ensure system has enough storage for the cache_quota
...	...	@@ -893,7 +891,7 @@ class ArtifactCache():
893	891	"BuildStream requires a minimum cache quota of 2G.")
894	892	elif cache_quota > cache_size + available_space: # Check maximum
895	893	if '%' in self.context.config_cache_quota:
896		- available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100
	894	+ available = (available_space / total_size) * 100
897	895	available = '{}% of total disk space'.format(round(available, 1))
898	896	else:
899	897	available = utils._pretty_size(available_space)
...	...	@@ -919,6 +917,20 @@ class ArtifactCache():
919	917	self._cache_quota = cache_quota - headroom
920	918	self._cache_lower_threshold = self._cache_quota / 2
921	919
	920	+ # _get_volume_space_info_for
	921	+ #
	922	+ # Get the available space and total space for the given volume
	923	+ #
	924	+ # Args:
	925	+ # volume: volume for which to get the size
	926	+ #
	927	+ # Returns:
	928	+ # A tuple containing first the availabe number of bytes on the requested
	929	+ # volume, then the total number of bytes of the volume.
	930	+ def _get_volume_space_info_for(self, volume):
	931	+ stat = os.statvfs(volume)
	932	+ return stat.f_bsize * stat.f_bavail, stat.f_bsize * stat.f_blocks
	933	+
922	934
923	935	# _configured_remote_artifact_cache_specs():
924	936	#

buildstream/buildelement.py

---

...	...	@@ -215,10 +215,6 @@ class BuildElement(Element):
215	215	# Setup environment
216	216	sandbox.set_environment(self.get_environment())
217	217
218		- # Enable command batching across prepare() and assemble()
219		- self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
220		- collect=self.get_variable('install-root'))
221		-
222	218	def stage(self, sandbox):
223	219
224	220	# Stage deps in the sandbox root

buildstream/plugins/elements/autotools.py

---

...	...	@@ -55,7 +55,7 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
55	55	details on common configuration options for build elements.
56	56	"""
57	57
58		-from buildstream import BuildElement
	58	+from buildstream import BuildElement, SandboxFlags
59	59
60	60
61	61	# Element implementation for the 'autotools' kind.
...	...	@@ -63,6 +63,12 @@ class AutotoolsElement(BuildElement):
63	63	# Supports virtual directories (required for remote execution)
64	64	BST_VIRTUAL_DIRECTORY = True
65	65
	66	+ # Enable command batching across prepare() and assemble()
	67	+ def configure_sandbox(self, sandbox):
	68	+ super().configure_sandbox(sandbox)
	69	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	70	+ collect=self.get_variable('install-root'))
	71	+
66	72
67	73	# Plugin entry point
68	74	def setup():

buildstream/plugins/elements/cmake.py

---

...	...	@@ -54,7 +54,7 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
54	54	details on common configuration options for build elements.
55	55	"""
56	56
57		-from buildstream import BuildElement
	57	+from buildstream import BuildElement, SandboxFlags
58	58
59	59
60	60	# Element implementation for the 'cmake' kind.
...	...	@@ -62,6 +62,12 @@ class CMakeElement(BuildElement):
62	62	# Supports virtual directories (required for remote execution)
63	63	BST_VIRTUAL_DIRECTORY = True
64	64
	65	+ # Enable command batching across prepare() and assemble()
	66	+ def configure_sandbox(self, sandbox):
	67	+ super().configure_sandbox(sandbox)
	68	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	69	+ collect=self.get_variable('install-root'))
	70	+
65	71
66	72	# Plugin entry point
67	73	def setup():

buildstream/plugins/elements/distutils.py

---

...	...	@@ -31,12 +31,19 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
31	31	details on common configuration options for build elements.
32	32	"""
33	33
34		-from buildstream import BuildElement
	34	+from buildstream import BuildElement, SandboxFlags
35	35
36	36
37	37	# Element implementation for the python 'distutils' kind.
38	38	class DistutilsElement(BuildElement):
39		- pass
	39	+ # Supports virtual directories (required for remote execution)
	40	+ BST_VIRTUAL_DIRECTORY = True
	41	+
	42	+ # Enable command batching across prepare() and assemble()
	43	+ def configure_sandbox(self, sandbox):
	44	+ super().configure_sandbox(sandbox)
	45	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	46	+ collect=self.get_variable('install-root'))
40	47
41	48
42	49	# Plugin entry point

buildstream/plugins/elements/make.py

---

...	...	@@ -36,7 +36,7 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
36	36	details on common configuration options for build elements.
37	37	"""
38	38
39		-from buildstream import BuildElement
	39	+from buildstream import BuildElement, SandboxFlags
40	40
41	41
42	42	# Element implementation for the 'make' kind.
...	...	@@ -44,6 +44,12 @@ class MakeElement(BuildElement):
44	44	# Supports virtual directories (required for remote execution)
45	45	BST_VIRTUAL_DIRECTORY = True
46	46
	47	+ # Enable command batching across prepare() and assemble()
	48	+ def configure_sandbox(self, sandbox):
	49	+ super().configure_sandbox(sandbox)
	50	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	51	+ collect=self.get_variable('install-root'))
	52	+
47	53
48	54	# Plugin entry point
49	55	def setup():

buildstream/plugins/elements/makemaker.py

---

...	...	@@ -31,12 +31,19 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
31	31	details on common configuration options for build elements.
32	32	"""
33	33
34		-from buildstream import BuildElement
	34	+from buildstream import BuildElement, SandboxFlags
35	35
36	36
37	37	# Element implementation for the 'makemaker' kind.
38	38	class MakeMakerElement(BuildElement):
39		- pass
	39	+ # Supports virtual directories (required for remote execution)
	40	+ BST_VIRTUAL_DIRECTORY = True
	41	+
	42	+ # Enable command batching across prepare() and assemble()
	43	+ def configure_sandbox(self, sandbox):
	44	+ super().configure_sandbox(sandbox)
	45	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	46	+ collect=self.get_variable('install-root'))
40	47
41	48
42	49	# Plugin entry point

buildstream/plugins/elements/manual.py

---

...	...	@@ -31,12 +31,19 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
31	31	details on common configuration options for build elements.
32	32	"""
33	33
34		-from buildstream import BuildElement
	34	+from buildstream import BuildElement, SandboxFlags
35	35
36	36
37	37	# Element implementation for the 'manual' kind.
38	38	class ManualElement(BuildElement):
39		- pass
	39	+ # Supports virtual directories (required for remote execution)
	40	+ BST_VIRTUAL_DIRECTORY = True
	41	+
	42	+ # Enable command batching across prepare() and assemble()
	43	+ def configure_sandbox(self, sandbox):
	44	+ super().configure_sandbox(sandbox)
	45	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	46	+ collect=self.get_variable('install-root'))
40	47
41	48
42	49	# Plugin entry point

buildstream/plugins/elements/meson.py

---

...	...	@@ -51,7 +51,7 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
51	51	details on common configuration options for build elements.
52	52	"""
53	53
54		-from buildstream import BuildElement
	54	+from buildstream import BuildElement, SandboxFlags
55	55
56	56
57	57	# Element implementation for the 'meson' kind.
...	...	@@ -59,6 +59,12 @@ class MesonElement(BuildElement):
59	59	# Supports virtual directories (required for remote execution)
60	60	BST_VIRTUAL_DIRECTORY = True
61	61
	62	+ # Enable command batching across prepare() and assemble()
	63	+ def configure_sandbox(self, sandbox):
	64	+ super().configure_sandbox(sandbox)
	65	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	66	+ collect=self.get_variable('install-root'))
	67	+
62	68
63	69	# Plugin entry point
64	70	def setup():

buildstream/plugins/elements/modulebuild.py

---

...	...	@@ -31,12 +31,19 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
31	31	details on common configuration options for build elements.
32	32	"""
33	33
34		-from buildstream import BuildElement
	34	+from buildstream import BuildElement, SandboxFlags
35	35
36	36
37	37	# Element implementation for the 'modulebuild' kind.
38	38	class ModuleBuildElement(BuildElement):
39		- pass
	39	+ # Supports virtual directories (required for remote execution)
	40	+ BST_VIRTUAL_DIRECTORY = True
	41	+
	42	+ # Enable command batching across prepare() and assemble()
	43	+ def configure_sandbox(self, sandbox):
	44	+ super().configure_sandbox(sandbox)
	45	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	46	+ collect=self.get_variable('install-root'))
40	47
41	48
42	49	# Plugin entry point

buildstream/plugins/elements/pip.py

---

...	...	@@ -31,12 +31,19 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
31	31	details on common configuration options for build elements.
32	32	"""
33	33
34		-from buildstream import BuildElement
	34	+from buildstream import BuildElement, SandboxFlags
35	35
36	36
37	37	# Element implementation for the 'pip' kind.
38	38	class PipElement(BuildElement):
39		- pass
	39	+ # Supports virtual directories (required for remote execution)
	40	+ BST_VIRTUAL_DIRECTORY = True
	41	+
	42	+ # Enable command batching across prepare() and assemble()
	43	+ def configure_sandbox(self, sandbox):
	44	+ super().configure_sandbox(sandbox)
	45	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	46	+ collect=self.get_variable('install-root'))
40	47
41	48
42	49	# Plugin entry point

buildstream/plugins/elements/qmake.py

---

...	...	@@ -31,7 +31,7 @@ See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
31	31	details on common configuration options for build elements.
32	32	"""
33	33
34		-from buildstream import BuildElement
	34	+from buildstream import BuildElement, SandboxFlags
35	35
36	36
37	37	# Element implementation for the 'qmake' kind.
...	...	@@ -39,6 +39,12 @@ class QMakeElement(BuildElement):
39	39	# Supports virtual directories (required for remote execution)
40	40	BST_VIRTUAL_DIRECTORY = True
41	41
	42	+ # Enable command batching across prepare() and assemble()
	43	+ def configure_sandbox(self, sandbox):
	44	+ super().configure_sandbox(sandbox)
	45	+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
	46	+ collect=self.get_variable('install-root'))
	47	+
42	48
43	49	# Plugin entry point
44	50	def setup():

tests/artifactcache/expiry.py

---

...	...	@@ -18,6 +18,7 @@
18	18	#
19	19
20	20	import os
	21	+from unittest import mock
21	22
22	23	import pytest
23	24
...	...	@@ -311,6 +312,8 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
311	312	("0", True),
312	313	("-1", False),
313	314	("pony", False),
	315	+ ("7K", False),
	316	+ ("70%", False),
314	317	("200%", False)
315	318	])
316	319	@pytest.mark.datafiles(DATA_DIR)
...	...	@@ -324,7 +327,35 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success):
324	327	}
325	328	})
326	329
327		- res = cli.run(project=project, args=['workspace', 'list'])
	330	+ # We patch how we get space information
	331	+ # Ideally we would instead create a FUSE device on which we control
	332	+ # everything.
	333	+ # If the value is a percentage, we fix the current values to take into
	334	+ # account the block size, since this is important in how we compute the size
	335	+
	336	+ if quota.endswith("%"): # We set the used space at 60% of total space
	337	+ stats = os.statvfs(".")
	338	+ free_space = 0.6 * stats.f_bsize * stats.f_blocks
	339	+ total_space = stats.f_bsize * stats.f_blocks
	340	+ else:
	341	+ free_space = 6000
	342	+ total_space = 10000
	343	+
	344	+ volume_space_patch = mock.patch(
	345	+ "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for",
	346	+ autospec=True,
	347	+ return_value=(free_space, total_space),
	348	+ )
	349	+
	350	+ cache_size_patch = mock.patch(
	351	+ "buildstream._artifactcache.artifactcache.ArtifactCache.get_cache_size",
	352	+ autospec=True,
	353	+ return_value=0,
	354	+ )
	355	+
	356	+ with volume_space_patch, cache_size_patch:
	357	+ res = cli.run(project=project, args=['workspace', 'list'])
	358	+
328	359	if success:
329	360	res.assert_success()
330	361	else:

tests/testutils/mock_os.py deleted

---

1		-from contextlib import contextmanager
2		-import os
3		-
4		-
5		-# MockAttributeResult
6		-#
7		-# A class to take a dictionary of kwargs and make them accessible via
8		-# attributes of the object.
9		-#
10		-class MockAttributeResult(dict):
11		- __getattr__ = dict.get
12		-
13		-
14		-# mock_statvfs():
15		-#
16		-# Gets a function which mocks statvfs and returns a statvfs result with the kwargs accessible.
17		-#
18		-# Returns:
19		-# func(path) -> object: object will have all the kwargs accessible via object.kwarg
20		-#
21		-# Example:
22		-# statvfs = mock_statvfs(f_blocks=10)
23		-# result = statvfs("regardless/of/path")
24		-# assert result.f_blocks == 10 # True
25		-def mock_statvfs(**kwargs):
26		- def statvfs(path):
27		- return MockAttributeResult(kwargs)
28		- return statvfs
29		-
30		-
31		-# monkey_patch()
32		-#
33		-# with monkey_patch("statvfs", custom_statvfs):
34		-# assert os.statvfs == custom_statvfs # True
35		-# assert os.statvfs == custom_statvfs # False
36		-#
37		-@contextmanager
38		-def monkey_patch(to_patch, patched_func):
39		- orig = getattr(os, to_patch)
40		- setattr(os, to_patch, patched_func)
41		- try:
42		- yield
43		- finally:
44		- setattr(os, to_patch, orig)

tests/utils/misc.py

---

	1	+import os
	2	+from unittest import mock
	3	+
1	4	from buildstream import _yaml
2		-from ..testutils import mock_os
3		-from ..testutils.runcli import cli
4	5
5		-import os
6		-import pytest
	6	+from ..testutils.runcli import cli
7	7
8	8
9	9	KiB = 1024
...	...	@@ -13,7 +13,6 @@ TiB = (GiB * 1024)
13	13
14	14
15	15	def test_parse_size_over_1024T(cli, tmpdir):
16		- BLOCK_SIZE = 4096
17	16	cli.configure({
18	17	'cache': {
19	18	'quota': 2048 * TiB
...	...	@@ -23,9 +22,13 @@ def test_parse_size_over_1024T(cli, tmpdir):
23	22	os.makedirs(str(project))
24	23	_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
25	24
26		- bavail = (1025 * TiB) / BLOCK_SIZE
27		- patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE)
28		- with mock_os.monkey_patch("statvfs", patched_statvfs):
	25	+ volume_space_patch = mock.patch(
	26	+ "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for",
	27	+ autospec=True,
	28	+ return_value=(1025 * TiB, 1025 * TiB)
	29	+ )
	30	+
	31	+ with volume_space_patch:
29	32	result = cli.run(project, args=["build", "file.bst"])
30	33	failure_msg = 'Your system does not have enough available space to support the cache quota specified.'
31	34	assert failure_msg in result.stderr

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.