[Notes] [Git][BuildStream/buildstream][BenjaminSchubert/fix-quota-tests] Mock storage space checks for tests.

[Benjamin Schubert <gitlab mg gitlab com>]

Title: GitLab

Benjamin Schubert pushed to branch BenjaminSchubert/fix-quota-tests at BuildStream / buildstream

Commits:

• d21a1218
  by Benjamin Schubert at 2018-12-07T13:43:57Z

  Mock storage space checks for tests.
  
  Fix #530
  
  - Extract free space computation in a function for easier
  mocking
  - Mock space computation during cache quota tests
  - Mock cache size during cachque quota tests
  - Add two more tests when the configuration would require
  to much storage space
  

4 changed files:

• buildstream/_artifactcache/artifactcache.py
• tests/artifactcache/expiry.py
• − tests/testutils/mock_os.py
• tests/utils/misc.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -874,9 +874,7 @@ class ArtifactCache():
874	874	"\nValid values are, for example: 800M 10G 1T 50%\n"
875	875	.format(str(e))) from e
876	876
877		- stat = os.statvfs(artifactdir_volume)
878		- available_space = (stat.f_bsize * stat.f_bavail)
879		-
	877	+ available_space, total_size = self._get_volume_space_info_for(artifactdir_volume)
880	878	cache_size = self.get_cache_size()
881	879
882	880	# Ensure system has enough storage for the cache_quota
...	...	@@ -893,7 +891,7 @@ class ArtifactCache():
893	891	"BuildStream requires a minimum cache quota of 2G.")
894	892	elif cache_quota > cache_size + available_space: # Check maximum
895	893	if '%' in self.context.config_cache_quota:
896		- available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100
	894	+ available = (available_space / total_size) * 100
897	895	available = '{}% of total disk space'.format(round(available, 1))
898	896	else:
899	897	available = utils._pretty_size(available_space)
...	...	@@ -919,6 +917,20 @@ class ArtifactCache():
919	917	self._cache_quota = cache_quota - headroom
920	918	self._cache_lower_threshold = self._cache_quota / 2
921	919
	920	+ # _get_volume_space_info_for
	921	+ #
	922	+ # Get the available space and total space for the given volume
	923	+ #
	924	+ # Args:
	925	+ # volume: volume for which to get the size
	926	+ #
	927	+ # Returns:
	928	+ # A tuple containing first the availabe number of bytes on the requested
	929	+ # volume, then the total number of bytes of the volume.
	930	+ def _get_volume_space_info_for(self, volume):
	931	+ stat = os.statvfs(volume)
	932	+ return stat.f_bsize * stat.f_bavail, stat.f_bsize * stat.f_blocks
	933	+
922	934
923	935	# _configured_remote_artifact_cache_specs():
924	936	#

tests/artifactcache/expiry.py

---

...	...	@@ -18,6 +18,7 @@
18	18	#
19	19
20	20	import os
	21	+from unittest import mock
21	22
22	23	import pytest
23	24
...	...	@@ -311,6 +312,8 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
311	312	("0", True),
312	313	("-1", False),
313	314	("pony", False),
	315	+ ("7K", False),
	316	+ ("70%", False),
314	317	("200%", False)
315	318	])
316	319	@pytest.mark.datafiles(DATA_DIR)
...	...	@@ -324,7 +327,35 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success):
324	327	}
325	328	})
326	329
327		- res = cli.run(project=project, args=['workspace', 'list'])
	330	+ # We patch how we get space information
	331	+ # Ideally we would instead create a FUSE device on which we control
	332	+ # everything.
	333	+ # If the value is a percentage, we fix the current values to take into
	334	+ # account the block size, since this is important in how we compute the size
	335	+
	336	+ if quota.endswith("%"): # We set the used space at 60% of total space
	337	+ stats = os.statvfs(".")
	338	+ free_space = 0.6 * stats.f_bsize * stats.f_blocks
	339	+ total_space = stats.f_bsize * stats.f_blocks
	340	+ else:
	341	+ free_space = 6000
	342	+ total_space = 10000
	343	+
	344	+ volume_space_patch = mock.patch(
	345	+ "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for",
	346	+ autospec=True,
	347	+ return_value=(free_space, total_space),
	348	+ )
	349	+
	350	+ cache_size_patch = mock.patch(
	351	+ "buildstream._artifactcache.artifactcache.ArtifactCache.get_cache_size",
	352	+ autospec=True,
	353	+ return_value=0,
	354	+ )
	355	+
	356	+ with volume_space_patch, cache_size_patch:
	357	+ res = cli.run(project=project, args=['workspace', 'list'])
	358	+
328	359	if success:
329	360	res.assert_success()
330	361	else:

tests/testutils/mock_os.py deleted

---

1		-from contextlib import contextmanager
2		-import os
3		-
4		-
5		-# MockAttributeResult
6		-#
7		-# A class to take a dictionary of kwargs and make them accessible via
8		-# attributes of the object.
9		-#
10		-class MockAttributeResult(dict):
11		- __getattr__ = dict.get
12		-
13		-
14		-# mock_statvfs():
15		-#
16		-# Gets a function which mocks statvfs and returns a statvfs result with the kwargs accessible.
17		-#
18		-# Returns:
19		-# func(path) -> object: object will have all the kwargs accessible via object.kwarg
20		-#
21		-# Example:
22		-# statvfs = mock_statvfs(f_blocks=10)
23		-# result = statvfs("regardless/of/path")
24		-# assert result.f_blocks == 10 # True
25		-def mock_statvfs(**kwargs):
26		- def statvfs(path):
27		- return MockAttributeResult(kwargs)
28		- return statvfs
29		-
30		-
31		-# monkey_patch()
32		-#
33		-# with monkey_patch("statvfs", custom_statvfs):
34		-# assert os.statvfs == custom_statvfs # True
35		-# assert os.statvfs == custom_statvfs # False
36		-#
37		-@contextmanager
38		-def monkey_patch(to_patch, patched_func):
39		- orig = getattr(os, to_patch)
40		- setattr(os, to_patch, patched_func)
41		- try:
42		- yield
43		- finally:
44		- setattr(os, to_patch, orig)

tests/utils/misc.py

---

	1	+import os
	2	+from unittest import mock
	3	+
1	4	from buildstream import _yaml
2		-from ..testutils import mock_os
3		-from ..testutils.runcli import cli
4	5
5		-import os
6		-import pytest
	6	+from ..testutils.runcli import cli
7	7
8	8
9	9	KiB = 1024
...	...	@@ -13,7 +13,6 @@ TiB = (GiB * 1024)
13	13
14	14
15	15	def test_parse_size_over_1024T(cli, tmpdir):
16		- BLOCK_SIZE = 4096
17	16	cli.configure({
18	17	'cache': {
19	18	'quota': 2048 * TiB
...	...	@@ -23,9 +22,13 @@ def test_parse_size_over_1024T(cli, tmpdir):
23	22	os.makedirs(str(project))
24	23	_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
25	24
26		- bavail = (1025 * TiB) / BLOCK_SIZE
27		- patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE)
28		- with mock_os.monkey_patch("statvfs", patched_statvfs):
	25	+ volume_space_patch = mock.patch(
	26	+ "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for",
	27	+ autospec=True,
	28	+ return_value=(1025 * TiB, 1025 * TiB)
	29	+ )
	30	+
	31	+ with volume_space_patch:
29	32	result = cli.run(project, args=["build", "file.bst"])
30	33	failure_msg = 'Your system does not have enough available space to support the cache quota specified.'
31	34	assert failure_msg in result.stderr

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.