[Notes] [Git][BuildStream/buildstream][master] 3 commits: tests: Migrated cache quota test into artifactcache/cache_size.py

[Tristan Van Berkom <gitlab mg gitlab com>]

Title: GitLab

Tristan Van Berkom pushed to branch master at BuildStream / buildstream

Commits:

• 0098a900
  by Tristan Van Berkom at 2019-01-23T13:57:53Z

  tests: Migrated cache quota test into artifactcache/cache_size.py
  
  Instead of sitting mysteriously alone in internals/utils.py
  

• 9be31641
  by Tristan Van Berkom at 2019-01-23T13:57:53Z

  _artifactcache.py: Raise ArtifactError() when quota size exceeds disk space.
  
  This is not an error related to loading data, like a parse error
  in the quota specification is, but a problem raised by the artifact
  cache - this allows us to assert more specific machine readable
  errors in test cases (instead of checking the string in stderr, which
  this patch also fixes).
  
  This also removes a typo from the error message in the said error.
  
    * tests/artifactcache/cache_size.py
  
      Updated test case to expect the artifact error, which consequently
      changes the test case to properly assert a machine readable error
      instead of asserting text in the stderr (which is the real, secret
      motivation behind this patch).
  
    * tests/artifactcache/expiry.py: Reworked test_invalid_cache_quota()
  
      Now expect the artifact error for the tests which check configurations
      which create caches too large to fit on the disk.
  

• 3a6e953f
  by Tristan Van Berkom at 2019-01-24T01:59:13Z

  Merge branch 'tristan/insufficient-storage-error' into 'master'
  
  Tristan/insufficient storage error
  
  See merge request BuildStream/buildstream!1102

4 changed files:

• buildstream/_artifactcache.py
• tests/artifactcache/cache_size.py
• tests/artifactcache/expiry.py
• − tests/internals/utils.py

Changes:

buildstream/_artifactcache.py

---

...	...	@@ -882,16 +882,16 @@ class ArtifactCache():
882	882	else:
883	883	available = utils._pretty_size(available_space)
884	884
885		- raise LoadError(LoadErrorReason.INVALID_DATA,
886		- ("Your system does not have enough available " +
887		- "space to support the cache quota specified.\n" +
888		- "\nYou have specified a quota of {quota} total disk space.\n" +
889		- "- The filesystem containing {local_cache_path} only " +
890		- "has: {available_size} available.")
891		- .format(
892		- quota=self.context.config_cache_quota,
893		- local_cache_path=self.context.artifactdir,
894		- available_size=available))
	885	+ raise ArtifactError("Your system does not have enough available " +
	886	+ "space to support the cache quota specified.",
	887	+ detail=("You have specified a quota of {quota} total disk space.\n" +
	888	+ "The filesystem containing {local_cache_path} only " +
	889	+ "has {available_size} available.")
	890	+ .format(
	891	+ quota=self.context.config_cache_quota,
	892	+ local_cache_path=self.context.artifactdir,
	893	+ available_size=available),
	894	+ reason='insufficient-storage-for-quota')
895	895
896	896	# Place a slight headroom (2e9 (2GB) on the cache_quota) into
897	897	# cache_quota to try and avoid exceptions.

tests/artifactcache/cache_size.py

---

1	1	import os
2	2	import pytest
	3	+from unittest import mock
3	4
4	5	from buildstream import _yaml
5	6	from buildstream._artifactcache import CACHE_SIZE_FILE
	7	+from buildstream._exceptions import ErrorDomain
6	8
7	9	from tests.testutils import cli, create_element_size
8	10
...	...	@@ -60,3 +62,29 @@ def test_cache_size_write(cli, tmpdir):
60	62	with open(sizefile, "r") as f:
61	63	size_data = f.read()
62	64	size = int(size_data)
	65	+
	66	+
	67	+def test_quota_over_1024T(cli, tmpdir):
	68	+ KiB = 1024
	69	+ MiB = (KiB * 1024)
	70	+ GiB = (MiB * 1024)
	71	+ TiB = (GiB * 1024)
	72	+
	73	+ cli.configure({
	74	+ 'cache': {
	75	+ 'quota': 2048 * TiB
	76	+ }
	77	+ })
	78	+ project = tmpdir.join("main")
	79	+ os.makedirs(str(project))
	80	+ _yaml.dump({'name': 'main'}, str(project.join("project.conf")))
	81	+
	82	+ volume_space_patch = mock.patch(
	83	+ "buildstream._artifactcache.ArtifactCache._get_volume_space_info_for",
	84	+ autospec=True,
	85	+ return_value=(1025 * TiB, 1025 * TiB)
	86	+ )
	87	+
	88	+ with volume_space_patch:
	89	+ result = cli.run(project, args=["build", "file.bst"])
	90	+ result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')

tests/artifactcache/expiry.py

---

...	...	@@ -304,20 +304,28 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
304	304
305	305	# Ensure that only valid cache quotas make it through the loading
306	306	# process.
307		-@pytest.mark.parametrize("quota,success", [
308		- ("1", True),
309		- ("1K", True),
310		- ("50%", True),
311		- ("infinity", True),
312		- ("0", True),
313		- ("-1", False),
314		- ("pony", False),
315		- ("7K", False),
316		- ("70%", False),
317		- ("200%", False)
	307	+#
	308	+# This test virtualizes the condition to assume a storage volume
	309	+# has 10K total disk space, and 6K of it is already in use (not
	310	+# including any space used by the artifact cache).
	311	+#
	312	+@pytest.mark.parametrize("quota,err_domain,err_reason", [
	313	+ # Valid configurations
	314	+ ("1", 'success', None),
	315	+ ("1K", 'success', None),
	316	+ ("50%", 'success', None),
	317	+ ("infinity", 'success', None),
	318	+ ("0", 'success', None),
	319	+ # Invalid configurations
	320	+ ("-1", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
	321	+ ("pony", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
	322	+ ("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
	323	+ # Not enough space for these caches
	324	+ ("7K", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota'),
	325	+ ("70%", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')
318	326	])
319	327	@pytest.mark.datafiles(DATA_DIR)
320		-def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success):
	328	+def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reason):
321	329	project = os.path.join(datafiles.dirname, datafiles.basename)
322	330	os.makedirs(os.path.join(project, 'elements'))
323	331
...	...	@@ -356,10 +364,10 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success):
356	364	with volume_space_patch, cache_size_patch:
357	365	res = cli.run(project=project, args=['workspace', 'list'])
358	366
359		- if success:
	367	+ if err_domain == 'success':
360	368	res.assert_success()
361	369	else:
362		- res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
	370	+ res.assert_main_error(err_domain, err_reason)
363	371
364	372
365	373	@pytest.mark.datafiles(DATA_DIR)

tests/internals/utils.py deleted

---

1		-import os
2		-from unittest import mock
3		-
4		-from buildstream import _yaml
5		-
6		-from ..testutils.runcli import cli
7		-
8		-
9		-KiB = 1024
10		-MiB = (KiB * 1024)
11		-GiB = (MiB * 1024)
12		-TiB = (GiB * 1024)
13		-
14		-
15		-def test_parse_size_over_1024T(cli, tmpdir):
16		- cli.configure({
17		- 'cache': {
18		- 'quota': 2048 * TiB
19		- }
20		- })
21		- project = tmpdir.join("main")
22		- os.makedirs(str(project))
23		- _yaml.dump({'name': 'main'}, str(project.join("project.conf")))
24		-
25		- volume_space_patch = mock.patch(
26		- "buildstream._artifactcache.ArtifactCache._get_volume_space_info_for",
27		- autospec=True,
28		- return_value=(1025 * TiB, 1025 * TiB)
29		- )
30		-
31		- with volume_space_patch:
32		- result = cli.run(project, args=["build", "file.bst"])
33		- failure_msg = 'Your system does not have enough available space to support the cache quota specified.'
34		- assert failure_msg in result.stderr

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.