Tristan Van Berkom pushed to branch master at BuildStream / buildstream
Commits:
-
d03bf316
by Benjamin Schubert at 2018-12-13T10:24:11Z
-
a116f576
by Tristan Van Berkom at 2018-12-13T10:58:46Z
4 changed files:
- buildstream/_artifactcache/artifactcache.py
- tests/artifactcache/expiry.py
- − tests/testutils/mock_os.py
- tests/utils/misc.py
Changes:
... | ... | @@ -874,9 +874,7 @@ class ArtifactCache(): |
874 | 874 |
"\nValid values are, for example: 800M 10G 1T 50%\n"
|
875 | 875 |
.format(str(e))) from e
|
876 | 876 |
|
877 |
- stat = os.statvfs(artifactdir_volume)
|
|
878 |
- available_space = (stat.f_bsize * stat.f_bavail)
|
|
879 |
- |
|
877 |
+ available_space, total_size = self._get_volume_space_info_for(artifactdir_volume)
|
|
880 | 878 |
cache_size = self.get_cache_size()
|
881 | 879 |
|
882 | 880 |
# Ensure system has enough storage for the cache_quota
|
... | ... | @@ -893,7 +891,7 @@ class ArtifactCache(): |
893 | 891 |
"BuildStream requires a minimum cache quota of 2G.")
|
894 | 892 |
elif cache_quota > cache_size + available_space: # Check maximum
|
895 | 893 |
if '%' in self.context.config_cache_quota:
|
896 |
- available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100
|
|
894 |
+ available = (available_space / total_size) * 100
|
|
897 | 895 |
available = '{}% of total disk space'.format(round(available, 1))
|
898 | 896 |
else:
|
899 | 897 |
available = utils._pretty_size(available_space)
|
... | ... | @@ -919,6 +917,20 @@ class ArtifactCache(): |
919 | 917 |
self._cache_quota = cache_quota - headroom
|
920 | 918 |
self._cache_lower_threshold = self._cache_quota / 2
|
921 | 919 |
|
920 |
+ # _get_volume_space_info_for
|
|
921 |
+ #
|
|
922 |
+ # Get the available space and total space for the given volume
|
|
923 |
+ #
|
|
924 |
+ # Args:
|
|
925 |
+ # volume: volume for which to get the size
|
|
926 |
+ #
|
|
927 |
+ # Returns:
|
|
928 |
+ # A tuple containing first the availabe number of bytes on the requested
|
|
929 |
+ # volume, then the total number of bytes of the volume.
|
|
930 |
+ def _get_volume_space_info_for(self, volume):
|
|
931 |
+ stat = os.statvfs(volume)
|
|
932 |
+ return stat.f_bsize * stat.f_bavail, stat.f_bsize * stat.f_blocks
|
|
933 |
+ |
|
922 | 934 |
|
923 | 935 |
# _configured_remote_artifact_cache_specs():
|
924 | 936 |
#
|
... | ... | @@ -18,6 +18,7 @@ |
18 | 18 |
#
|
19 | 19 |
|
20 | 20 |
import os
|
21 |
+from unittest import mock
|
|
21 | 22 |
|
22 | 23 |
import pytest
|
23 | 24 |
|
... | ... | @@ -311,6 +312,8 @@ def test_never_delete_required_track(cli, datafiles, tmpdir): |
311 | 312 |
("0", True),
|
312 | 313 |
("-1", False),
|
313 | 314 |
("pony", False),
|
315 |
+ ("7K", False),
|
|
316 |
+ ("70%", False),
|
|
314 | 317 |
("200%", False)
|
315 | 318 |
])
|
316 | 319 |
@pytest.mark.datafiles(DATA_DIR)
|
... | ... | @@ -324,7 +327,35 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success): |
324 | 327 |
}
|
325 | 328 |
})
|
326 | 329 |
|
327 |
- res = cli.run(project=project, args=['workspace', 'list'])
|
|
330 |
+ # We patch how we get space information
|
|
331 |
+ # Ideally we would instead create a FUSE device on which we control
|
|
332 |
+ # everything.
|
|
333 |
+ # If the value is a percentage, we fix the current values to take into
|
|
334 |
+ # account the block size, since this is important in how we compute the size
|
|
335 |
+ |
|
336 |
+ if quota.endswith("%"): # We set the used space at 60% of total space
|
|
337 |
+ stats = os.statvfs(".")
|
|
338 |
+ free_space = 0.6 * stats.f_bsize * stats.f_blocks
|
|
339 |
+ total_space = stats.f_bsize * stats.f_blocks
|
|
340 |
+ else:
|
|
341 |
+ free_space = 6000
|
|
342 |
+ total_space = 10000
|
|
343 |
+ |
|
344 |
+ volume_space_patch = mock.patch(
|
|
345 |
+ "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for",
|
|
346 |
+ autospec=True,
|
|
347 |
+ return_value=(free_space, total_space),
|
|
348 |
+ )
|
|
349 |
+ |
|
350 |
+ cache_size_patch = mock.patch(
|
|
351 |
+ "buildstream._artifactcache.artifactcache.ArtifactCache.get_cache_size",
|
|
352 |
+ autospec=True,
|
|
353 |
+ return_value=0,
|
|
354 |
+ )
|
|
355 |
+ |
|
356 |
+ with volume_space_patch, cache_size_patch:
|
|
357 |
+ res = cli.run(project=project, args=['workspace', 'list'])
|
|
358 |
+ |
|
328 | 359 |
if success:
|
329 | 360 |
res.assert_success()
|
330 | 361 |
else:
|
1 |
-from contextlib import contextmanager
|
|
2 |
-import os
|
|
3 |
- |
|
4 |
- |
|
5 |
-# MockAttributeResult
|
|
6 |
-#
|
|
7 |
-# A class to take a dictionary of kwargs and make them accessible via
|
|
8 |
-# attributes of the object.
|
|
9 |
-#
|
|
10 |
-class MockAttributeResult(dict):
|
|
11 |
- __getattr__ = dict.get
|
|
12 |
- |
|
13 |
- |
|
14 |
-# mock_statvfs():
|
|
15 |
-#
|
|
16 |
-# Gets a function which mocks statvfs and returns a statvfs result with the kwargs accessible.
|
|
17 |
-#
|
|
18 |
-# Returns:
|
|
19 |
-# func(path) -> object: object will have all the kwargs accessible via object.kwarg
|
|
20 |
-#
|
|
21 |
-# Example:
|
|
22 |
-# statvfs = mock_statvfs(f_blocks=10)
|
|
23 |
-# result = statvfs("regardless/of/path")
|
|
24 |
-# assert result.f_blocks == 10 # True
|
|
25 |
-def mock_statvfs(**kwargs):
|
|
26 |
- def statvfs(path):
|
|
27 |
- return MockAttributeResult(kwargs)
|
|
28 |
- return statvfs
|
|
29 |
- |
|
30 |
- |
|
31 |
-# monkey_patch()
|
|
32 |
-#
|
|
33 |
-# with monkey_patch("statvfs", custom_statvfs):
|
|
34 |
-# assert os.statvfs == custom_statvfs # True
|
|
35 |
-# assert os.statvfs == custom_statvfs # False
|
|
36 |
-#
|
|
37 |
-@contextmanager
|
|
38 |
-def monkey_patch(to_patch, patched_func):
|
|
39 |
- orig = getattr(os, to_patch)
|
|
40 |
- setattr(os, to_patch, patched_func)
|
|
41 |
- try:
|
|
42 |
- yield
|
|
43 |
- finally:
|
|
44 |
- setattr(os, to_patch, orig)
|
1 |
+import os
|
|
2 |
+from unittest import mock
|
|
3 |
+ |
|
1 | 4 |
from buildstream import _yaml
|
2 |
-from ..testutils import mock_os
|
|
3 |
-from ..testutils.runcli import cli
|
|
4 | 5 |
|
5 |
-import os
|
|
6 |
-import pytest
|
|
6 |
+from ..testutils.runcli import cli
|
|
7 | 7 |
|
8 | 8 |
|
9 | 9 |
KiB = 1024
|
... | ... | @@ -13,7 +13,6 @@ TiB = (GiB * 1024) |
13 | 13 |
|
14 | 14 |
|
15 | 15 |
def test_parse_size_over_1024T(cli, tmpdir):
|
16 |
- BLOCK_SIZE = 4096
|
|
17 | 16 |
cli.configure({
|
18 | 17 |
'cache': {
|
19 | 18 |
'quota': 2048 * TiB
|
... | ... | @@ -23,9 +22,13 @@ def test_parse_size_over_1024T(cli, tmpdir): |
23 | 22 |
os.makedirs(str(project))
|
24 | 23 |
_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
|
25 | 24 |
|
26 |
- bavail = (1025 * TiB) / BLOCK_SIZE
|
|
27 |
- patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE)
|
|
28 |
- with mock_os.monkey_patch("statvfs", patched_statvfs):
|
|
25 |
+ volume_space_patch = mock.patch(
|
|
26 |
+ "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for",
|
|
27 |
+ autospec=True,
|
|
28 |
+ return_value=(1025 * TiB, 1025 * TiB)
|
|
29 |
+ )
|
|
30 |
+ |
|
31 |
+ with volume_space_patch:
|
|
29 | 32 |
result = cli.run(project, args=["build", "file.bst"])
|
30 | 33 |
failure_msg = 'Your system does not have enough available space to support the cache quota specified.'
|
31 | 34 |
assert failure_msg in result.stderr
|