[Notes] [Git][BuildStream/buildstream][master] 2 commits: Mock storage space checks for tests.



Title: GitLab

Tristan Van Berkom pushed to branch master at BuildStream / buildstream

Commits:

4 changed files:

Changes:

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -874,9 +874,7 @@ class ArtifactCache():
    874 874
                                 "\nValid values are, for example: 800M 10G 1T 50%\n"
    
    875 875
                                 .format(str(e))) from e
    
    876 876
     
    
    877
    -        stat = os.statvfs(artifactdir_volume)
    
    878
    -        available_space = (stat.f_bsize * stat.f_bavail)
    
    879
    -
    
    877
    +        available_space, total_size = self._get_volume_space_info_for(artifactdir_volume)
    
    880 878
             cache_size = self.get_cache_size()
    
    881 879
     
    
    882 880
             # Ensure system has enough storage for the cache_quota
    
    ... ... @@ -893,7 +891,7 @@ class ArtifactCache():
    893 891
                                 "BuildStream requires a minimum cache quota of 2G.")
    
    894 892
             elif cache_quota > cache_size + available_space:  # Check maximum
    
    895 893
                 if '%' in self.context.config_cache_quota:
    
    896
    -                available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100
    
    894
    +                available = (available_space / total_size) * 100
    
    897 895
                     available = '{}% of total disk space'.format(round(available, 1))
    
    898 896
                 else:
    
    899 897
                     available = utils._pretty_size(available_space)
    
    ... ... @@ -919,6 +917,20 @@ class ArtifactCache():
    919 917
             self._cache_quota = cache_quota - headroom
    
    920 918
             self._cache_lower_threshold = self._cache_quota / 2
    
    921 919
     
    
    920
    +    # _get_volume_space_info_for
    
    921
    +    #
    
    922
    +    # Get the available space and total space for the given volume
    
    923
    +    #
    
    924
    +    # Args:
    
    925
    +    #     volume: volume for which to get the size
    
    926
    +    #
    
    927
    +    # Returns:
    
    928
    +    #     A tuple containing first the availabe number of bytes on the requested
    
    929
    +    #     volume, then the total number of bytes of the volume.
    
    930
    +    def _get_volume_space_info_for(self, volume):
    
    931
    +        stat = os.statvfs(volume)
    
    932
    +        return stat.f_bsize * stat.f_bavail, stat.f_bsize * stat.f_blocks
    
    933
    +
    
    922 934
     
    
    923 935
     # _configured_remote_artifact_cache_specs():
    
    924 936
     #
    

  • tests/artifactcache/expiry.py
    ... ... @@ -18,6 +18,7 @@
    18 18
     #
    
    19 19
     
    
    20 20
     import os
    
    21
    +from unittest import mock
    
    21 22
     
    
    22 23
     import pytest
    
    23 24
     
    
    ... ... @@ -311,6 +312,8 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
    311 312
         ("0", True),
    
    312 313
         ("-1", False),
    
    313 314
         ("pony", False),
    
    315
    +    ("7K", False),
    
    316
    +    ("70%", False),
    
    314 317
         ("200%", False)
    
    315 318
     ])
    
    316 319
     @pytest.mark.datafiles(DATA_DIR)
    
    ... ... @@ -324,7 +327,35 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success):
    324 327
             }
    
    325 328
         })
    
    326 329
     
    
    327
    -    res = cli.run(project=project, args=['workspace', 'list'])
    
    330
    +    # We patch how we get space information
    
    331
    +    # Ideally we would instead create a FUSE device on which we control
    
    332
    +    # everything.
    
    333
    +    # If the value is a percentage, we fix the current values to take into
    
    334
    +    # account the block size, since this is important in how we compute the size
    
    335
    +
    
    336
    +    if quota.endswith("%"):  # We set the used space at 60% of total space
    
    337
    +        stats = os.statvfs(".")
    
    338
    +        free_space = 0.6 * stats.f_bsize * stats.f_blocks
    
    339
    +        total_space = stats.f_bsize * stats.f_blocks
    
    340
    +    else:
    
    341
    +        free_space = 6000
    
    342
    +        total_space = 10000
    
    343
    +
    
    344
    +    volume_space_patch = mock.patch(
    
    345
    +        "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for",
    
    346
    +        autospec=True,
    
    347
    +        return_value=(free_space, total_space),
    
    348
    +    )
    
    349
    +
    
    350
    +    cache_size_patch = mock.patch(
    
    351
    +        "buildstream._artifactcache.artifactcache.ArtifactCache.get_cache_size",
    
    352
    +        autospec=True,
    
    353
    +        return_value=0,
    
    354
    +    )
    
    355
    +
    
    356
    +    with volume_space_patch, cache_size_patch:
    
    357
    +        res = cli.run(project=project, args=['workspace', 'list'])
    
    358
    +
    
    328 359
         if success:
    
    329 360
             res.assert_success()
    
    330 361
         else:
    

  • tests/testutils/mock_os.py deleted
    1
    -from contextlib import contextmanager
    
    2
    -import os
    
    3
    -
    
    4
    -
    
    5
    -# MockAttributeResult
    
    6
    -#
    
    7
    -# A class to take a dictionary of kwargs and make them accessible via
    
    8
    -# attributes of the object.
    
    9
    -#
    
    10
    -class MockAttributeResult(dict):
    
    11
    -    __getattr__ = dict.get
    
    12
    -
    
    13
    -
    
    14
    -# mock_statvfs():
    
    15
    -#
    
    16
    -# Gets a function which mocks statvfs and returns a statvfs result with the kwargs accessible.
    
    17
    -#
    
    18
    -# Returns:
    
    19
    -#    func(path) -> object: object will have all the kwargs accessible via object.kwarg
    
    20
    -#
    
    21
    -# Example:
    
    22
    -#    statvfs = mock_statvfs(f_blocks=10)
    
    23
    -#    result = statvfs("regardless/of/path")
    
    24
    -#    assert result.f_blocks == 10 # True
    
    25
    -def mock_statvfs(**kwargs):
    
    26
    -    def statvfs(path):
    
    27
    -        return MockAttributeResult(kwargs)
    
    28
    -    return statvfs
    
    29
    -
    
    30
    -
    
    31
    -# monkey_patch()
    
    32
    -#
    
    33
    -# with monkey_patch("statvfs", custom_statvfs):
    
    34
    -#    assert os.statvfs == custom_statvfs # True
    
    35
    -# assert os.statvfs == custom_statvfs # False
    
    36
    -#
    
    37
    -@contextmanager
    
    38
    -def monkey_patch(to_patch, patched_func):
    
    39
    -    orig = getattr(os, to_patch)
    
    40
    -    setattr(os, to_patch, patched_func)
    
    41
    -    try:
    
    42
    -        yield
    
    43
    -    finally:
    
    44
    -        setattr(os, to_patch, orig)

  • tests/utils/misc.py
    1
    +import os
    
    2
    +from unittest import mock
    
    3
    +
    
    1 4
     from buildstream import _yaml
    
    2
    -from ..testutils import mock_os
    
    3
    -from ..testutils.runcli import cli
    
    4 5
     
    
    5
    -import os
    
    6
    -import pytest
    
    6
    +from ..testutils.runcli import cli
    
    7 7
     
    
    8 8
     
    
    9 9
     KiB = 1024
    
    ... ... @@ -13,7 +13,6 @@ TiB = (GiB * 1024)
    13 13
     
    
    14 14
     
    
    15 15
     def test_parse_size_over_1024T(cli, tmpdir):
    
    16
    -    BLOCK_SIZE = 4096
    
    17 16
         cli.configure({
    
    18 17
             'cache': {
    
    19 18
                 'quota': 2048 * TiB
    
    ... ... @@ -23,9 +22,13 @@ def test_parse_size_over_1024T(cli, tmpdir):
    23 22
         os.makedirs(str(project))
    
    24 23
         _yaml.dump({'name': 'main'}, str(project.join("project.conf")))
    
    25 24
     
    
    26
    -    bavail = (1025 * TiB) / BLOCK_SIZE
    
    27
    -    patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE)
    
    28
    -    with mock_os.monkey_patch("statvfs", patched_statvfs):
    
    25
    +    volume_space_patch = mock.patch(
    
    26
    +        "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for",
    
    27
    +        autospec=True,
    
    28
    +        return_value=(1025 * TiB, 1025 * TiB)
    
    29
    +    )
    
    30
    +
    
    31
    +    with volume_space_patch:
    
    29 32
             result = cli.run(project, args=["build", "file.bst"])
    
    30 33
             failure_msg = 'Your system does not have enough available space to support the cache quota specified.'
    
    31 34
             assert failure_msg in result.stderr



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]