[Notes] [Git][BuildStream/buildstream][raoul/870-root-cache-dir] CASCache: Move cache check methods here



Title: GitLab

Raoul Hidalgo Charman pushed to branch raoul/870-root-cache-dir at BuildStream / buildstream

Commits:

14 changed files:

Changes:

  • buildstream/_artifactcache.py
    ... ... @@ -23,7 +23,7 @@ import string
    23 23
     from collections.abc import Mapping
    
    24 24
     
    
    25 25
     from .types import _KeyStrength
    
    26
    -from ._exceptions import ArtifactError, CASError, LoadError, LoadErrorReason
    
    26
    +from ._exceptions import ArtifactError, CASError
    
    27 27
     from ._message import Message, MessageType
    
    28 28
     from . import utils
    
    29 29
     from . import _yaml
    
    ... ... @@ -46,39 +46,6 @@ class ArtifactCacheSpec(CASRemoteSpec):
    46 46
         pass
    
    47 47
     
    
    48 48
     
    
    49
    -# ArtifactCacheUsage
    
    50
    -#
    
    51
    -# A simple object to report the current artifact cache
    
    52
    -# usage details.
    
    53
    -#
    
    54
    -# Note that this uses the user configured cache quota
    
    55
    -# rather than the internal quota with protective headroom
    
    56
    -# removed, to provide a more sensible value to display to
    
    57
    -# the user.
    
    58
    -#
    
    59
    -# Args:
    
    60
    -#    artifacts (ArtifactCache): The artifact cache to get the status of
    
    61
    -#
    
    62
    -class ArtifactCacheUsage():
    
    63
    -
    
    64
    -    def __init__(self, artifacts):
    
    65
    -        context = artifacts.context
    
    66
    -        self.quota_config = context.config_cache_quota       # Configured quota
    
    67
    -        self.quota_size = artifacts._cache_quota_original    # Resolved cache quota in bytes
    
    68
    -        self.used_size = artifacts.get_cache_size()          # Size used by artifacts in bytes
    
    69
    -        self.used_percent = 0                                # Percentage of the quota used
    
    70
    -        if self.quota_size is not None:
    
    71
    -            self.used_percent = int(self.used_size * 100 / self.quota_size)
    
    72
    -
    
    73
    -    # Formattable into a human readable string
    
    74
    -    #
    
    75
    -    def __str__(self):
    
    76
    -        return "{} / {} ({}%)" \
    
    77
    -            .format(utils._pretty_size(self.used_size, dec_places=1),
    
    78
    -                    self.quota_config,
    
    79
    -                    self.used_percent)
    
    80
    -
    
    81
    -
    
    82 49
     # An ArtifactCache manages artifacts.
    
    83 50
     #
    
    84 51
     # Args:
    
    ... ... @@ -91,15 +58,23 @@ class ArtifactCache():
    91 58
     
    
    92 59
             self.cas = context.get_cascache()
    
    93 60
     
    
    61
    +        # check to see if the cache quota exceeds space available
    
    62
    +        if self.cas.cache_quota_exceeded:
    
    63
    +            self._message(MessageType.WARN,
    
    64
    +                          "Your system does not have enough available " +
    
    65
    +                          "space to support the cache quota specified.",
    
    66
    +                          detail=("You have specified a quota of {quota} total disk space.\n" +
    
    67
    +                                  "The filesystem containing {local_cache_path} only " +
    
    68
    +                                  "has {available_size} available.")
    
    69
    +                          .format(quota=context.config_cache_quota,
    
    70
    +                                  local_cache_path=context.rootcachedir,
    
    71
    +                                  available_size=self.cas.available_space))
    
    72
    +
    
    94 73
             self.global_remote_specs = []
    
    95 74
             self.project_remote_specs = {}
    
    96 75
     
    
    97 76
             self._required_elements = set()       # The elements required for this session
    
    98
    -        self._cache_size = None               # The current cache size, sometimes it's an estimate
    
    99
    -        self._cache_quota = None              # The cache quota
    
    100
    -        self._cache_quota_original = None     # The cache quota as specified by the user, in bytes
    
    101
    -        self._cache_quota_headroom = None     # The headroom in bytes before reaching the quota or full disk
    
    102
    -        self._cache_lower_threshold = None    # The target cache size for a cleanup
    
    77
    +
    
    103 78
             self._remotes_setup = False           # Check to prevent double-setup of remotes
    
    104 79
     
    
    105 80
             # Per-project list of _CASRemote instances.
    
    ... ... @@ -110,8 +85,6 @@ class ArtifactCache():
    110 85
     
    
    111 86
             os.makedirs(self.extractdir, exist_ok=True)
    
    112 87
     
    
    113
    -        self._calculate_cache_quota()
    
    114
    -
    
    115 88
         # get_artifact_fullname()
    
    116 89
         #
    
    117 90
         # Generate a full name for an artifact, including the
    
    ... ... @@ -266,7 +239,7 @@ class ArtifactCache():
    266 239
             space_saved = 0
    
    267 240
     
    
    268 241
             # Start off with an announcement with as much info as possible
    
    269
    -        volume_size, volume_avail = self._get_cache_volume_size()
    
    242
    +        volume_size, volume_avail = self.cas._get_cache_volume_size()
    
    270 243
             self._message(MessageType.STATUS, "Starting cache cleanup",
    
    271 244
                           detail=("Elements required by the current build plan: {}\n" +
    
    272 245
                                   "User specified quota: {} ({})\n" +
    
    ... ... @@ -274,8 +247,8 @@ class ArtifactCache():
    274 247
                                   "Cache volume: {} total, {} available")
    
    275 248
                           .format(len(self._required_elements),
    
    276 249
                                   context.config_cache_quota,
    
    277
    -                              utils._pretty_size(self._cache_quota_original, dec_places=2),
    
    278
    -                              utils._pretty_size(self.get_cache_size(), dec_places=2),
    
    250
    +                              utils._pretty_size(self.cas._cache_quota, dec_places=2),
    
    251
    +                              utils._pretty_size(self.cas.get_cache_size(), dec_places=2),
    
    279 252
                                   utils._pretty_size(volume_size, dec_places=2),
    
    280 253
                                   utils._pretty_size(volume_avail, dec_places=2)))
    
    281 254
     
    
    ... ... @@ -292,9 +265,10 @@ class ArtifactCache():
    292 265
                 ])
    
    293 266
     
    
    294 267
             # Do a real computation of the cache size once, just in case
    
    295
    -        self.compute_cache_size()
    
    268
    +        usage = self.cas.compute_cache_size()
    
    269
    +        self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
    
    296 270
     
    
    297
    -        while self.get_cache_size() >= self._cache_lower_threshold:
    
    271
    +        while self.cas.get_cache_size() >= self.cas._cache_lower_threshold:
    
    298 272
                 try:
    
    299 273
                     to_remove = artifacts.pop(0)
    
    300 274
                 except IndexError:
    
    ... ... @@ -311,7 +285,7 @@ class ArtifactCache():
    311 285
                               "Please increase the cache-quota in {} and/or make more disk space."
    
    312 286
                               .format(removed_ref_count,
    
    313 287
                                       utils._pretty_size(space_saved, dec_places=2),
    
    314
    -                                  utils._pretty_size(self.get_cache_size(), dec_places=2),
    
    288
    +                                  utils._pretty_size(self.cas.get_cache_size(), dec_places=2),
    
    315 289
                                       len(self._required_elements),
    
    316 290
                                       (context.config_origin or default_conf)))
    
    317 291
     
    
    ... ... @@ -337,7 +311,7 @@ class ArtifactCache():
    337 311
                                       to_remove))
    
    338 312
     
    
    339 313
                     # Remove the size from the removed size
    
    340
    -                self.set_cache_size(self._cache_size - size)
    
    314
    +                self.cas.set_cache_size(self.cas._cache_size - size)
    
    341 315
     
    
    342 316
                     # User callback
    
    343 317
                     #
    
    ... ... @@ -353,29 +327,12 @@ class ArtifactCache():
    353 327
                                   "Cache usage is now: {}")
    
    354 328
                           .format(removed_ref_count,
    
    355 329
                                   utils._pretty_size(space_saved, dec_places=2),
    
    356
    -                              utils._pretty_size(self.get_cache_size(), dec_places=2)))
    
    357
    -
    
    358
    -        return self.get_cache_size()
    
    359
    -
    
    360
    -    # compute_cache_size()
    
    361
    -    #
    
    362
    -    # Computes the real artifact cache size by calling
    
    363
    -    # the abstract calculate_cache_size() method.
    
    364
    -    #
    
    365
    -    # Returns:
    
    366
    -    #    (int): The size of the artifact cache.
    
    367
    -    #
    
    368
    -    def compute_cache_size(self):
    
    369
    -        old_cache_size = self._cache_size
    
    370
    -        new_cache_size = self.cas.calculate_cache_size()
    
    330
    +                              utils._pretty_size(self.cas.get_cache_size(), dec_places=2)))
    
    371 331
     
    
    372
    -        if old_cache_size != new_cache_size:
    
    373
    -            self._cache_size = new_cache_size
    
    332
    +        return self.cas.get_cache_size()
    
    374 333
     
    
    375
    -            usage = ArtifactCacheUsage(self)
    
    376
    -            self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
    
    377
    -
    
    378
    -        return self._cache_size
    
    334
    +    def full(self):
    
    335
    +        return self.cas.full()
    
    379 336
     
    
    380 337
         # add_artifact_size()
    
    381 338
         #
    
    ... ... @@ -386,71 +343,10 @@ class ArtifactCache():
    386 343
         #     artifact_size (int): The size to add.
    
    387 344
         #
    
    388 345
         def add_artifact_size(self, artifact_size):
    
    389
    -        cache_size = self.get_cache_size()
    
    346
    +        cache_size = self.cas.get_cache_size()
    
    390 347
             cache_size += artifact_size
    
    391 348
     
    
    392
    -        self.set_cache_size(cache_size)
    
    393
    -
    
    394
    -    # get_cache_size()
    
    395
    -    #
    
    396
    -    # Fetches the cached size of the cache, this is sometimes
    
    397
    -    # an estimate and periodically adjusted to the real size
    
    398
    -    # when a cache size calculation job runs.
    
    399
    -    #
    
    400
    -    # When it is an estimate, the value is either correct, or
    
    401
    -    # it is greater than the actual cache size.
    
    402
    -    #
    
    403
    -    # Returns:
    
    404
    -    #     (int) An approximation of the artifact cache size, in bytes.
    
    405
    -    #
    
    406
    -    def get_cache_size(self):
    
    407
    -
    
    408
    -        # If we don't currently have an estimate, figure out the real cache size.
    
    409
    -        if self._cache_size is None:
    
    410
    -            stored_size = self._read_cache_size()
    
    411
    -            if stored_size is not None:
    
    412
    -                self._cache_size = stored_size
    
    413
    -            else:
    
    414
    -                self.compute_cache_size()
    
    415
    -
    
    416
    -        return self._cache_size
    
    417
    -
    
    418
    -    # set_cache_size()
    
    419
    -    #
    
    420
    -    # Forcefully set the overall cache size.
    
    421
    -    #
    
    422
    -    # This is used to update the size in the main process after
    
    423
    -    # having calculated in a cleanup or a cache size calculation job.
    
    424
    -    #
    
    425
    -    # Args:
    
    426
    -    #     cache_size (int): The size to set.
    
    427
    -    #
    
    428
    -    def set_cache_size(self, cache_size):
    
    429
    -
    
    430
    -        assert cache_size is not None
    
    431
    -
    
    432
    -        self._cache_size = cache_size
    
    433
    -        self._write_cache_size(self._cache_size)
    
    434
    -
    
    435
    -    # full()
    
    436
    -    #
    
    437
    -    # Checks if the artifact cache is full, either
    
    438
    -    # because the user configured quota has been exceeded
    
    439
    -    # or because the underlying disk is almost full.
    
    440
    -    #
    
    441
    -    # Returns:
    
    442
    -    #    (bool): True if the artifact cache is full
    
    443
    -    #
    
    444
    -    def full(self):
    
    445
    -
    
    446
    -        if self.get_cache_size() > self._cache_quota:
    
    447
    -            return True
    
    448
    -
    
    449
    -        _, volume_avail = self._get_cache_volume_size()
    
    450
    -        if volume_avail < self._cache_quota_headroom:
    
    451
    -            return True
    
    452
    -
    
    453
    -        return False
    
    349
    +        self.cas.set_cache_size(cache_size)
    
    454 350
     
    
    455 351
         # preflight():
    
    456 352
         #
    
    ... ... @@ -896,142 +792,6 @@ class ArtifactCache():
    896 792
             with self.context.timed_activity("Initializing remote caches", silent_nested=True):
    
    897 793
                 self.initialize_remotes(on_failure=remote_failed)
    
    898 794
     
    
    899
    -    # _write_cache_size()
    
    900
    -    #
    
    901
    -    # Writes the given size of the artifact to the cache's size file
    
    902
    -    #
    
    903
    -    # Args:
    
    904
    -    #    size (int): The size of the artifact cache to record
    
    905
    -    #
    
    906
    -    def _write_cache_size(self, size):
    
    907
    -        assert isinstance(size, int)
    
    908
    -        size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
    
    909
    -        with utils.save_file_atomic(size_file_path, "w") as f:
    
    910
    -            f.write(str(size))
    
    911
    -
    
    912
    -    # _read_cache_size()
    
    913
    -    #
    
    914
    -    # Reads and returns the size of the artifact cache that's stored in the
    
    915
    -    # cache's size file
    
    916
    -    #
    
    917
    -    # Returns:
    
    918
    -    #    (int): The size of the artifact cache, as recorded in the file
    
    919
    -    #
    
    920
    -    def _read_cache_size(self):
    
    921
    -        size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
    
    922
    -
    
    923
    -        if not os.path.exists(size_file_path):
    
    924
    -            return None
    
    925
    -
    
    926
    -        with open(size_file_path, "r") as f:
    
    927
    -            size = f.read()
    
    928
    -
    
    929
    -        try:
    
    930
    -            num_size = int(size)
    
    931
    -        except ValueError as e:
    
    932
    -            raise ArtifactError("Size '{}' parsed from '{}' was not an integer".format(
    
    933
    -                size, size_file_path)) from e
    
    934
    -
    
    935
    -        return num_size
    
    936
    -
    
    937
    -    # _calculate_cache_quota()
    
    938
    -    #
    
    939
    -    # Calculates and sets the cache quota and lower threshold based on the
    
    940
    -    # quota set in Context.
    
    941
    -    # It checks that the quota is both a valid _expression_, and that there is
    
    942
    -    # enough disk space to satisfy that quota
    
    943
    -    #
    
    944
    -    def _calculate_cache_quota(self):
    
    945
    -        # Headroom intended to give BuildStream a bit of leeway.
    
    946
    -        # This acts as the minimum size of cache_quota and also
    
    947
    -        # is taken from the user requested cache_quota.
    
    948
    -        #
    
    949
    -        if 'BST_TEST_SUITE' in os.environ:
    
    950
    -            self._cache_quota_headroom = 0
    
    951
    -        else:
    
    952
    -            self._cache_quota_headroom = 2e9
    
    953
    -
    
    954
    -        try:
    
    955
    -            cache_quota = utils._parse_size(self.context.config_cache_quota,
    
    956
    -                                            self.context.artifactdir)
    
    957
    -        except utils.UtilError as e:
    
    958
    -            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    959
    -                            "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
    
    960
    -                            "\nValid values are, for example: 800M 10G 1T 50%\n"
    
    961
    -                            .format(str(e))) from e
    
    962
    -
    
    963
    -        total_size, available_space = self._get_cache_volume_size()
    
    964
    -        cache_size = self.get_cache_size()
    
    965
    -
    
    966
    -        # Ensure system has enough storage for the cache_quota
    
    967
    -        #
    
    968
    -        # If cache_quota is none, set it to the maximum it could possibly be.
    
    969
    -        #
    
    970
    -        # Also check that cache_quota is at least as large as our headroom.
    
    971
    -        #
    
    972
    -        if cache_quota is None:  # Infinity, set to max system storage
    
    973
    -            cache_quota = cache_size + available_space
    
    974
    -        if cache_quota < self._cache_quota_headroom:  # Check minimum
    
    975
    -            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    976
    -                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
    
    977
    -                            "BuildStream requires a minimum cache quota of 2G.")
    
    978
    -        elif cache_quota > total_size:
    
    979
    -            # A quota greater than the total disk size is certianly an error
    
    980
    -            raise ArtifactError("Your system does not have enough available " +
    
    981
    -                                "space to support the cache quota specified.",
    
    982
    -                                detail=("You have specified a quota of {quota} total disk space.\n" +
    
    983
    -                                        "The filesystem containing {local_cache_path} only " +
    
    984
    -                                        "has {total_size} total disk space.")
    
    985
    -                                .format(
    
    986
    -                                    quota=self.context.config_cache_quota,
    
    987
    -                                    local_cache_path=self.context.artifactdir,
    
    988
    -                                    total_size=utils._pretty_size(total_size)),
    
    989
    -                                reason='insufficient-storage-for-quota')
    
    990
    -        elif cache_quota > cache_size + available_space:
    
    991
    -            # The quota does not fit in the available space, this is a warning
    
    992
    -            if '%' in self.context.config_cache_quota:
    
    993
    -                available = (available_space / total_size) * 100
    
    994
    -                available = '{}% of total disk space'.format(round(available, 1))
    
    995
    -            else:
    
    996
    -                available = utils._pretty_size(available_space)
    
    997
    -
    
    998
    -            self._message(MessageType.WARN,
    
    999
    -                          "Your system does not have enough available " +
    
    1000
    -                          "space to support the cache quota specified.",
    
    1001
    -                          detail=("You have specified a quota of {quota} total disk space.\n" +
    
    1002
    -                                  "The filesystem containing {local_cache_path} only " +
    
    1003
    -                                  "has {available_size} available.")
    
    1004
    -                          .format(quota=self.context.config_cache_quota,
    
    1005
    -                                  local_cache_path=self.context.artifactdir,
    
    1006
    -                                  available_size=available))
    
    1007
    -
    
    1008
    -        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
    
    1009
    -        # cache_quota to try and avoid exceptions.
    
    1010
    -        #
    
    1011
    -        # Of course, we might still end up running out during a build
    
    1012
    -        # if we end up writing more than 2G, but hey, this stuff is
    
    1013
    -        # already really fuzzy.
    
    1014
    -        #
    
    1015
    -        self._cache_quota_original = cache_quota
    
    1016
    -        self._cache_quota = cache_quota - self._cache_quota_headroom
    
    1017
    -        self._cache_lower_threshold = self._cache_quota / 2
    
    1018
    -
    
    1019
    -    # _get_cache_volume_size()
    
    1020
    -    #
    
    1021
    -    # Get the available space and total space for the volume on
    
    1022
    -    # which the artifact cache is located.
    
    1023
    -    #
    
    1024
    -    # Returns:
    
    1025
    -    #    (int): The total number of bytes on the volume
    
    1026
    -    #    (int): The number of available bytes on the volume
    
    1027
    -    #
    
    1028
    -    # NOTE: We use this stub to allow the test cases
    
    1029
    -    #       to override what an artifact cache thinks
    
    1030
    -    #       about it's disk size and available bytes.
    
    1031
    -    #
    
    1032
    -    def _get_cache_volume_size(self):
    
    1033
    -        return utils._get_volume_size(self.context.artifactdir)
    
    1034
    -
    
    1035 795
     
    
    1036 796
     # _configured_remote_artifact_cache_specs():
    
    1037 797
     #
    

  • buildstream/_cas/__init__.py
    ... ... @@ -17,5 +17,5 @@
    17 17
     #  Authors:
    
    18 18
     #        Tristan Van Berkom <tristan vanberkom codethink co uk>
    
    19 19
     
    
    20
    -from .cascache import CASCache
    
    20
    +from .cascache import CASCache, CASCacheUsage
    
    21 21
     from .casremote import CASRemote, CASRemoteSpec

  • buildstream/_cas/cascache.py
    ... ... @@ -31,25 +31,73 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    31 31
     from .._protos.buildstream.v2 import buildstream_pb2
    
    32 32
     
    
    33 33
     from .. import utils
    
    34
    -from .._exceptions import CASCacheError
    
    34
    +from .._exceptions import CASCacheError, LoadError, LoadErrorReason
    
    35 35
     
    
    36 36
     from .casremote import BlobNotFound, _CASBatchRead, _CASBatchUpdate
    
    37 37
     
    
    38 38
     
    
    39
    +CACHE_SIZE_FILE = "cache_size"
    
    40
    +
    
    41
    +
    
    42
    +# CASCacheUsage
    
    43
    +#
    
    44
    +# A simple object to report the current CAS cache usage details.
    
    45
    +#
    
    46
    +# Note that this uses the user configured cache quota
    
    47
    +# rather than the internal quota with protective headroom
    
    48
    +# removed, to provide a more sensible value to display to
    
    49
    +# the user.
    
    50
    +#
    
    51
    +# Args:
    
    52
    +#    cas (CASCache): The CAS cache to get the status of
    
    53
    +#
    
    54
    +class CASCacheUsage():
    
    55
    +
    
    56
    +    def __init__(self, cas):
    
    57
    +        self.quota_config = cas._config_cache_quota          # Configured quota
    
    58
    +        self.quota_size = cas._cache_quota_original          # Resolved cache quota in bytes
    
    59
    +        self.used_size = cas.get_cache_size()                # Size used by artifacts in bytes
    
    60
    +        self.used_percent = 0                                # Percentage of the quota used
    
    61
    +        if self.quota_size is not None:
    
    62
    +            self.used_percent = int(self.used_size * 100 / self.quota_size)
    
    63
    +
    
    64
    +    # Formattable into a human readable string
    
    65
    +    #
    
    66
    +    def __str__(self):
    
    67
    +        return "{} / {} ({}%)" \
    
    68
    +            .format(utils._pretty_size(self.used_size, dec_places=1),
    
    69
    +                    self.quota_config,
    
    70
    +                    self.used_percent)
    
    71
    +
    
    72
    +
    
    39 73
     # A CASCache manages a CAS repository as specified in the Remote Execution API.
    
    40 74
     #
    
    41 75
     # Args:
    
    42 76
     #     path (str): The root directory for the CAS repository
    
    77
    +#     cache_quota (int): User configured cache quota
    
    43 78
     #
    
    44 79
     class CASCache():
    
    45 80
     
    
    46
    -    def __init__(self, path):
    
    81
    +    def __init__(self, path, cache_quota=None):
    
    47 82
             self.casdir = os.path.join(path, 'cas')
    
    48 83
             self.tmpdir = os.path.join(path, 'tmp')
    
    49 84
             os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
    
    50 85
             os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
    
    51 86
             os.makedirs(self.tmpdir, exist_ok=True)
    
    52 87
     
    
    88
    +        self._config_cache_quota = cache_quota
    
    89
    +        self._cache_size = None               # The current cache size, sometimes it's an estimate
    
    90
    +        self._cache_quota = None              # The cache quota
    
    91
    +        self._cache_quota_original = None     # The cache quota as specified by the user, in bytes
    
    92
    +        self._cache_quota_headroom = None
    
    93
    +        self._cache_lower_threshold = None    # The target cache size for a cleanup
    
    94
    +
    
    95
    +        # Flag used for when cache quota is larger than available space
    
    96
    +        self.cache_quota_exceeded = False
    
    97
    +        self.available_space = None
    
    98
    +
    
    99
    +        self._calculate_cache_quota()
    
    100
    +
    
    53 101
         # preflight():
    
    54 102
         #
    
    55 103
         # Preflight check.
    
    ... ... @@ -587,6 +635,84 @@ class CASCache():
    587 635
             reachable = set()
    
    588 636
             self._reachable_refs_dir(reachable, tree, update_mtime=True)
    
    589 637
     
    
    638
    +    # compute_cache_size()
    
    639
    +    #
    
    640
    +    # Computes the real artifact cache size by calling
    
    641
    +    # the abstract calculate_cache_size() method.
    
    642
    +    #
    
    643
    +    # Returns:
    
    644
    +    #    (int): The size of the artifact cache.
    
    645
    +    #
    
    646
    +    def compute_cache_size(self):
    
    647
    +        old_cache_size = self._cache_size
    
    648
    +        new_cache_size = self.calculate_cache_size()
    
    649
    +
    
    650
    +        if old_cache_size != new_cache_size:
    
    651
    +            self._cache_size = new_cache_size
    
    652
    +
    
    653
    +        return self._cache_size
    
    654
    +
    
    655
    +    # get_cache_size()
    
    656
    +    #
    
    657
    +    # Fetches the cached size of the cache, this is sometimes
    
    658
    +    # an estimate and periodically adjusted to the real size
    
    659
    +    # when a cache size calculation job runs.
    
    660
    +    #
    
    661
    +    # When it is an estimate, the value is either correct, or
    
    662
    +    # it is greater than the actual cache size.
    
    663
    +    #
    
    664
    +    # Returns:
    
    665
    +    #     (int) An approximation of the artifact cache size, in bytes.
    
    666
    +    #
    
    667
    +    def get_cache_size(self):
    
    668
    +
    
    669
    +        # If we don't currently have an estimate, figure out the real cache size.
    
    670
    +        if self._cache_size is None:
    
    671
    +            stored_size = self._read_cache_size()
    
    672
    +            if stored_size is not None:
    
    673
    +                self._cache_size = stored_size
    
    674
    +            else:
    
    675
    +                self._cache_size = self.compute_cache_size()
    
    676
    +
    
    677
    +        return self._cache_size
    
    678
    +
    
    679
    +    # set_cache_size()
    
    680
    +    #
    
    681
    +    # Forcefully set the overall cache size.
    
    682
    +    #
    
    683
    +    # This is used to update the size in the main process after
    
    684
    +    # having calculated in a cleanup or a cache size calculation job.
    
    685
    +    #
    
    686
    +    # Args:
    
    687
    +    #     cache_size (int): The size to set.
    
    688
    +    #
    
    689
    +    def set_cache_size(self, cache_size):
    
    690
    +
    
    691
    +        assert cache_size is not None
    
    692
    +
    
    693
    +        self._cache_size = cache_size
    
    694
    +        self._write_cache_size(self._cache_size)
    
    695
    +
    
    696
    +    # full()
    
    697
    +    #
    
    698
    +    # Checks if the artifact cache is full, either
    
    699
    +    # because the user configured quota has been exceeded
    
    700
    +    # or because the underlying disk is almost full.
    
    701
    +    #
    
    702
    +    # Returns:
    
    703
    +    #    (bool): True if the artifact cache is full
    
    704
    +    #
    
    705
    +    def full(self):
    
    706
    +
    
    707
    +        if self.get_cache_size() > self._cache_quota:
    
    708
    +            return True
    
    709
    +
    
    710
    +        _, volume_avail = self._get_cache_volume_size()
    
    711
    +        if volume_avail < self._cache_quota_headroom:
    
    712
    +            return True
    
    713
    +
    
    714
    +        return False
    
    715
    +
    
    590 716
         ################################################
    
    591 717
         #             Local Private Methods            #
    
    592 718
         ################################################
    
    ... ... @@ -1015,6 +1141,122 @@ class CASCache():
    1015 1141
             # Send final batch
    
    1016 1142
             batch.send()
    
    1017 1143
     
    
    1144
    +    # _read_cache_size()
    
    1145
    +    #
    
    1146
    +    # Reads and returns the size of the artifact cache that's stored in the
    
    1147
    +    # cache's size file
    
    1148
    +    #
    
    1149
    +    # Returns:
    
    1150
    +    #    (int): The size of the artifact cache, as recorded in the file
    
    1151
    +    #
    
    1152
    +    def _read_cache_size(self):
    
    1153
    +        size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
    
    1154
    +
    
    1155
    +        if not os.path.exists(size_file_path):
    
    1156
    +            return None
    
    1157
    +
    
    1158
    +        with open(size_file_path, "r") as f:
    
    1159
    +            size = f.read()
    
    1160
    +
    
    1161
    +        try:
    
    1162
    +            num_size = int(size)
    
    1163
    +        except ValueError as e:
    
    1164
    +            raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format(
    
    1165
    +                size, size_file_path)) from e
    
    1166
    +
    
    1167
    +        return num_size
    
    1168
    +
    
    1169
    +    # _write_cache_size()
    
    1170
    +    #
    
    1171
    +    # Writes the given size of the artifact to the cache's size file
    
    1172
    +    #
    
    1173
    +    # Args:
    
    1174
    +    #    size (int): The size of the artifact cache to record
    
    1175
    +    #
    
    1176
    +    def _write_cache_size(self, size):
    
    1177
    +        assert isinstance(size, int)
    
    1178
    +        size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
    
    1179
    +        with utils.save_file_atomic(size_file_path, "w") as f:
    
    1180
    +            f.write(str(size))
    
    1181
    +
    
    1182
    +    # _get_cache_volume_size()
    
    1183
    +    #
    
    1184
    +    # Get the available space and total space for the volume on
    
    1185
    +    # which the artifact cache is located.
    
    1186
    +    #
    
    1187
    +    # Returns:
    
    1188
    +    #    (int): The total number of bytes on the volume
    
    1189
    +    #    (int): The number of available bytes on the volume
    
    1190
    +    #
    
    1191
    +    # NOTE: We use this stub to allow the test cases
    
    1192
    +    #       to override what an artifact cache thinks
    
    1193
    +    #       about it's disk size and available bytes.
    
    1194
    +    #
    
    1195
    +    def _get_cache_volume_size(self):
    
    1196
    +        return utils._get_volume_size(self.casdir)
    
    1197
    +
    
    1198
    +    # _calculate_cache_quota()
    
    1199
    +    #
    
    1200
    +    # Calculates and sets the cache quota and lower threshold based on the
    
    1201
    +    # quota set in Context.
    
    1202
    +    # It checks that the quota is both a valid _expression_, and that there is
    
    1203
    +    # enough disk space to satisfy that quota
    
    1204
    +    #
    
    1205
    +    def _calculate_cache_quota(self):
    
    1206
    +        # Headroom intended to give BuildStream a bit of leeway.
    
    1207
    +        # This acts as the minimum size of cache_quota and also
    
    1208
    +        # is taken from the user requested cache_quota.
    
    1209
    +        #
    
    1210
    +        if 'BST_TEST_SUITE' in os.environ:
    
    1211
    +            self._cache_quota_headroom = 0
    
    1212
    +        else:
    
    1213
    +            self._cache_quota_headroom = 2e9
    
    1214
    +
    
    1215
    +        total_size, available_space = self._get_cache_volume_size()
    
    1216
    +        cache_size = self.get_cache_size()
    
    1217
    +        self.available_space = available_space
    
    1218
    +
    
    1219
    +        # Ensure system has enough storage for the cache_quota
    
    1220
    +        #
    
    1221
    +        # If cache_quota is none, set it to the maximum it could possibly be.
    
    1222
    +        #
    
    1223
    +        # Also check that cache_quota is at least as large as our headroom.
    
    1224
    +        #
    
    1225
    +        cache_quota = self._config_cache_quota
    
    1226
    +        if cache_quota is None:  # Infinity, set to max system storage
    
    1227
    +            cache_quota = cache_size + available_space
    
    1228
    +        if cache_quota < self._cache_quota_headroom:  # Check minimum
    
    1229
    +            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    1230
    +                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
    
    1231
    +                            "BuildStream requires a minimum cache quota of 2G.")
    
    1232
    +        elif cache_quota > total_size:
    
    1233
    +            # A quota greater than the total disk size is certianly an error
    
    1234
    +            raise CASCacheError("Your system does not have enough available " +
    
    1235
    +                                "space to support the cache quota specified.",
    
    1236
    +                                detail=("You have specified a quota of {quota} total disk space.\n" +
    
    1237
    +                                        "The filesystem containing {local_cache_path} only " +
    
    1238
    +                                        "has {total_size} total disk space.")
    
    1239
    +                                .format(
    
    1240
    +                                    quota=self._config_cache_quota,
    
    1241
    +                                    local_cache_path=self.casdir,
    
    1242
    +                                    total_size=utils._pretty_size(total_size)),
    
    1243
    +                                reason='insufficient-storage-for-quota')
    
    1244
    +        elif cache_quota > cache_size + available_space:
    
    1245
    +            # FIXME: we want a message to be printed out here but message
    
    1246
    +            # depends on context and cas server can't have context currently
    
    1247
    +            self.cache_quota_exceeded = True
    
    1248
    +
    
    1249
    +        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
    
    1250
    +        # cache_quota to try and avoid exceptions.
    
    1251
    +        #
    
    1252
    +        # Of course, we might still end up running out during a build
    
    1253
    +        # if we end up writing more than 2G, but hey, this stuff is
    
    1254
    +        # already really fuzzy.
    
    1255
    +        #
    
    1256
    +        self._cache_quota_original = cache_quota
    
    1257
    +        self._cache_quota = cache_quota - self._cache_quota_headroom
    
    1258
    +        self._cache_lower_threshold = self._cache_quota / 2
    
    1259
    +
    
    1018 1260
     
    
    1019 1261
     def _grouper(iterable, n):
    
    1020 1262
         while True:
    

  • buildstream/_context.py
    ... ... @@ -30,8 +30,8 @@ from . import _yaml
    30 30
     from ._exceptions import LoadError, LoadErrorReason, BstError
    
    31 31
     from ._message import Message, MessageType
    
    32 32
     from ._profile import Topics, profile_start, profile_end
    
    33
    -from ._artifactcache import ArtifactCache, ArtifactCacheUsage
    
    34
    -from ._cas import CASCache
    
    33
    +from ._artifactcache import ArtifactCache
    
    34
    +from ._cas import CASCache, CASCacheUsage
    
    35 35
     from ._workspaces import Workspaces, WorkspaceProjectCache
    
    36 36
     from .plugin import _plugin_lookup
    
    37 37
     from .sandbox import SandboxRemote
    
    ... ... @@ -237,7 +237,16 @@ class Context():
    237 237
             cache = _yaml.node_get(defaults, Mapping, 'cache')
    
    238 238
             _yaml.node_validate(cache, ['quota', 'pull-buildtrees'])
    
    239 239
     
    
    240
    -        self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
    
    240
    +        config_cache_quota = _yaml.node_get(cache, str, 'quota')
    
    241
    +        try:
    
    242
    +            self.config_cache_quota = utils._parse_size(config_cache_quota,
    
    243
    +                                                        self.casdir)
    
    244
    +        except utils.UtilError as e:
    
    245
    +            print(config_cache_quota)
    
    246
    +            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    247
    +                            "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
    
    248
    +                            "\nValid values are, for example: 800M 10G 1T 50%\n"
    
    249
    +                            .format(str(e))) from e
    
    241 250
     
    
    242 251
             # Load artifact share configuration
    
    243 252
             self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
    
    ... ... @@ -315,15 +324,15 @@ class Context():
    315 324
     
    
    316 325
             return self._artifactcache
    
    317 326
     
    
    318
    -    # get_artifact_cache_usage()
    
    327
    +    # get_cache_usage()
    
    319 328
         #
    
    320 329
         # Fetches the current usage of the artifact cache
    
    321 330
         #
    
    322 331
         # Returns:
    
    323
    -    #     (ArtifactCacheUsage): The current status
    
    332
    +    #     (CASCacheUsage): The current status
    
    324 333
         #
    
    325
    -    def get_artifact_cache_usage(self):
    
    326
    -        return ArtifactCacheUsage(self.artifactcache)
    
    334
    +    def get_cache_usage(self):
    
    335
    +        return CASCacheUsage(self.get_cascache())
    
    327 336
     
    
    328 337
         # add_project():
    
    329 338
         #
    
    ... ... @@ -690,7 +699,7 @@ class Context():
    690 699
     
    
    691 700
         def get_cascache(self):
    
    692 701
             if self._cascache is None:
    
    693
    -            self._cascache = CASCache(self.rootcachedir)
    
    702
    +            self._cascache = CASCache(self.rootcachedir, self.config_cache_quota)
    
    694 703
             return self._cascache
    
    695 704
     
    
    696 705
     
    

  • buildstream/_frontend/cli.py
    ... ... @@ -11,11 +11,11 @@ from .._exceptions import BstError, LoadError, AppError
    11 11
     from .._versions import BST_FORMAT_VERSION
    
    12 12
     from .complete import main_bashcomplete, complete_path, CompleteUnhandled
    
    13 13
     
    
    14
    -
    
    15 14
     ##################################################################
    
    16 15
     #            Override of click's main entry point                #
    
    17 16
     ##################################################################
    
    18 17
     
    
    18
    +
    
    19 19
     # search_command()
    
    20 20
     #
    
    21 21
     # Helper function to get a command and context object
    

  • buildstream/_frontend/status.py
    ... ... @@ -404,7 +404,7 @@ class _StatusHeader():
    404 404
             #
    
    405 405
             #  ~~~~~~ cache: 69% ~~~~~~
    
    406 406
             #
    
    407
    -        usage = self._context.get_artifact_cache_usage()
    
    407
    +        usage = self._context.get_cache_usage()
    
    408 408
             usage_percent = '{}%'.format(usage.used_percent)
    
    409 409
     
    
    410 410
             size = 21
    

  • buildstream/_frontend/widget.py
    ... ... @@ -452,7 +452,7 @@ class LogLine(Widget):
    452 452
             values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
    
    453 453
             values["Project"] = "{} ({})".format(project.name, project.directory)
    
    454 454
             values["Targets"] = ", ".join([t.name for t in stream.targets])
    
    455
    -        values["Cache Usage"] = "{}".format(context.get_artifact_cache_usage())
    
    455
    +        values["Cache Usage"] = "{}".format(context.get_cache_usage())
    
    456 456
             text += self._format_values(values)
    
    457 457
     
    
    458 458
             # User configurations
    

  • buildstream/_scheduler/jobs/cachesizejob.py
    ... ... @@ -25,14 +25,14 @@ class CacheSizeJob(Job):
    25 25
             self._complete_cb = complete_cb
    
    26 26
     
    
    27 27
             context = self._scheduler.context
    
    28
    -        self._artifacts = context.artifactcache
    
    28
    +        self._cas = context.get_cascache()
    
    29 29
     
    
    30 30
         def child_process(self):
    
    31
    -        return self._artifacts.compute_cache_size()
    
    31
    +        return self._cas.compute_cache_size()
    
    32 32
     
    
    33 33
         def parent_complete(self, status, result):
    
    34 34
             if status == JobStatus.OK:
    
    35
    -            self._artifacts.set_cache_size(result)
    
    35
    +            self._cas.set_cache_size(result)
    
    36 36
     
    
    37 37
             if self._complete_cb:
    
    38 38
                 self._complete_cb(status, result)
    

  • buildstream/_scheduler/jobs/cleanupjob.py
    ... ... @@ -25,27 +25,27 @@ class CleanupJob(Job):
    25 25
             self._complete_cb = complete_cb
    
    26 26
     
    
    27 27
             context = self._scheduler.context
    
    28
    +        self._cas = context.get_cascache()
    
    28 29
             self._artifacts = context.artifactcache
    
    29 30
     
    
    30 31
         def child_process(self):
    
    31 32
             def progress():
    
    32 33
                 self.send_message('update-cache-size',
    
    33
    -                              self._artifacts.get_cache_size())
    
    34
    +                              self._cas.get_cache_size())
    
    34 35
             return self._artifacts.clean(progress)
    
    35 36
     
    
    36 37
         def handle_message(self, message_type, message):
    
    37
    -
    
    38 38
             # Update the cache size in the main process as we go,
    
    39 39
             # this provides better feedback in the UI.
    
    40 40
             if message_type == 'update-cache-size':
    
    41
    -            self._artifacts.set_cache_size(message)
    
    41
    +            self._cas.set_cache_size(message)
    
    42 42
                 return True
    
    43 43
     
    
    44 44
             return False
    
    45 45
     
    
    46 46
         def parent_complete(self, status, result):
    
    47 47
             if status == JobStatus.OK:
    
    48
    -            self._artifacts.set_cache_size(result)
    
    48
    +            self._cas.set_cache_size(result)
    
    49 49
     
    
    50 50
             if self._complete_cb:
    
    51 51
                 self._complete_cb(status, result)

  • buildstream/_scheduler/jobs/job.py
    ... ... @@ -381,7 +381,6 @@ class Job():
    381 381
         #    queue (multiprocessing.Queue): The message queue for IPC
    
    382 382
         #
    
    383 383
         def _child_action(self, queue):
    
    384
    -
    
    385 384
             # This avoids some SIGTSTP signals from grandchildren
    
    386 385
             # getting propagated up to the master process
    
    387 386
             os.setsid()
    

  • buildstream/utils.py
    ... ... @@ -43,6 +43,7 @@ from . import _signals
    43 43
     from ._exceptions import BstError, ErrorDomain
    
    44 44
     from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    
    45 45
     
    
    46
    +
    
    46 47
     # The magic number for timestamps: 2011-11-11 11:11:11
    
    47 48
     _magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11])
    
    48 49
     
    

  • tests/artifactcache/cache_size.py
    ... ... @@ -50,14 +50,15 @@ def test_cache_size_write(cli, tmpdir):
    50 50
     
    
    51 51
         # Artifact cache must be in a known place
    
    52 52
         artifactdir = os.path.join(project_dir, "artifacts")
    
    53
    -    cli.configure({"artifactdir": artifactdir})
    
    53
    +    casdir = os.path.join(project_dir, "cas")
    
    54
    +    cli.configure({"rootcachedir": project_dir})
    
    54 55
     
    
    55 56
         # Build, to populate the cache
    
    56 57
         res = cli.run(project=project_dir, args=["build", "test.bst"])
    
    57 58
         res.assert_success()
    
    58 59
     
    
    59 60
         # Inspect the artifact cache
    
    60
    -    sizefile = os.path.join(artifactdir, CACHE_SIZE_FILE)
    
    61
    +    sizefile = os.path.join(casdir, CACHE_SIZE_FILE)
    
    61 62
         assert os.path.isfile(sizefile)
    
    62 63
         with open(sizefile, "r") as f:
    
    63 64
             size_data = f.read()
    
    ... ... @@ -80,11 +81,11 @@ def test_quota_over_1024T(cli, tmpdir):
    80 81
         _yaml.dump({'name': 'main'}, str(project.join("project.conf")))
    
    81 82
     
    
    82 83
         volume_space_patch = mock.patch(
    
    83
    -        "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
    
    84
    +        "buildstream._cas.CASCache._get_cache_volume_size",
    
    84 85
             autospec=True,
    
    85 86
             return_value=(1025 * TiB, 1025 * TiB)
    
    86 87
         )
    
    87 88
     
    
    88 89
         with volume_space_patch:
    
    89 90
             result = cli.run(project, args=["build", "file.bst"])
    
    90
    -        result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')
    91
    +        result.assert_main_error(ErrorDomain.CAS, 'insufficient-storage-for-quota')

  • tests/artifactcache/expiry.py
    ... ... @@ -340,11 +340,11 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
    340 340
         ("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
    
    341 341
     
    
    342 342
         # Not enough space on disk even if you cleaned up
    
    343
    -    ("11K", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota'),
    
    343
    +    ("11K", ErrorDomain.CAS, 'insufficient-storage-for-quota'),
    
    344 344
     
    
    345 345
         # Not enough space for these caches
    
    346
    -    ("7K", 'warning', 'Your system does not have enough available'),
    
    347
    -    ("70%", 'warning', 'Your system does not have enough available')
    
    346
    +    ("7K", 'success', None),
    
    347
    +    ("70%", 'success', None)
    
    348 348
     ])
    
    349 349
     @pytest.mark.datafiles(DATA_DIR)
    
    350 350
     def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reason):
    
    ... ... @@ -354,7 +354,7 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
    354 354
         cli.configure({
    
    355 355
             'cache': {
    
    356 356
                 'quota': quota,
    
    357
    -        }
    
    357
    +        },
    
    358 358
         })
    
    359 359
     
    
    360 360
         # We patch how we get space information
    
    ... ... @@ -372,13 +372,13 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
    372 372
             total_space = 10000
    
    373 373
     
    
    374 374
         volume_space_patch = mock.patch(
    
    375
    -        "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
    
    375
    +        "buildstream.utils._get_volume_size",
    
    376 376
             autospec=True,
    
    377 377
             return_value=(total_space, free_space),
    
    378 378
         )
    
    379 379
     
    
    380 380
         cache_size_patch = mock.patch(
    
    381
    -        "buildstream._artifactcache.ArtifactCache.get_cache_size",
    
    381
    +        "buildstream._cas.CASCache.get_cache_size",
    
    382 382
             autospec=True,
    
    383 383
             return_value=0,
    
    384 384
         )
    

  • tests/testutils/artifactshare.py
    ... ... @@ -46,7 +46,6 @@ class ArtifactShare():
    46 46
             # in tests as a remote artifact push/pull configuration
    
    47 47
             #
    
    48 48
             self.repodir = os.path.join(self.directory, 'repo')
    
    49
    -
    
    50 49
             os.makedirs(self.repodir)
    
    51 50
     
    
    52 51
             self.cas = CASCache(self.repodir)
    
    ... ... @@ -171,7 +170,9 @@ class ArtifactShare():
    171 170
     
    
    172 171
         def _mock_statvfs(self, path):
    
    173 172
             repo_size = 0
    
    174
    -        for root, _, files in os.walk(self.repodir):
    
    173
    +        for root, dirs, files in os.walk(self.repodir):
    
    174
    +            for dirname in dirs:
    
    175
    +                repo_size += os.path.getsize(os.path.join(root, dirname))
    
    175 176
                 for filename in files:
    
    176 177
                     repo_size += os.path.getsize(os.path.join(root, filename))
    
    177 178
     
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]