[Notes] [Git][BuildStream/buildstream][Qinusty/491]

Josh pushed to branch Qinusty/491 at BuildStream / buildstream

Commits:

f2ccf876

by Josh Smith at 2018-07-25T15:51:31Z

_context.py: Cache size is now restricted to available disk space

This address issue #491.

When attempting to run buildstream with a configuration specifying
a cache quota larger than your available disk space, buildstream
will alert the user and exit.

Note:
This takes into consideration your current cache usage and
therefore restricts the overall size of your artifact cache folder.

2 changed files:

buildstream/_context.py
buildstream/utils.py

Changes:

buildstream/_context.py

@@ -197,30 +197,54 @@ class Context():
                              "\nValid values are, for example: 800M 10G 1T 50%\n"
                              .format(str(e))) from e
 -        # If we are asked not to set a quota, we set it to the maximum
 -        # disk space available minus a headroom of 2GB, such that we
 -        # at least try to avoid raising Exceptions.
 +        if 'BST_TEST_SUITE' in os.environ:
 +            headroom = 0
 +        else:
 +            headroom = 2e9
++
 +        stat = os.statvfs(artifactdir_volume)
 +        available_space = (stat.f_bsize * stat.f_bavail)
 +        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
 +        # bst_available_space to try and avoid exceptions.
+         #
          # Of course, we might still end up running out during a build
          # if we end up writing more than 2G, but hey, this stuff is
          # already really fuzzy.
+         #
 -        if cache_quota is None:
 -            stat = os.statvfs(artifactdir_volume)
 -            # Again, the artifact directory may not yet have been
 -            # created
 -            if not os.path.exists(self.artifactdir):
 -                cache_size = 0
 -            else:
 -                cache_size = utils._get_dir_size(self.artifactdir)
 -            cache_quota = cache_size + stat.f_bsize * stat.f_bavail
 +        bst_available_space = available_space - headroom
 -        if 'BST_TEST_SUITE' in os.environ:
 -            headroom = 0
 +        # Again, the artifact directory may not yet have been created yet
 +        #
 +        if not os.path.exists(self.artifactdir):
 +            cache_size = 0
          else:
 -            headroom = 2e9
 +            cache_size = utils._get_dir_size(self.artifactdir)
 -        self.cache_quota = cache_quota - headroom
 +        # Ensure system has enough storage for the cache_quota
 +        #
 +        # If cache_quota is none, set it to the maximum it could possibly be.
 +        #
 +        if cache_quota is None:  # Infinity, set to max system storage
 +            cache_quota = cache_size + available_space
 +        elif cache_quota < headroom:
 +            raise LoadError(LoadErrorReason.INVALID_DATA,
 +                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
 +                            "BuildStream requires a minimum cache quota of 2GB.")
 +        elif cache_size + cache_quota > bst_available_space:
 +            raise LoadError(LoadErrorReason.INVALID_DATA,
 +                            ("Your system does not have enough available " +
 +                             "space to support the cache quota specified.\n" +
 +                             "You currently have:\n" +
 +                             "- {used} of cache in use at {local_cache_path}\n" +
 +                             "- {available} of available system storage.\n" +
 +                             "Note: BuildStream requires the cache quota be at least {headroom} " +
 +                             "smaller than your available system storage").format(
 +                                 used=utils._pretty_size(cache_size),
 +                                 local_cache_path=self.artifactdir,
 +                                 available=utils._pretty_size(available_space),
 +                                 headroom=utils._pretty_size(headroom)))
++
 +        self.cache_quota = cache_quota
          self.cache_lower_threshold = self.cache_quota / 2
          # Load artifact share configuration

buildstream/utils.py

@@ -608,6 +608,27 @@ def _parse_size(size, volume):
      return int(num) * 1024**units.index(unit)
 +# _pretty_size()
 +#
 +# Converts a number of bytes into a string representation in KB, MB, GB, TB
 +# represented as K, M, G, T etc.
 +#
 +# Args:
 +#   size (int): The size to convert in bytes.
 +#   dec_places (int): The number of decimal places to output to.
 +#
 +# Returns:
 +#   (str): The string representation of the number of bytes in the largest
 +def _pretty_size(size, dec_places=0):
 +    psize = size
 +    unit = 'B'
 +    for unit in ('B', 'K', 'M', 'G', 'T'):
 +        if psize < 1024:
 +            break
 +        else:
 +            psize /= 1024
 +    return "{size:g}{unit}".format(size=round(psize, dec_places), unit=unit)
++
  # A sentinel to be used as a default argument for functions that need
  # to distinguish between a kwarg set to None and an unset kwarg.
  _sentinel = object()

[Notes] [Git][BuildStream/buildstream][Qinusty/491] _context.py: Cache size is now restricted to available disk space

Josh pushed to branch Qinusty/491 at BuildStream / buildstream

Commits:

2 changed files:

Changes: