[Notes] [Git][BuildStream/buildstream][bst-1.2] 2 commits: Use ArtifactC

Valentin David pushed to branch bst-1.2 at BuildStream / buildstream

Commits:

6a7fdf12

by Jonathan Maw at 2018-08-22T13:22:27Z

Use ArtifactCache's get_cache_size when calculating the quota

It would have been more convenient to store the quota size in the
ArtifactCache, except that the scheduler needs the quota size and
doesn't have access to the ArtifactCache.

baee80b4

by Valentin David at 2018-08-22T17:28:55Z

Merge branch 'valentindavid/faster_startup-1.2' into 'bst-1.2'

Use ArtifactCache's get_cache_size when calculating the quota

See merge request BuildStream/buildstream!701

2 changed files:

buildstream/_context.py
buildstream/_frontend/app.py

Changes:

buildstream/_context.py

@@ -124,6 +124,8 @@ class Context():
          self._workspaces = None
          self._log_handle = None
          self._log_filename = None
 +        self._config_cache_quota = None
 +        self._artifactdir_volume = None
      # load()
+     #
@@ -187,67 +189,19 @@ class Context():
          while not os.path.exists(artifactdir_volume):
              artifactdir_volume = os.path.dirname(artifactdir_volume)
 +        self._artifactdir_volume = artifactdir_volume
++
          # We read and parse the cache quota as specified by the user
          cache_quota = _yaml.node_get(cache, str, 'quota', default_value='infinity')
          try:
 -            cache_quota = utils._parse_size(cache_quota, artifactdir_volume)
 +            cache_quota = utils._parse_size(cache_quota, self._artifactdir_volume)
          except utils.UtilError as e:
              raise LoadError(LoadErrorReason.INVALID_DATA,
                              "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
                              "\nValid values are, for example: 800M 10G 1T 50%\n"
                              .format(str(e))) from e
 -        # Headroom intended to give BuildStream a bit of leeway.
 -        # This acts as the minimum size of cache_quota and also
 -        # is taken from the user requested cache_quota.
 -        #
 -        if 'BST_TEST_SUITE' in os.environ:
 -            headroom = 0
 -        else:
 -            headroom = 2e9
+-
 -        stat = os.statvfs(artifactdir_volume)
 -        available_space = (stat.f_bsize * stat.f_bavail)
+-
 -        # Again, the artifact directory may not yet have been created yet
 -        #
 -        if not os.path.exists(self.artifactdir):
 -            cache_size = 0
 -        else:
 -            cache_size = utils._get_dir_size(self.artifactdir)
+-
 -        # Ensure system has enough storage for the cache_quota
 -        #
 -        # If cache_quota is none, set it to the maximum it could possibly be.
 -        #
 -        # Also check that cache_quota is atleast as large as our headroom.
 -        #
 -        if cache_quota is None:  # Infinity, set to max system storage
 -            cache_quota = cache_size + available_space
 -        if cache_quota < headroom:  # Check minimum
 -            raise LoadError(LoadErrorReason.INVALID_DATA,
 -                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
 -                            "BuildStream requires a minimum cache quota of 2G.")
 -        elif cache_quota > cache_size + available_space:  # Check maximum
 -            raise LoadError(LoadErrorReason.INVALID_DATA,
 -                            ("Your system does not have enough available " +
 -                             "space to support the cache quota specified.\n" +
 -                             "You currently have:\n" +
 -                             "- {used} of cache in use at {local_cache_path}\n" +
 -                             "- {available} of available system storage").format(
 -                                 used=utils._pretty_size(cache_size),
 -                                 local_cache_path=self.artifactdir,
 -                                 available=utils._pretty_size(available_space)))
+-
 -        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
 -        # cache_quota to try and avoid exceptions.
 -        #
 -        # Of course, we might still end up running out during a build
 -        # if we end up writing more than 2G, but hey, this stuff is
 -        # already really fuzzy.
 -        #
 -        self.cache_quota = cache_quota - headroom
 -        self.cache_lower_threshold = self.cache_quota / 2
 +        self._config_cache_quota = cache_quota
          # Load artifact share configuration
          self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
@@ -571,6 +525,53 @@ class Context():
      def get_log_filename(self):
          return self._log_filename
 +    def set_cache_quota(self, cache_size):
 +        # Headroom intended to give BuildStream a bit of leeway.
 +        # This acts as the minimum size of cache_quota and also
 +        # is taken from the user requested cache_quota.
 +        #
 +        if 'BST_TEST_SUITE' in os.environ:
 +            headroom = 0
 +        else:
 +            headroom = 2e9
++
 +        stat = os.statvfs(self._artifactdir_volume)
 +        available_space = (stat.f_bsize * stat.f_bavail)
++
 +        # Ensure system has enough storage for the cache_quota
 +        #
 +        # If cache_quota is none, set it to the maximum it could possibly be.
 +        #
 +        # Also check that cache_quota is atleast as large as our headroom.
 +        #
 +        cache_quota = self._config_cache_quota
 +        if cache_quota is None:  # Infinity, set to max system storage
 +            cache_quota = cache_size + available_space
 +        if cache_quota < headroom:  # Check minimum
 +            raise LoadError(LoadErrorReason.INVALID_DATA,
 +                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
 +                            "BuildStream requires a minimum cache quota of 2G.")
 +        elif cache_quota > cache_size + available_space:  # Check maximum
 +            raise LoadError(LoadErrorReason.INVALID_DATA,
 +                            ("Your system does not have enough available " +
 +                             "space to support the cache quota specified.\n" +
 +                             "You currently have:\n" +
 +                             "- {used} of cache in use at {local_cache_path}\n" +
 +                             "- {available} of available system storage").format(
 +                                 used=utils._pretty_size(cache_size),
 +                                 local_cache_path=self.artifactdir,
 +                                 available=utils._pretty_size(available_space)))
++
 +        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
 +        # cache_quota to try and avoid exceptions.
 +        #
 +        # Of course, we might still end up running out during a build
 +        # if we end up writing more than 2G, but hey, this stuff is
 +        # already really fuzzy.
 +        #
 +        self.cache_quota = cache_quota - headroom
 +        self.cache_lower_threshold = self.cache_quota / 2
++
      # _record_message()
+     #
      # Records the message if recording is enabled

buildstream/_frontend/app.py

@@ -202,6 +202,10 @@ class App():
          Platform.create_instance(self.context)
 +        platform = Platform.get_platform()
 +        cache_size = platform._artifact_cache.calculate_cache_size()
 +        self.context.set_cache_quota(cache_size)
++
          # Create the logger right before setting the message handler
          self.logger = LogLine(self.context,
                                self._content_profile,

[Notes] [Git][BuildStream/buildstream][bst-1.2] 2 commits: Use ArtifactCache's get_cache_size when calculating the quota

Valentin David pushed to branch bst-1.2 at BuildStream / buildstream

Commits:

2 changed files:

Changes: