[Notes] [Git][BuildStream/buildstream][relative_workspaces] 3 commits:

Phillip Smyth pushed to branch relative_workspaces at BuildStream / buildstream

Commits:

f5c8ff61

by Josh Smith at 2018-07-27T14:10:45Z

_context.py: Cache size is now restricted to available disk space

This address issue #491.

When attempting to run buildstream with a configuration specifying
a cache quota larger than your available disk space, buildstream
will alert the user and exit.

Note:
This takes into consideration your current cache usage and
therefore restricts the overall size of your artifact cache folder.

32ddb544

by Qinusty at 2018-07-27T14:57:30Z

Merge branch 'Qinusty/491' into 'master'

Cache quota is now restricted to available disk space

See merge request BuildStream/buildstream!563

30279ff8
by Phillip Smyth at 2018-07-27T15:02:37Z
```
Implementing relative workspaces
```

Changes:

buildstream/_context.py

@@ -197,29 +197,55 @@ class Context():
                              "\nValid values are, for example: 800M 10G 1T 50%\n"
                              .format(str(e))) from e
 -        # If we are asked not to set a quota, we set it to the maximum
 -        # disk space available minus a headroom of 2GB, such that we
 -        # at least try to avoid raising Exceptions.
 +        # Headroom intended to give BuildStream a bit of leeway.
 +        # This acts as the minimum size of cache_quota and also
 +        # is taken from the user requested cache_quota.
+         #
 -        # Of course, we might still end up running out during a build
 -        # if we end up writing more than 2G, but hey, this stuff is
 -        # already really fuzzy.
 -        #
 -        if cache_quota is None:
 -            stat = os.statvfs(artifactdir_volume)
 -            # Again, the artifact directory may not yet have been
 -            # created
 -            if not os.path.exists(self.artifactdir):
 -                cache_size = 0
 -            else:
 -                cache_size = utils._get_dir_size(self.artifactdir)
 -            cache_quota = cache_size + stat.f_bsize * stat.f_bavail
+-
          if 'BST_TEST_SUITE' in os.environ:
              headroom = 0
          else:
              headroom = 2e9
 +        stat = os.statvfs(artifactdir_volume)
 +        available_space = (stat.f_bsize * stat.f_bavail)
++
 +        # Again, the artifact directory may not yet have been created yet
 +        #
 +        if not os.path.exists(self.artifactdir):
 +            cache_size = 0
 +        else:
 +            cache_size = utils._get_dir_size(self.artifactdir)
++
 +        # Ensure system has enough storage for the cache_quota
 +        #
 +        # If cache_quota is none, set it to the maximum it could possibly be.
 +        #
 +        # Also check that cache_quota is atleast as large as our headroom.
 +        #
 +        if cache_quota is None:  # Infinity, set to max system storage
 +            cache_quota = cache_size + available_space
 +        if cache_quota < headroom:  # Check minimum
 +            raise LoadError(LoadErrorReason.INVALID_DATA,
 +                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
 +                            "BuildStream requires a minimum cache quota of 2G.")
 +        elif cache_quota > cache_size + available_space:  # Check maximum
 +            raise LoadError(LoadErrorReason.INVALID_DATA,
 +                            ("Your system does not have enough available " +
 +                             "space to support the cache quota specified.\n" +
 +                             "You currently have:\n" +
 +                             "- {used} of cache in use at {local_cache_path}\n" +
 +                             "- {available} of available system storage").format(
 +                                 used=utils._pretty_size(cache_size),
 +                                 local_cache_path=self.artifactdir,
 +                                 available=utils._pretty_size(available_space)))
++
 +        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
 +        # cache_quota to try and avoid exceptions.
 +        #
 +        # Of course, we might still end up running out during a build
 +        # if we end up writing more than 2G, but hey, this stuff is
 +        # already really fuzzy.
 +        #
          self.cache_quota = cache_quota - headroom
          self.cache_lower_threshold = self.cache_quota / 2

buildstream/_frontend/widget.py

@@ -415,7 +415,7 @@ class LogLine(Widget):
              if "%{workspace-dirs" in format_:
                  workspace = element._get_workspace()
                  if workspace is not None:
 -                    path = workspace.path.replace(os.getenv('HOME', '/root'), '~')
 +                    path = workspace.get_absolute_path().replace(os.getenv('HOME', '/root'), '~')
                      line = p.fmt_subst(line, 'workspace-dirs', "Workspace: {}".format(path))
                  else:
                      line = p.fmt_subst(

buildstream/_stream.py

@@ -478,7 +478,7 @@ class Stream():
          workspace = workspaces.get_workspace(target._get_full_name())
          if workspace and not force:
              raise StreamError("Workspace '{}' is already defined at: {}"
 -                              .format(target.name, workspace.path))
 +                              .format(target.name, workspace.get_absolute_path()))
          # If we're going to checkout, we need at least a fetch,
          # if we were asked to track first, we're going to fetch anyway.
@@ -500,14 +500,14 @@ class Stream():
              workspaces.save_config()
              shutil.rmtree(directory)
          try:
 -            os.makedirs(directory, exist_ok=True)
 +            os.makedirs(workdir, exist_ok=True)
          except OSError as e:
              raise StreamError("Failed to create workspace directory: {}".format(e)) from e
          workspaces.create_workspace(target._get_full_name(), workdir)
          if not no_checkout:
 -            with target.timed_activity("Staging sources to {}".format(directory)):
 +            with target.timed_activity("Staging sources to {}".format(workdir)):
                  target._open_workspace()
          workspaces.save_config()
@@ -528,12 +528,12 @@ class Stream():
          # Remove workspace directory if prompted
          if remove_dir:
              with self._context.timed_activity("Removing workspace directory {}"
 -                                              .format(workspace.path)):
 +                                              .format(workspace.get_absolute_path())):
                  try:
 -                    shutil.rmtree(workspace.path)
 +                    shutil.rmtree(workspace.get_absolute_path())
                  except OSError as e:
                      raise StreamError("Could not remove  '{}': {}"
 -                                      .format(workspace.path, e)) from e
 +                                      .format(workspace.get_absolute_path(), e)) from e
          # Delete the workspace and save the configuration
          workspaces.delete_workspace(element_name)
@@ -576,28 +576,30 @@ class Stream():
          for element in elements:
              workspace = workspaces.get_workspace(element._get_full_name())
+-
 +            workspace_path = workspace.get_absolute_path()
              if soft:
                  workspace.prepared = False
                  self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
 -                              .format(element.name, workspace.path))
 +                              .format(element.name, workspace_path))
                  continue
              with element.timed_activity("Removing workspace directory {}"
 -                                        .format(workspace.path)):
 +                                        .format(workspace_path)):
                  try:
 -                    shutil.rmtree(workspace.path)
 +                    shutil.rmtree(workspace_path)
                  except OSError as e:
                      raise StreamError("Could not remove  '{}': {}"
 -                                      .format(workspace.path, e)) from e
 +                                      .format(workspace_path, e)) from e
              workspaces.delete_workspace(element._get_full_name())
 -            workspaces.create_workspace(element._get_full_name(), workspace.path)
 +            workspaces.create_workspace(element._get_full_name(), workspace_path)
 -            with element.timed_activity("Staging sources to {}".format(workspace.path)):
 +            with element.timed_activity("Staging sources to {}".format(workspace_path)):
                  element._open_workspace()
 -            self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path))
 +            self._message(MessageType.INFO,
 +                          "Reset workspace for {} at: {}".format(element.name,
 +                                                                 workspace_path))
          workspaces.save_config()
@@ -634,7 +636,7 @@ class Stream():
          for element_name, workspace_ in self._context.get_workspaces().list():
              workspace_detail = {
                  'element': element_name,
 -                'directory': workspace_.path,
 +                'directory': workspace_.get_absolute_path(),
+             }
              workspaces.append(workspace_detail)

buildstream/_workspaces.py

@@ -29,7 +29,7 @@ BST_WORKSPACE_FORMAT_VERSION = 3
  # Hold on to a list of members which get serialized
  _WORKSPACE_MEMBERS = [
      'prepared',
 -    'path',
 +    '_path',
      'last_successful',
      'running_files'
+ ]
@@ -56,7 +56,7 @@ class Workspace():
      def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
          self.prepared = prepared
          self.last_successful = last_successful
 -        self.path = path
 +        self._path = path
          self.running_files = running_files if running_files is not None else {}
          self._toplevel_project = toplevel_project
@@ -70,6 +70,7 @@ class Workspace():
      #     (dict) A dict representation of the workspace
+     #
      def to_dict(self):
 +        # Strip the leading underscore from _path
          return {key: val for key, val in self.__dict__.items()
                  if key in _WORKSPACE_MEMBERS and val is not None}
@@ -133,7 +134,7 @@ class Workspace():
          if os.path.isdir(fullpath):
              utils.copy_files(fullpath, directory)
          else:
 -            destfile = os.path.join(directory, os.path.basename(self.path))
 +            destfile = os.path.join(directory, os.path.basename(self.get_absolute_path()))
              utils.safe_copy(fullpath, destfile)
      # add_running_files()
@@ -189,7 +190,7 @@ class Workspace():
                  filelist = utils.list_relative_paths(fullpath)
                  filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
              else:
 -                filelist = [(self.path, fullpath)]
 +                filelist = [(self.get_absolute_path(), fullpath)]
              self._key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
@@ -200,7 +201,7 @@ class Workspace():
      # Returns: The absolute path of the element's workspace.
+     #
      def get_absolute_path(self):
 -        return os.path.join(self._toplevel_project.directory, self.path)
 +        return os.path.join(self._toplevel_project.directory, self._path)
  # Workspaces()
@@ -236,6 +237,9 @@ class Workspaces():
      #    path (str) - The path in which the workspace should be kept
+     #
      def create_workspace(self, element_name, path):
 +        if path.startswith(self._toplevel_project.directory):
 +            path = os.path.relpath(path, self._toplevel_project.directory)
++
          self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
          return self._workspaces[element_name]

buildstream/element.py

@@ -1324,7 +1324,7 @@ class Element(Plugin):
                  # If mount_workspaces is set and we're doing incremental builds,
                  # the workspace is already mounted into the sandbox.
                  if not (mount_workspaces and self.__can_build_incrementally()):
 -                    with self.timed_activity("Staging local files at {}".format(workspace.path)):
 +                    with self.timed_activity("Staging local files at {}".format(workspace.get_absolute_path())):
                          workspace.stage(directory)
              else:
                  # No workspace, stage directly
@@ -1484,7 +1484,7 @@ class Element(Plugin):
                          sandbox_path = os.path.join(sandbox_root,
                                                      self.__staged_sources_directory.lstrip(os.sep))
                          try:
 -                            utils.copy_files(workspace.path, sandbox_path)
 +                            utils.copy_files(workspace.get_absolute_path(), sandbox_path)
                          except UtilError as e:
                              self.warn("Failed to preserve workspace state for failed build sysroot: {}"
                                        .format(e))
@@ -1786,7 +1786,7 @@ class Element(Plugin):
                  source._init_workspace(temp)
              # Now hardlink the files into the workspace target.
 -            utils.link_files(temp, workspace.path)
 +            utils.link_files(temp, workspace.get_absolute_path())
      # _get_workspace():
+     #

buildstream/utils.py

@@ -612,6 +612,27 @@ def _parse_size(size, volume):
      return int(num) * 1024**units.index(unit)
 +# _pretty_size()
 +#
 +# Converts a number of bytes into a string representation in KB, MB, GB, TB
 +# represented as K, M, G, T etc.
 +#
 +# Args:
 +#   size (int): The size to convert in bytes.
 +#   dec_places (int): The number of decimal places to output to.
 +#
 +# Returns:
 +#   (str): The string representation of the number of bytes in the largest
 +def _pretty_size(size, dec_places=0):
 +    psize = size
 +    unit = 'B'
 +    for unit in ('B', 'K', 'M', 'G', 'T'):
 +        if psize < 1024:
 +            break
 +        else:
 +            psize /= 1024
 +    return "{size:g}{unit}".format(size=round(psize, dec_places), unit=unit)
++
  # A sentinel to be used as a default argument for functions that need
  # to distinguish between a kwarg set to None and an unset kwarg.
  _sentinel = object()

tests/frontend/workspace.py

@@ -18,7 +18,10 @@ DATA_DIR = os.path.join(
+ )
 -def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
 +def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', rerun=False, new_dir=None):
 +    if os.path.isdir(os.path.join(str(tmpdir), 'repo')):
 +        shutil.rmtree(os.path.join(str(tmpdir), 'repo'))
++
      project = os.path.join(datafiles.dirname, datafiles.basename)
      bin_files_path = os.path.join(project, 'files', 'bin-files')
      element_path = os.path.join(project, 'elements')
@@ -45,11 +48,12 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
                              element_name))
      # Assert that there is no reference, a track & fetch is needed
 -    state = cli.get_element_state(project, element_name)
 -    if track:
 -        assert state == 'no reference'
 -    else:
 -        assert state == 'fetch needed'
 +    if not rerun:
 +        state = cli.get_element_state(project, element_name)
 +        if track:
 +            assert state == 'no reference'
 +        else:
 +            assert state == 'fetch needed'
      # Now open the workspace, this should have the effect of automatically
      # tracking & fetching the source from the repo.
@@ -59,15 +63,19 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
      args.extend([element_name, workspace])
      result = cli.run(project=project, args=args)
 -    result.assert_success()
 -    # Assert that we are now buildable because the source is
 -    # now cached.
 -    assert cli.get_element_state(project, element_name) == 'buildable'
 +    if not rerun:
 +        result.assert_success()
 -    # Check that the executable hello file is found in the workspace
 -    filename = os.path.join(workspace, 'usr', 'bin', 'hello')
 -    assert os.path.exists(filename)
 +        # Assert that we are now buildable because the source is
 +        # now cached.
 +        assert cli.get_element_state(project, element_name) == 'buildable'
++
 +        # Check that the executable hello file is found in the workspace
 +        filename = os.path.join(workspace, 'usr', 'bin', 'hello')
 +        assert os.path.exists(filename)
 +    else:
 +        result.assert_main_error(ErrorDomain.STREAM, None)
      return (element_name, project, workspace)
@@ -78,6 +86,43 @@ def test_open(cli, tmpdir, datafiles, kind):
      open_workspace(cli, tmpdir, datafiles, kind, False)
 +@pytest.mark.datafiles(DATA_DIR)
 +@pytest.mark.parametrize("kind", repo_kinds)
 +def test_open_after_move_interally(cli, tmpdir, datafiles, kind):
 +    open_workspace(cli, tmpdir, datafiles, kind, False)
 +    workspace = os.path.join(str(tmpdir), 'workspace')
 +    shutil.move(workspace, os.path.join(str(tmpdir), 'workspace2'))
 +    open_workspace(cli, tmpdir, datafiles, kind, False, '', True)
 +    filename = os.path.join(workspace + '2', 'usr', 'bin', 'hello')
 +    assert os.path.exists(filename)
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +@pytest.mark.parametrize("kind", repo_kinds)
 +def test_open_after_move_externally(cli, tmpdir, datafiles, kind):
 +    open_workspace(cli, tmpdir, datafiles, kind, False)
 +    workspace = os.path.join(str(tmpdir), 'workspace')
 +    tmp_dir = os.path.join(str(tmpdir), '2')
 +    shutil.move(workspace, os.path.join(tmp_dir, 'workspace'))
 +    open_workspace(cli, tmpdir, datafiles, kind, False, '', True, tmp_dir)
 +    filename = os.path.join(os.path.join(tmp_dir, 'workspace'), 'usr', 'bin', 'hello')
 +    assert os.path.exists(filename)
 +    shutil.rmtree(tmp_dir)
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +@pytest.mark.parametrize("kind", repo_kinds)
 +def test_open_after_move_project(cli, tmpdir, datafiles, kind):
 +    open_workspace(cli, tmpdir, datafiles, kind, False)
 +    workspace = os.path.join(str(tmpdir), 'workspace')
 +    tmp_dir = os.path.join(str(tmpdir), '..', 'new_dir')
 +    shutil.move(workspace, os.path.join(tmp_dir, 'workspace'))
 +    open_workspace(cli, tmpdir, datafiles, kind, False, '', True, tmp_dir)
 +    filename = os.path.join(os.path.join(tmp_dir, 'workspace'), 'usr', 'bin', 'hello')
 +    assert os.path.exists(filename)
 +    shutil.rmtree(tmp_dir)
++
++
  @pytest.mark.datafiles(DATA_DIR)
  def test_open_bzr_customize(cli, tmpdir, datafiles):
      element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "bzr", False)

tests/integration/shell.py

@@ -330,6 +330,7 @@ def test_sysroot_workspace_visible(cli, tmpdir, datafiles):
      # Obtain a copy of the hello.c content from the workspace
+     #
      workspace_hello_path = os.path.join(cli.directory, 'workspace', 'hello.c')
++
      assert os.path.exists(workspace_hello_path)
      with open(workspace_hello_path, 'r') as f:
          workspace_hello = f.read()

[Notes] [Git][BuildStream/buildstream][relative_workspaces] 3 commits: _context.py: Cache size is now restricted to available disk space

Phillip Smyth pushed to branch relative_workspaces at BuildStream / buildstream

Commits:

8 changed files:

Changes: