[Notes] [Git][BuildStream/buildstream][jonathan/workspace-fragment-creat

Jonathan Maw pushed to branch jonathan/workspace-fragment-create at BuildStream / buildstream

Commits:

b22bc1c2

by Jonathan Maw at 2018-11-27T17:08:39Z

_project.py: Find project from workspace if outside of a project

Searching for the workspace file is preferred to searching for the
project file for the moment, because it avoids the circumstance of
the workspaces being inside a directory where a higher-level directory
may have project.conf (but may not be a buildstream project).

1e3a0c60

by Jonathan Maw at 2018-11-27T17:08:43Z

_stream.py: Opened workspaces store the project that opened them

33a911cc

by Jonathan Maw at 2018-11-27T17:08:43Z

_stream.py: workspace_close deletes the stored project reference

c0ebef05

by Jonathan Maw at 2018-11-27T17:08:43Z

_stream.py: Re-create the .bstproject when resetting the workspace

dcb13f54

by Jonathan Maw at 2018-11-27T17:08:43Z

_workspaces.py: Do not include .bstproject.yaml in the cache key

25a9c86e

by Jonathan Maw at 2018-11-27T17:08:43Z

cli: Interactively warn if the user is trying to close the workspace they're using to load the project

This involves changes in:
* _stream.py:
  * Add the helper Stream.workspace_is_required()
* userconfig.yaml:
  * Add a default value for prompt.really-workspace-close-project-inaccessible
* _context.py:
  * Load the prompt 'really-workspace-close-project-inaccessible' from
    user config.
* cli.py:
  * If buildstream is invoked interactively, prompt the user to confirm
    that they want to close the workspace they're using to load this
    project.

1e644e67

by Jonathan Maw at 2018-11-27T17:08:44Z

tests: Test bst commands from an external workspace

e4797e80

by Jonathan Maw at 2018-11-27T17:08:44Z

NEWS: Add an entry for being able to run commands from a workspace

Changes:

NEWS

@@ -70,6 +70,9 @@ buildstream 1.3.1
    o Add sandbox API for command batching and use it for build, script, and
      compose elements.
 +  o Opening a workspace now creates a .bstproject.yaml file that allows buildstream
 +    commands to be run from a workspace that is not inside a project.
++
  =================
  buildstream 1.1.5

buildstream/_context.py

@@ -31,7 +31,7 @@ from ._exceptions import LoadError, LoadErrorReason, BstError
  from ._message import Message, MessageType
  from ._profile import Topics, profile_start, profile_end
  from ._artifactcache import ArtifactCache
 -from ._workspaces import Workspaces
 +from ._workspaces import Workspaces, WorkspaceProjectCache
  from .plugin import _plugin_lookup
@@ -121,6 +121,10 @@ class Context():
          # remove a workspace directory.
          self.prompt_workspace_close_remove_dir = None
 +        # Boolean, whether we double-check with the user that they meant to
 +        # close the workspace when they're using it to access the project.
 +        self.prompt_workspace_close_project_inaccessible = None
++
          # Boolean, whether we double-check with the user that they meant to do
          # a hard reset of a workspace, potentially losing changes.
          self.prompt_workspace_reset_hard = None
@@ -139,6 +143,7 @@ class Context():
          self._projects = []
          self._project_overrides = {}
          self._workspaces = None
 +        self._workspace_project_cache = WorkspaceProjectCache()
          self._log_handle = None
          self._log_filename = None
@@ -248,12 +253,15 @@ class Context():
              defaults, Mapping, 'prompt')
          _yaml.node_validate(prompt, [
              'auto-init', 'really-workspace-close-remove-dir',
 +            'really-workspace-close-project-inaccessible',
              'really-workspace-reset-hard',
          ])
          self.prompt_auto_init = _node_get_option_str(
              prompt, 'auto-init', ['ask', 'no']) == 'ask'
          self.prompt_workspace_close_remove_dir = _node_get_option_str(
              prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask'
 +        self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
 +            prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
          self.prompt_workspace_reset_hard = _node_get_option_str(
              prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask'
@@ -310,6 +318,16 @@ class Context():
      def get_workspaces(self):
          return self._workspaces
 +    # get_workspace_project_cache():
 +    #
 +    # Return the WorkspaceProjectCache object used for this BuildStream invocation
 +    #
 +    # Returns:
 +    #    (WorkspaceProjectCache): The WorkspaceProjectCache object
 +    #
 +    def get_workspace_project_cache(self):
 +        return self._workspace_project_cache
++
      # get_overrides():
+     #
      # Fetch the override dictionary for the active project. This returns

buildstream/_frontend/cli.py

@@ -747,11 +747,18 @@ def workspace_close(app, remove_dir, all_, elements):
          elements = app.stream.redirect_element_names(elements)
 -        # Check that the workspaces in question exist
 +        # Check that the workspaces in question exist, and that it's safe to
 +        # remove them.
          nonexisting = []
          for element_name in elements:
              if not app.stream.workspace_exists(element_name):
                  nonexisting.append(element_name)
 +            if (app.stream.workspace_is_required(element_name) and app.interactive and
 +                    app.context.prompt_workspace_close_project_inaccessible):
 +                click.echo("Removing '{}' will prevent you from running buildstream commands".format(element_name))
 +                if not click.confirm('Are you sure you want to close this workspace?'):
 +                    click.echo('Aborting', err=True)
 +                    sys.exit(-1)
          if nonexisting:
              raise AppError("Workspace does not exist", detail="\n".join(nonexisting))

buildstream/_project.py

@@ -94,8 +94,10 @@ class Project():
          # The project name
          self.name = None
 -        # The project directory
 -        self.directory = self._find_project_dir(directory)
 +        self._context = context  # The invocation Context, a private member
++
 +        # The project directory, and whether the project was found from an external workspace
 +        self.directory, self._required_workspace_element = self._find_project_dir(directory)
          # Absolute path to where elements are loaded from within the project
          self.element_path = None
@@ -116,7 +118,6 @@ class Project():
+         #
          # Private Members
+         #
 -        self._context = context  # The invocation Context
          self._default_mirror = default_mirror    # The name of the preferred mirror.
@@ -370,6 +371,14 @@ class Project():
          self._load_second_pass()
 +    # required_workspace_element()
 +    #
 +    # Returns the element whose workspace is required to load this project,
 +    # if any.
 +    #
 +    def required_workspace_element(self):
 +        return self._required_workspace_element
++
      # cleanup()
+     #
      # Cleans up resources used loading elements
@@ -662,18 +671,26 @@ class Project():
      # Raises:
      #    LoadError if project.conf is not found
+     #
 +    # Returns:
 +    #    (str) - the directory that contains the project, and
 +    #    (str) - the name of the element required to find the project, or an empty string
 +    #
      def _find_project_dir(self, directory):
 -        directory = os.path.abspath(directory)
 -        while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
 -            parent_dir = os.path.dirname(directory)
 -            if directory == parent_dir:
 +        workspace_element = ""
 +        project_directory = utils._search_upward_for_file(directory, _PROJECT_CONF_FILE)
 +        if not project_directory:
 +            workspace_project_cache = self._context.get_workspace_project_cache()
 +            workspace_project = workspace_project_cache.get(directory)
 +            if workspace_project:
 +                project_directory = workspace_project.get_default_path()
 +                workspace_element = workspace_project.get_default_element()
 +            else:
                  raise LoadError(
                      LoadErrorReason.MISSING_PROJECT_CONF,
                      '{} not found in current directory or any of its parent directories'
                      .format(_PROJECT_CONF_FILE))
 -            directory = parent_dir
 -        return directory
 +        return project_directory, workspace_element
      def _load_plugin_factories(self, config, output):
          plugin_source_origins = []   # Origins of custom sources

buildstream/_stream.py

@@ -28,7 +28,7 @@ import tarfile
  from contextlib import contextmanager
  from tempfile import TemporaryDirectory
 -from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
 +from ._exceptions import StreamError, ImplError, BstError, set_last_task_error, LoadError, LoadErrorReason
  from ._message import Message, MessageType
  from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
  from ._pipeline import Pipeline, PipelineSelection
@@ -550,6 +550,8 @@ class Stream():
          # So far this function has tried to catch as many issues as possible with out making any changes
          # Now it dose the bits that can not be made atomic.
          targetGenerator = zip(elements, expanded_directories)
 +        workspace_project_cache = self._context.get_workspace_project_cache()
 +        project = self._context.get_toplevel_project()
          for target, directory in targetGenerator:
              self._message(MessageType.INFO, "Creating workspace for element {}"
                            .format(target.name))
@@ -574,6 +576,10 @@ class Stream():
                  with target.timed_activity("Staging sources to {}".format(directory)):
                      target._open_workspace()
 +            workspace_project = workspace_project_cache.add(directory, project.directory,
 +                                                            target._get_full_name())
 +            workspace_project.write()
++
              # Saving the workspace once it is set up means that if the next workspace fails to be created before
              # the configuration gets saved. The successfully created workspace still gets saved.
              workspaces.save_config()
@@ -601,6 +607,16 @@ class Stream():
                  except OSError as e:
                      raise StreamError("Could not remove  '{}': {}"
                                        .format(workspace.get_absolute_path(), e)) from e
 +        else:
 +            workspace_project_cache = self._context.get_workspace_project_cache()
 +            try:
 +                workspace_project_cache.remove(workspace.get_absolute_path())
 +            except LoadError as e:
 +                # We might be closing a workspace with a deleted directory
 +                if e.reason == LoadErrorReason.MISSING_FILE:
 +                    pass
 +                else:
 +                    raise
          # Delete the workspace and save the configuration
          workspaces.delete_workspace(element_name)
@@ -644,6 +660,8 @@ class Stream():
          for element in elements:
              workspace = workspaces.get_workspace(element._get_full_name())
              workspace_path = workspace.get_absolute_path()
 +            workspace_project_cache = self._context.get_workspace_project_cache()
 +            workspace_project = workspace_project_cache.get(workspace_path)
              if soft:
                  workspace.prepared = False
                  self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
@@ -664,6 +682,8 @@ class Stream():
              with element.timed_activity("Staging sources to {}".format(workspace_path)):
                  element._open_workspace()
 +            workspace_project.write()
++
              self._message(MessageType.INFO,
                            "Reset workspace for {} at: {}".format(element.name,
                                                                   workspace_path))
@@ -694,6 +714,20 @@ class Stream():
          return False
 +    # workspace_is_required()
 +    #
 +    # Checks whether the workspace belonging to element_name is required to
 +    # load the project
 +    #
 +    # Args:
 +    #    element_name (str): The element whose workspace may be required
 +    #
 +    # Returns:
 +    #    (bool): True if the workspace is required
 +    def workspace_is_required(self, element_name):
 +        required_elm = self._project.required_workspace_element()
 +        return required_elm == element_name
++
      # workspace_list
+     #
      # Serializes the workspaces and dumps them in YAML to stdout.

buildstream/_workspaces.py

@@ -379,10 +379,15 @@ class Workspace():
          if recalculate or self._key is None:
              fullpath = self.get_absolute_path()
 +            excluded_files = (WORKSPACE_PROJECT_FILE,)
++
              # Get a list of tuples of the the project relative paths and fullpaths
              if os.path.isdir(fullpath):
                  filelist = utils.list_relative_paths(fullpath)
 -                filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
 +                filelist = [
 +                    (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
 +                    if relpath not in excluded_files
 +                ]
              else:
                  filelist = [(self.get_absolute_path(), fullpath)]

buildstream/data/userconfig.yaml

@@ -128,6 +128,14 @@ prompt:
+   #
    really-workspace-close-remove-dir: ask
 +  # Whether to really proceed with 'bst workspace close' when doing so would
 +  # stop them from running bst commands in this workspace.
 +  #
 +  #  ask - Ask the user if they are sure.
 +  #  yes - Always close, without asking.
 +  #
 +  really-workspace-close-project-inaccessible: ask
++
    # Whether to really proceed with 'bst workspace reset' doing a hard reset of
    # a workspace, potentially losing changes.
+   #

tests/frontend/workspace.py

@@ -31,6 +31,7 @@ import shutil
  import subprocess
  from ruamel.yaml.comments import CommentedSet
  from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
 +from tests.testutils import create_artifact_share
  from buildstream import _yaml
  from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
@@ -615,9 +616,12 @@ def test_list(cli, tmpdir, datafiles):
  @pytest.mark.datafiles(DATA_DIR)
  @pytest.mark.parametrize("kind", repo_kinds)
  @pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
 -def test_build(cli, tmpdir, datafiles, kind, strict):
 +@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
 +def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
 +    tmpdir = tmpdir_factory.mktemp('')
      element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
      checkout = os.path.join(str(tmpdir), 'checkout')
 +    args_pre = ['-C', workspace] if call_from == "workspace" else []
      # Modify workspace
      shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
@@ -640,15 +644,14 @@ def test_build(cli, tmpdir, datafiles, kind, strict):
      # Build modified workspace
      assert cli.get_element_state(project, element_name) == 'buildable'
      assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
 -    result = cli.run(project=project, args=['build', element_name])
 +    result = cli.run(project=project, args=args_pre + ['build', element_name])
      result.assert_success()
      assert cli.get_element_state(project, element_name) == 'cached'
      assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
      # Checkout the result
 -    result = cli.run(project=project, args=[
 -        'checkout', element_name, checkout
 -    ])
 +    result = cli.run(project=project,
 +                     args=args_pre + ['checkout', element_name, checkout])
      result.assert_success()
      # Check that the pony.conf from the modified workspace exists
@@ -1055,3 +1058,131 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
          result = cli.run(project=project, args=["build", element_name])
          assert "BUG" not in result.stderr
          assert cli.get_element_state(project, element_name) != "cached"
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_external_fetch(cli, datafiles, tmpdir_factory):
 +    # Fetching from a workspace outside a project doesn't fail horribly
 +    tmpdir = tmpdir_factory.mktemp('')
 +    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
++
 +    result = cli.run(project=project, args=['-C', workspace, 'fetch', element_name])
 +    result.assert_success()
++
 +    # We already fetched it by opening the workspace, but we're also checking
 +    # `bst show` works here
 +    assert cli.get_element_state(project, element_name) == 'buildable'
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_external_push_pull(cli, datafiles, tmpdir_factory):
 +    # Pushing and pulling to/from an artifact cache works from an external workspace
 +    tmpdir = tmpdir_factory.mktemp('')
 +    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
++
 +    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
 +        result = cli.run(project=project, args=['-C', workspace, 'build', element_name])
 +        result.assert_success()
++
 +        cli.configure({
 +            'artifacts': {'url': share.repo, 'push': True}
 +        })
++
 +        result = cli.run(project=project, args=['-C', workspace, 'push', element_name])
 +        result.assert_success()
++
 +        result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all', element_name])
 +        result.assert_success()
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_external_track(cli, datafiles, tmpdir_factory):
 +    # Tracking does not get horribly confused
 +    tmpdir = tmpdir_factory.mktemp('')
 +    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", True)
++
 +    # The workspace is necessarily already tracked, so we only care that
 +    # there's no weird errors.
 +    result = cli.run(project=project, args=['-C', workspace, 'track', element_name])
 +    result.assert_success()
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_external_open_other(cli, datafiles, tmpdir_factory):
 +    # >From inside an external workspace, open another workspace
 +    tmpdir1 = tmpdir_factory.mktemp('')
 +    tmpdir2 = tmpdir_factory.mktemp('')
 +    # Making use of the assumption that it's the same project in both invocations of open_workspace
 +    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
 +    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
++
 +    # Closing the other element first, because I'm too lazy to create an
 +    # element without opening it
 +    result = cli.run(project=project, args=['workspace', 'close', beta_element])
 +    result.assert_success()
++
 +    result = cli.run(project=project, args=[
 +        '-C', alpha_workspace, 'workspace', 'open', '--force', '--directory', beta_workspace, beta_element
 +    ])
 +    result.assert_success()
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_external_close_other(cli, datafiles, tmpdir_factory):
 +    # >From inside an external workspace, close the other workspace
 +    tmpdir1 = tmpdir_factory.mktemp('')
 +    tmpdir2 = tmpdir_factory.mktemp('')
 +    # Making use of the assumption that it's the same project in both invocations of open_workspace
 +    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
 +    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
++
 +    result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
 +    result.assert_success()
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_external_close_self(cli, datafiles, tmpdir_factory):
 +    # >From inside an external workspace, close it
 +    tmpdir1 = tmpdir_factory.mktemp('')
 +    tmpdir2 = tmpdir_factory.mktemp('')
 +    # Making use of the assumption that it's the same project in both invocations of open_workspace
 +    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
 +    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
++
 +    result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', alpha_element])
 +    result.assert_success()
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_external_reset_other(cli, datafiles, tmpdir_factory):
 +    tmpdir1 = tmpdir_factory.mktemp('')
 +    tmpdir2 = tmpdir_factory.mktemp('')
 +    # Making use of the assumption that it's the same project in both invocations of open_workspace
 +    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
 +    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
++
 +    result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element])
 +    result.assert_success()
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_external_reset_self(cli, datafiles, tmpdir):
 +    element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
++
 +    # Command succeeds
 +    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', element])
 +    result.assert_success()
++
 +    # Successive commands still work (i.e. .bstproject.yaml hasn't been deleted)
 +    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
 +    result.assert_success()
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_external_list(cli, datafiles, tmpdir_factory):
 +    tmpdir = tmpdir_factory.mktemp('')
 +    # Making use of the assumption that it's the same project in both invocations of open_workspace
 +    element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
++
 +    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
 +    result.assert_success()

tests/integration/shell.py

@@ -353,3 +353,29 @@ def test_integration_devices(cli, tmpdir, datafiles):
      result = execute_shell(cli, project, ["true"], element=element_name)
      assert result.exit_code == 0
++
++
 +# Test that a shell can be opened from an external workspace
 +@pytest.mark.datafiles(DATA_DIR)
 +@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
 +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
 +def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell):
 +    tmpdir = tmpdir_factory.mktemp("")
 +    project = os.path.join(datafiles.dirname, datafiles.basename)
 +    element_name = 'autotools/amhello.bst'
 +    workspace_dir = os.path.join(str(tmpdir), 'workspace')
++
 +    result = cli.run(project=project, args=[
 +        'workspace', 'open', '--directory', workspace_dir, element_name
 +    ])
 +    result.assert_success()
++
 +    result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name])
 +    result.assert_success()
++
 +    command = ['shell']
 +    if build_shell == 'build':
 +        command.append('--build')
 +    command.extend([element_name, '--', 'true'])
 +    result = cli.run(project=project, cwd=workspace_dir, args=command)
 +    result.assert_success()

[Notes] [Git][BuildStream/buildstream][jonathan/workspace-fragment-create] 8 commits: _project.py: Find project from workspace if outside of a project

Jonathan Maw pushed to branch jonathan/workspace-fragment-create at BuildStream / buildstream

Commits:

9 changed files:

Changes: