[Notes] [Git][BuildStream/buildstream][relative

Phil Dawson pushed to branch relative_workspaces at BuildStream / buildstream

Commits:

6d349610

by Javier Jardón at 2018-08-07T10:53:23Z

source/install_linux_distro.rst: Fix formatting

493d19d2

by James Ennis at 2018-08-07T11:08:36Z

install_linux_distro.rst: Change install instructions for python-arpy package on Arch

  - python-arpy is unobtainable via pacman
  - Also a minor change to how we display install instructions
  from Fedora's dnf package manager

fc9869e2

by James Ennis at 2018-08-07T11:56:21Z

Merge branch 'jennis/change_arch_install_instructions' into 'master'

Change install instructions for python-arpy package on Arch

See merge request BuildStream/buildstream!614

42aa3999

by William Salmon at 2018-08-07T13:41:02Z

Add warning to git track if track and ref are not present

This is to address https://gitlab.com/BuildStream/buildstream/issues/471 that
documented unhelpfull behavour when tracking git sources.

2ceb5dec

by Will Salmon at 2018-08-07T14:46:36Z

Merge branch 'willsalmon/trackWarning' into 'master'

Add warning to git track if track and ref are not present

Closes #471

See merge request BuildStream/buildstream!580

eee4b674

by Jürg Billeter at 2018-08-07T15:36:35Z

Revert "Restrict version of pylint"

This reverts commit 4f168b9b6a02216e2fae24d758ae6b778e545869.

The latest version of pytest_pylint works fine with pylint 2, which
means there is no longer a reason to restrict the pylint version.

pylint 2 is required for Python 3.7.

ea27e389

by Jürg Billeter at 2018-08-07T15:36:35Z

_artifactcache/cascache.py: Fix for PEP 479 / Python 3.7

Do not rely on `StopIteration` bubbling up.

https://www.python.org/dev/peps/pep-0479/

fa5a59f0

by Jürg Billeter at 2018-08-07T16:48:21Z

Merge branch 'juerg/python3.7' into 'master'

Python 3.7 support

See merge request BuildStream/buildstream!615

5a7a6404

by Phillip Smyth at 2018-08-08T08:54:49Z

Implementing relative workspaces

This fixes #191

Changes:

buildstream/_artifactcache/cascache.py

@@ -846,6 +846,9 @@ class _CASRemote():
  def _grouper(iterable, n):
 -    # pylint: disable=stop-iteration-return
      while True:
 -        yield itertools.chain([next(iterable)], itertools.islice(iterable, n - 1))
 +        try:
 +            current = next(iterable)
 +        except StopIteration:
 +            return
 +        yield itertools.chain([current], itertools.islice(iterable, n - 1))

buildstream/_frontend/widget.py

@@ -418,7 +418,9 @@ class LogLine(Widget):
              if "%{workspace-dirs" in format_:
                  workspace = element._get_workspace()
                  if workspace is not None:
 -                    path = workspace.path.replace(os.getenv('HOME', '/root'), '~')
 +                    path = workspace.get_absolute_path()
 +                    if path.startswith("~/"):
 +                        path = os.path.join(os.getenv('HOME', '/root'), path[2:])
                      line = p.fmt_subst(line, 'workspace-dirs', "Workspace: {}".format(path))
                  else:
                      line = p.fmt_subst(

buildstream/_pipeline.py

@@ -358,10 +358,24 @@ class Pipeline():
                      inconsistent.append(element)
          if inconsistent:
 -            detail = "Exact versions are missing for the following elements\n" + \
 -                     "Try tracking these elements first with `bst track`\n\n"
 +            detail = "Exact versions are missing for the following elements:\n\n"
++
 +            missingTrack = 0
              for element in inconsistent:
 -                detail += "  " + element._get_full_name() + "\n"
 +                detail += "  " + element._get_full_name()
 +                for source in element.sources():
 +                    if not source._get_consistency() and not source.get_ref():
 +                        if hasattr(source, 'tracking') and source.tracking is None:
 +                            detail += ": Source {} is missing ref and track. ".format(source._get_full_name()) + \
 +                                      "Please specify a ref or branch/tag to track."
 +                            missingTrack = 1
++
 +                detail += "\n"
++
 +            if missingTrack:
 +                detail += "\nThen track these elements with `bst track`\n"
 +            else:
 +                detail += "\nTry tracking these elements first with `bst track`\n"
              raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline")
      #############################################################

buildstream/_stream.py

@@ -460,7 +460,7 @@ class Stream():
                                                selection=PipelineSelection.REDIRECT,
                                                track_selection=PipelineSelection.REDIRECT)
          target = elements[0]
 -        workdir = os.path.abspath(directory)
 +        directory = os.path.abspath(directory)
          if not list(target.sources()):
              build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
@@ -476,7 +476,7 @@ class Stream():
          workspace = workspaces.get_workspace(target._get_full_name())
          if workspace and not force:
              raise StreamError("Workspace '{}' is already defined at: {}"
 -                              .format(target.name, workspace.path))
 +                              .format(target.name, workspace.get_absolute_path()))
          # If we're going to checkout, we need at least a fetch,
          # if we were asked to track first, we're going to fetch anyway.
@@ -502,7 +502,7 @@ class Stream():
          except OSError as e:
              raise StreamError("Failed to create workspace directory: {}".format(e)) from e
 -        workspaces.create_workspace(target._get_full_name(), workdir)
 +        workspaces.create_workspace(target._get_full_name(), directory)
          if not no_checkout:
              with target.timed_activity("Staging sources to {}".format(directory)):
@@ -526,12 +526,12 @@ class Stream():
          # Remove workspace directory if prompted
          if remove_dir:
              with self._context.timed_activity("Removing workspace directory {}"
 -                                              .format(workspace.path)):
 +                                              .format(workspace.get_absolute_path())):
                  try:
 -                    shutil.rmtree(workspace.path)
 +                    shutil.rmtree(workspace.get_absolute_path())
                  except OSError as e:
                      raise StreamError("Could not remove  '{}': {}"
 -                                      .format(workspace.path, e)) from e
 +                                      .format(workspace.get_absolute_path(), e)) from e
          # Delete the workspace and save the configuration
          workspaces.delete_workspace(element_name)
@@ -574,28 +574,30 @@ class Stream():
          for element in elements:
              workspace = workspaces.get_workspace(element._get_full_name())
+-
 +            workspace_path = workspace.get_absolute_path()
              if soft:
                  workspace.prepared = False
                  self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
 -                              .format(element.name, workspace.path))
 +                              .format(element.name, workspace_path))
                  continue
              with element.timed_activity("Removing workspace directory {}"
 -                                        .format(workspace.path)):
 +                                        .format(workspace_path)):
                  try:
 -                    shutil.rmtree(workspace.path)
 +                    shutil.rmtree(workspace_path)
                  except OSError as e:
                      raise StreamError("Could not remove  '{}': {}"
 -                                      .format(workspace.path, e)) from e
 +                                      .format(workspace_path, e)) from e
              workspaces.delete_workspace(element._get_full_name())
 -            workspaces.create_workspace(element._get_full_name(), workspace.path)
 +            workspaces.create_workspace(element._get_full_name(), workspace_path)
 -            with element.timed_activity("Staging sources to {}".format(workspace.path)):
 +            with element.timed_activity("Staging sources to {}".format(workspace_path)):
                  element._open_workspace()
 -            self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path))
 +            self._message(MessageType.INFO,
 +                          "Reset workspace for {} at: {}".format(element.name,
 +                                                                 workspace_path))
          workspaces.save_config()
@@ -632,7 +634,7 @@ class Stream():
          for element_name, workspace_ in self._context.get_workspaces().list():
              workspace_detail = {
                  'element': element_name,
 -                'directory': workspace_.path,
 +                'directory': workspace_.get_absolute_path(),
+             }
              workspaces.append(workspace_detail)

buildstream/_workspaces.py

@@ -26,14 +26,6 @@ from ._exceptions import LoadError, LoadErrorReason
  BST_WORKSPACE_FORMAT_VERSION = 3
 -# Hold on to a list of members which get serialized
 -_WORKSPACE_MEMBERS = [
 -    'prepared',
 -    'path',
 -    'last_successful',
 -    'running_files'
 -]
+-
  # Workspace()
+ #
@@ -56,7 +48,7 @@ class Workspace():
      def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
          self.prepared = prepared
          self.last_successful = last_successful
 -        self.path = path
 +        self._path = path
          self.running_files = running_files if running_files is not None else {}
          self._toplevel_project = toplevel_project
@@ -64,14 +56,20 @@ class Workspace():
      # to_dict()
+     #
 -    # Convert this object to a dict for serialization purposes
 +    # Convert a list of members which get serialized to a dict for serialization purposes
+     #
      # Returns:
      #     (dict) A dict representation of the workspace
+     #
      def to_dict(self):
 -        return {key: val for key, val in self.__dict__.items()
 -                if key in _WORKSPACE_MEMBERS and val is not None}
 +        ret = {
 +            'prepared': self.prepared,
 +            'path': self._path,
 +            'running_files': self.running_files
 +        }
 +        if self.last_successful is not None:
 +            ret["last_successful"] = self.last_successful
 +        return ret
      # from_dict():
+     #
@@ -103,15 +101,7 @@ class Workspace():
      #    True if the workspace differs from 'other', otherwise False
+     #
      def differs(self, other):
+-
 -        for member in _WORKSPACE_MEMBERS:
 -            member_a = getattr(self, member)
 -            member_b = getattr(other, member)
+-
 -            if member_a != member_b:
 -                return True
+-
 -        return False
 +        return self.to_dict() != other.to_dict()
      # invalidate_key()
+     #
@@ -133,7 +123,7 @@ class Workspace():
          if os.path.isdir(fullpath):
              utils.copy_files(fullpath, directory)
          else:
 -            destfile = os.path.join(directory, os.path.basename(self.path))
 +            destfile = os.path.join(directory, os.path.basename(self.get_absolute_path()))
              utils.safe_copy(fullpath, destfile)
      # add_running_files()
@@ -189,7 +179,7 @@ class Workspace():
                  filelist = utils.list_relative_paths(fullpath)
                  filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
              else:
 -                filelist = [(self.path, fullpath)]
 +                filelist = [(self.get_absolute_path(), fullpath)]
              self._key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
@@ -200,7 +190,7 @@ class Workspace():
      # Returns: The absolute path of the element's workspace.
+     #
      def get_absolute_path(self):
 -        return os.path.join(self._toplevel_project.directory, self.path)
 +        return os.path.join(self._toplevel_project.directory, self._path)
  # Workspaces()
@@ -236,6 +226,9 @@ class Workspaces():
      #    path (str) - The path in which the workspace should be kept
+     #
      def create_workspace(self, element_name, path):
 +        if path.startswith(self._toplevel_project.directory):
 +            path = os.path.relpath(path, self._toplevel_project.directory)
++
          self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
          return self._workspaces[element_name]

buildstream/element.py

@@ -1403,7 +1403,8 @@ class Element(Plugin):
                      # If mount_workspaces is set and we're doing incremental builds,
                      # the workspace is already mounted into the sandbox.
                      if not (mount_workspaces and self.__can_build_incrementally()):
 -                        with self.timed_activity("Staging local files at {}".format(workspace.path)):
 +                        with self.timed_activity("Staging local files at {}"
 +                                                 .format(workspace.get_absolute_path())):
                              workspace.stage(temp_staging_directory)
                  else:
                      # No workspace, stage directly
@@ -1411,6 +1412,7 @@ class Element(Plugin):
                          source._stage(temp_staging_directory)
                  vdirectory.import_files(temp_staging_directory)
++
          # Ensure deterministic mtime of sources at build time
          vdirectory.set_deterministic_mtime()
          # Ensure deterministic owners of sources at build time
@@ -1566,7 +1568,7 @@ class Element(Plugin):
                          path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep)
                          sandbox_vpath = sandbox_vroot.descend(path_components)
                          try:
 -                            sandbox_vpath.import_files(workspace.path)
 +                            sandbox_vpath.import_files(workspace.get_absolute_path())
                          except UtilError as e:
                              self.warn("Failed to preserve workspace state for failed build sysroot: {}"
                                        .format(e))
@@ -1893,7 +1895,7 @@ class Element(Plugin):
                  source._init_workspace(temp)
              # Now hardlink the files into the workspace target.
 -            utils.link_files(temp, workspace.path)
 +            utils.link_files(temp, workspace.get_absolute_path())
      # _get_workspace():
+     #

buildstream/plugins/sources/git.py

@@ -363,6 +363,12 @@ class GitSource(Source):
          # If self.tracking is not specified it's not an error, just silently return
          if not self.tracking:
 +            # Is there a better way to check if a ref is given.
 +            if self.mirror.ref is None:
 +                detail = 'Without a tracking branch ref can not be updated. Please ' + \
 +                         'provide a ref or a track.'
 +                raise SourceError("{}: No track or ref".format(self),
 +                                  detail=detail, reason="track-attempt-no-track")
              return None
          with self.timed_activity("Tracking {} from {}"

doc/source/install_linux_distro.rst

@@ -57,9 +57,20 @@ Install the dependencies with::
  For the default plugins::
    sudo pacman -S \
 -      bzr git lzip ostree patch python-arpy python-gobject
 +      bzr git lzip ostree patch python-gobject
 +The package *python-arpy* is required by the deb source plugin. This is not
 +obtainable via `pacman`, you must get *python-arpy* from AUR:
 +https://aur.archlinux.org/packages/python-arpy/
++
 +To install::
++
 +  wget https://aur.archlinux.org/cgit/aur.git/snapshot/python-arpy.tar.gz
 +  tar -xvf python-arpy.tar.gz
 +  cd python-arpy
 +  makepkg -si
++
  Debian
  ++++++
  Install the dependencies with::
@@ -218,14 +229,16 @@ Arch Linux
  ~~~~~~~~~~
  Packages for Arch exist in `AUR <https://wiki.archlinux.org/index.php/Arch_User_Repository#Installing_packages>`_.
  Two different package versions are available:
++
  * Latest release: `buildstream <https://aur.archlinux.org/packages/buildstream>`_
  * Latest development snapshot: `buildstream-git <https://aur.archlinux.org/packages/buildstream-git>`_
++
  Fedora
  ~~~~~~
  BuildStream is not yet in the official Fedora repositories, but you can
 -install it from a Copr:
 +install it from a Copr::
    sudo dnf copr enable bochecha/buildstream
    sudo dnf install buildstream

setup.py

@@ -272,6 +272,5 @@ setup(name='BuildStream',
                       'pytest-cov >= 2.5.0',
                       # Provide option to run tests in parallel, less reliable
                       'pytest-xdist',
 -                     'pytest >= 3.1.0',
 -                     'pylint >= 1.8 , < 2'],
 +                     'pytest >= 3.1.0'],
        zip_safe=False)

tests/frontend/workspace.py

@@ -18,12 +18,13 @@ DATA_DIR = os.path.join(
+ )
 -def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
 -    project = os.path.join(datafiles.dirname, datafiles.basename)
 -    bin_files_path = os.path.join(project, 'files', 'bin-files')
 -    element_path = os.path.join(project, 'elements')
 +def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None):
 +    if not workspace_dir:
 +        workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
 +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
 +    bin_files_path = os.path.join(project_path, 'files', 'bin-files')
 +    element_path = os.path.join(project_path, 'elements')
      element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
 -    workspace = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
      # Create our repo object of the given source type with
      # the bin files, and then collect the initial ref.
@@ -45,7 +46,7 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
                              element_name))
      # Assert that there is no reference, a track & fetch is needed
 -    state = cli.get_element_state(project, element_name)
 +    state = cli.get_element_state(project_path, element_name)
      if track:
          assert state == 'no reference'
      else:
@@ -56,20 +57,20 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
      args = ['workspace', 'open']
      if track:
          args.append('--track')
 -    args.extend([element_name, workspace])
 +    args.extend([element_name, workspace_dir])
 +    result = cli.run(project=project_path, args=args)
 -    result = cli.run(project=project, args=args)
      result.assert_success()
      # Assert that we are now buildable because the source is
      # now cached.
 -    assert cli.get_element_state(project, element_name) == 'buildable'
 +    assert cli.get_element_state(project_path, element_name) == 'buildable'
      # Check that the executable hello file is found in the workspace
 -    filename = os.path.join(workspace, 'usr', 'bin', 'hello')
 +    filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
      assert os.path.exists(filename)
 -    return (element_name, project, workspace)
 +    return (element_name, project_path, workspace_dir)
  @pytest.mark.datafiles(DATA_DIR)
@@ -190,6 +191,45 @@ def test_close(cli, tmpdir, datafiles, kind):
      assert not os.path.exists(workspace)
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_close_external_after_move_project(cli, tmpdir, datafiles):
 +    tmp_parent = os.path.dirname(str(tmpdir))
 +    workspace_dir = os.path.join(tmp_parent, "workspace")
 +    element_name, project_path, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir)
 +    assert os.path.exists(workspace_dir)
 +    tmp_dir = os.path.join(tmp_parent, 'external_project')
 +    shutil.move(project_path, tmp_dir)
 +    assert os.path.exists(tmp_dir)
++
 +    # Close the workspace
 +    result = cli.run(configure=False, project=tmp_dir, args=[
 +        'workspace', 'close', '--remove-dir', element_name
 +    ])
 +    result.assert_success()
++
 +    # Assert the workspace dir has been deleted
 +    assert not os.path.exists(workspace_dir)
 +    shutil.move(tmp_dir, project_path)
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_close_internal_after_move_project(cli, tmpdir, datafiles):
 +    element_name, project, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
 +    tmp_dir = os.path.join(os.path.dirname(str(tmpdir)), 'external_project')
 +    shutil.move(str(tmpdir), tmp_dir)
 +    assert os.path.exists(tmp_dir)
++
 +    # Close the workspace
 +    result = cli.run(configure=False, project=tmp_dir, args=[
 +        'workspace', 'close', '--remove-dir', element_name
 +    ])
 +    result.assert_success()
++
 +    # Assert the workspace dir has been deleted
 +    workspace = os.path.join(tmp_dir, 'workspace')
 +    assert not os.path.exists(workspace)
++
++
  @pytest.mark.datafiles(DATA_DIR)
  def test_close_removed(cli, tmpdir, datafiles):
      element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)

tests/sources/git.py

@@ -359,3 +359,45 @@ def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
      # Assert that we are just fine without it, and emit a warning to the user.
      assert "Ignoring inconsistent submodule" in result.stderr
++
++
 +@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
 +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 +def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
 +    project = os.path.join(datafiles.dirname, datafiles.basename)
++
 +    # Create the repo from 'repofiles' subdir
 +    repo = create_repo('git', str(tmpdir))
 +    ref = repo.create(os.path.join(project, 'repofiles'))
++
 +    # Write out our test target
 +    gitsource = repo.source_config(ref=None)
 +    gitsource.pop('track')
 +    element = {
 +        'kind': 'import',
 +        'sources': [
 +            gitsource
 +        ]
 +    }
++
 +    _yaml.dump(element, os.path.join(project, 'target.bst'))
++
 +    # Track will encounter an inconsistent submodule without any ref
 +    result = cli.run(project=project, args=['track', 'target.bst'])
 +    result.assert_main_error(ErrorDomain.STREAM, None)
 +    result.assert_task_error(ErrorDomain.SOURCE, 'track-attempt-no-track')
++
 +    # Assert that we are just fine without it, and emit a warning to the user.
 +    assert "FAILURE git source at" in result.stderr
 +    assert "Without a tracking branch ref can not be updated. Please " + \
 +        "provide a ref or a track." in result.stderr
++
 +    # Track will encounter an inconsistent submodule without any ref
 +    result = cli.run(project=project, args=['build', 'target.bst'])
 +    result.assert_main_error(ErrorDomain.PIPELINE, 'inconsistent-pipeline')
 +    result.assert_task_error(None, None)
++
 +    # Assert that we are just fine without it, and emit a warning to the user.
 +    assert "Exact versions are missing for the following elements" in result.stderr
 +    assert "is missing ref and track." in result.stderr
 +    assert "Then track these elements with `bst track`" in result.stderr

[Notes] [Git][BuildStream/buildstream][relative_workspaces] 9 commits: source/install_linux_distro.rst: Fix formatting

Phil Dawson pushed to branch relative_workspaces at BuildStream / buildstream

Commits:

11 changed files:

Changes: