[Notes] [Git][BuildStream/buildstream][bst-1.2] 5 commits: element.py: S

Jürg Billeter pushed to branch bst-1.2 at BuildStream / buildstream

Commits:

76a66abf

by Jürg Billeter at 2018-08-28T13:29:47Z

element.py: Schedule assemble for key of workspaced elements

For uncached workspaced elements, assemble is required even just to
calculate the cache key. As dynamic scheduling relies on cache keys,
schedule assemble for uncached workspace elements even if they have not
been marked as required yet.

Fixes #461.

1a96b537

by Valentin David at 2018-08-28T13:29:48Z

tests/frontend/workspace.py: Add test for workspaced dependencies

This adds a regression test for #461.

765dd3d9

by Jürg Billeter at 2018-08-28T13:29:48Z

element.py: Do not attempt to pull workspaced elements

0de2f87b

by Tristan Van Berkom at 2018-08-28T13:39:18Z

tests/frontend/workspaces.py: Removing some redundant tests

Removed redundant tests from recently merged !740, this new
test does not need to run for every different source kind.

7da1c309

by Jürg Billeter at 2018-08-28T16:32:34Z

Merge branch 'juerg/workspaced-dependencies-1.2' into 'bst-1.2'

Fix key invalidation for workspaced dependencies

See merge request BuildStream/buildstream!744

Changes:

buildstream/element.py

@@ -1085,9 +1085,12 @@ class Element(Plugin):
              # until the full cache query below.
              cached = self.__artifacts.contains(self, self.__weak_cache_key)
              if (not self.__assemble_scheduled and not self.__assemble_done and
 -                    not cached and not self._pull_pending() and self._is_required()):
 -                self._schedule_assemble()
 -                return
 +                    not cached and not self._pull_pending()):
 +                # For uncached workspaced elements, assemble is required
 +                # even if we only need the cache key
 +                if self._is_required() or self._get_workspace():
 +                    self._schedule_assemble()
 +                    return
          if self.__strict_cache_key is None:
              dependencies = [
@@ -1107,13 +1110,17 @@ class Element(Plugin):
              self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key)
          if (not self.__assemble_scheduled and not self.__assemble_done and
 -                not self.__cached and not self._pull_pending() and self._is_required()):
 +                not self.__cached and not self._pull_pending()):
              # Workspaced sources are considered unstable if a build is pending
              # as the build will modify the contents of the workspace.
              # Determine as early as possible if a build is pending to discard
              # unstable cache keys.
 -            self._schedule_assemble()
 -            return
++
 +            # For uncached workspaced elements, assemble is required
 +            # even if we only need the cache key
 +            if self._is_required() or self._get_workspace():
 +                self._schedule_assemble()
 +                return
          if self.__cache_key is None:
              # Calculate strong cache key
@@ -1382,7 +1389,6 @@ class Element(Plugin):
      # in a subprocess.
+     #
      def _schedule_assemble(self):
 -        assert self._is_required()
          assert not self.__assemble_scheduled
          self.__assemble_scheduled = True
@@ -1390,6 +1396,8 @@ class Element(Plugin):
          for dep in self.dependencies(Scope.BUILD, recurse=False):
              dep._set_required()
 +        self._set_required()
++
          # Invalidate workspace key as the build modifies the workspace directory
          workspace = self._get_workspace()
          if workspace:
@@ -1579,6 +1587,10 @@ class Element(Plugin):
      #   (bool): Whether a pull operation is pending
+     #
      def _pull_pending(self):
 +        if self._get_workspace():
 +            # Workspace builds are never pushed to artifact servers
 +            return False
++
          if self.__strong_cached:
              # Artifact already in local cache
              return False

tests/frontend/workspace.py

@@ -712,3 +712,73 @@ def test_inconsitent_pipeline_message(cli, tmpdir, datafiles, kind):
          'build', element_name
      ])
      result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline-workspaced")
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
 +def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict):
 +    checkout = os.path.join(str(tmpdir), 'checkout')
 +    element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'),
 +                                                      datafiles, 'git', False)
++
 +    element_path = os.path.join(project, 'elements')
 +    back_dep_element_name = 'workspace-test-back-dep.bst'
++
 +    # Write out our test target
 +    element = {
 +        'kind': 'compose',
 +        'depends': [
 +            {
 +                'filename': element_name,
 +                'type': 'build'
 +            }
 +        ]
 +    }
 +    _yaml.dump(element,
 +               os.path.join(element_path,
 +                            back_dep_element_name))
++
 +    # Modify workspace
 +    shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
 +    os.makedirs(os.path.join(workspace, 'etc'))
 +    with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
 +        f.write("PONY='pink'")
++
 +    # Configure strict mode
 +    strict_mode = True
 +    if strict != 'strict':
 +        strict_mode = False
 +    cli.configure({
 +        'projects': {
 +            'test': {
 +                'strict': strict_mode
 +            }
 +        }
 +    })
++
 +    # Build artifact with dependency's modified workspace
 +    assert cli.get_element_state(project, element_name) == 'buildable'
 +    assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
 +    assert cli.get_element_state(project, back_dep_element_name) == 'waiting'
 +    assert cli.get_element_key(project, back_dep_element_name) == "{:?<64}".format('')
 +    result = cli.run(project=project, args=['build', back_dep_element_name])
 +    result.assert_success()
 +    assert cli.get_element_state(project, element_name) == 'cached'
 +    assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
 +    assert cli.get_element_state(project, back_dep_element_name) == 'cached'
 +    assert cli.get_element_key(project, back_dep_element_name) != "{:?<64}".format('')
 +    result = cli.run(project=project, args=['build', back_dep_element_name])
 +    result.assert_success()
++
 +    # Checkout the result
 +    result = cli.run(project=project, args=[
 +        'checkout', back_dep_element_name, checkout
 +    ])
 +    result.assert_success()
++
 +    # Check that the pony.conf from the modified workspace exists
 +    filename = os.path.join(checkout, 'etc', 'pony.conf')
 +    assert os.path.exists(filename)
++
 +    # Check that the original /usr/bin/hello is not in the checkout
 +    assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello'))

[Notes] [Git][BuildStream/buildstream][bst-1.2] 5 commits: element.py: Schedule assemble for key of workspaced elements

Jürg Billeter pushed to branch bst-1.2 at BuildStream / buildstream

Commits:

2 changed files:

Changes: