[Notes] [Git][BuildStream/buildstream][master] 4 commits:

Tristan Van Berkom pushed to branch master at BuildStream / buildstream

Commits:

0b83d024

by Tristan Van Berkom at 2019-01-10T20:02:50Z

_pipeline.py: Fix the planner to retain the original order when depth sorting

The algorithm adds elements to a dictionary and then sorts the dictionary
by the discovered depths while recursing - using an OrderedDict is enough
to ensure a stable order.

This fixes `bst show --deps plan ...` reporting different results on
each invocation.

This fixes an aspect of #824

630e26f1

by Tristan Van Berkom at 2019-01-10T20:03:52Z

testutils/runcli.py: Added Result.get_start_order()

Added a function to report the list of elements which appeared in
a given queue in the order of their first appearance.

d1d7de57

by Tristan Van Berkom at 2019-01-11T04:13:16Z

tests/frontend/order.py: Adding tests ensuring stable element processing order

For each sample project and expected result order, this test ensures that:

  * bst show --deps plan: Always shows the expected build order.

  * bst source fetch: Always executes in the expected build order.

  * bst build: Always builds in the expected order (disabled).

The build part of the test is currently disabled as the scheduler
seems to be messing with the order even in a `--builders 1` scenario.

276b8d48

by Tristan Van Berkom at 2019-01-11T04:42:24Z

Merge branch 'tristan/element-processing-order' into 'master'

Make build plan element list stable

See merge request BuildStream/buildstream!1058

Changes:

buildstream/_pipeline.py

@@ -22,6 +22,7 @@
  import os
  import itertools
  from operator import itemgetter
 +from collections import OrderedDict
  from ._exceptions import PipelineError
  from ._message import Message, MessageType
@@ -479,7 +480,7 @@ class Pipeline():
+ #
  class _Planner():
      def __init__(self):
 -        self.depth_map = {}
 +        self.depth_map = OrderedDict()
          self.visiting_elements = set()
      # Here we want to traverse the same element more than once when

tests/frontend/order.py

 +import os
++
 +import pytest
 +from tests.testutils import cli, create_repo
++
 +from buildstream import _yaml
++
 +# Project directory
 +DATA_DIR = os.path.join(
 +    os.path.dirname(os.path.realpath(__file__)),
 +    "project",
 +)
++
++
 +def create_element(repo, name, path, dependencies, ref=None):
 +    element = {
 +        'kind': 'import',
 +        'sources': [
 +            repo.source_config(ref=ref)
 +        ],
 +        'depends': dependencies
 +    }
 +    _yaml.dump(element, os.path.join(path, name))
++
++
 +# This tests a variety of scenarios and checks that the order in
 +# which things are processed remains stable.
 +#
 +# This is especially important in order to ensure that our
 +# depth sorting and optimization of which elements should be
 +# processed first is doing it's job right, and that we are
 +# promoting elements to the build queue as soon as possible
 +#
 +# Parameters:
 +#    targets (target elements): The targets to invoke bst with
 +#    template (dict): The project template dictionary, for create_element()
 +#    expected (list): A list of element names in the expected order
 +#
 +@pytest.mark.datafiles(os.path.join(DATA_DIR))
 +@pytest.mark.parametrize("target,template,expected", [
 +    # First simple test
 +    ('3.bst', {
 +        '0.bst': ['1.bst'],
 +        '1.bst': [],
 +        '2.bst': ['0.bst'],
 +        '3.bst': ['0.bst', '1.bst', '2.bst']
 +    }, ['1.bst', '0.bst', '2.bst', '3.bst']),
++
 +    # A more complicated test with build of build dependencies
 +    ('target.bst', {
 +        'a.bst': [],
 +        'base.bst': [],
 +        'timezones.bst': [],
 +        'middleware.bst': [{'filename': 'base.bst', 'type': 'build'}],
 +        'app.bst': [{'filename': 'middleware.bst', 'type': 'build'}],
 +        'target.bst': ['a.bst', 'base.bst', 'middleware.bst', 'app.bst', 'timezones.bst']
 +    }, ['base.bst', 'middleware.bst', 'a.bst', 'app.bst', 'timezones.bst', 'target.bst']),
 +])
 +@pytest.mark.parametrize("operation", [('show'), ('fetch'), ('build')])
 +def test_order(cli, datafiles, tmpdir, operation, target, template, expected):
 +    project = os.path.join(datafiles.dirname, datafiles.basename)
 +    dev_files_path = os.path.join(project, 'files', 'dev-files')
 +    element_path = os.path.join(project, 'elements')
++
 +    # FIXME: Remove this when the test passes reliably.
 +    #
 +    #        There is no reason why the order should not
 +    #        be preserved when the builders is set to 1,
 +    #        the scheduler queue processing still seems to
 +    #        be losing the order.
 +    #
 +    if operation == 'build':
 +        pytest.skip("FIXME: This still only sometimes passes")
++
 +    # Configure to only allow one fetcher at a time, make it easy to
 +    # determine what is being planned in what order.
 +    cli.configure({
 +        'scheduler': {
 +            'fetchers': 1,
 +            'builders': 1
 +        }
 +    })
++
 +    # Build the project from the template, make import elements
 +    # all with the same repo
 +    #
 +    repo = create_repo('git', str(tmpdir))
 +    ref = repo.create(dev_files_path)
 +    for element, dependencies in template.items():
 +        create_element(repo, element, element_path, dependencies, ref=ref)
 +        repo.add_commit()
++
 +    # Run test and collect results
 +    if operation == 'show':
 +        result = cli.run(args=['show', '--deps', 'plan', '--format', '%{name}', target], project=project, silent=True)
 +        result.assert_success()
 +        results = result.output.splitlines()
 +    else:
 +        if operation == 'fetch':
 +            result = cli.run(args=['source', 'fetch', target], project=project, silent=True)
 +        else:
 +            result = cli.run(args=[operation, target], project=project, silent=True)
 +        result.assert_success()
 +        results = result.get_start_order(operation)
++
 +    # Assert the order
 +    print("Expected order: {}".format(expected))
 +    print("Observed result order: {}".format(results))
 +    assert results == expected

tests/testutils/runcli.py

@@ -167,6 +167,23 @@ class Result():
      def assert_shell_error(self, fail_message=''):
          assert self.exit_code == 1, fail_message
 +    # get_start_order()
 +    #
 +    # Gets the list of elements processed in a given queue, in the
 +    # order of their first appearances in the session.
 +    #
 +    # Args:
 +    #    activity (str): The queue activity name (like 'fetch')
 +    #
 +    # Returns:
 +    #    (list): A list of element names in the order which they first appeared in the result
 +    #
 +    def get_start_order(self, activity):
 +        results = re.findall(r'\[\s*{}:(\S+)\s*\]\s*START\s*.*\.log'.format(activity), self.stderr)
 +        if results is None:
 +            return []
 +        return list(results)
++
      # get_tracked_elements()
+     #
      # Produces a list of element names on which tracking occurred

[Notes] [Git][BuildStream/buildstream][master] 4 commits: _pipeline.py: Fix the planner to retain the original order when depth sorting

Tristan Van Berkom pushed to branch master at BuildStream / buildstream

Commits:

3 changed files:

Changes: