GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/fallback_mirror_git] 13 commits: NEWS: Move recent entries to a new 1.3.1 section

Title: GitLab

Valentin David pushed to branch valentindavid/fallback_mirror_git at BuildStream / buildstream

Commits:

17 changed files:

Changes:

  • NEWS
    1 
    +=================
    2 
    +buildstream 1.3.1
    3 
    +=================
    4 
    +
    5 
    +  o Failed builds are included in the cache as well.
    6 
    +    `bst checkout` will provide anything in `%{install-root}`.
    7 
    +    A build including cached fails will cause any dependant elements
    8 
    +    to not be scheduled and fail during artifact assembly,
    9 
    +    and display the retry prompt during an interactive session.
    10 
    +
    11 
    +  o Due to enabling the use of relative workspaces, "Legacy" workspaces
    12 
    +    may need to be closed and remade before the changes will affect them.
    13 
    +    Downgrading after using this feature may result in workspaces
    14 
    +    not functioning correctly
    15 
    +
    16 
    +  o Elements may now specify 'build-depends' and 'runtime-depends' fields
    17 
    +    to avoid having to specify the dependency type for every entry in
    18 
    +    'depends'.
    19 
    +
    20 
    +
    1 21 
     =================
    2 22 
     buildstream 1.1.5
    3 23 
     =================
    ... ... @@ -11,16 +31,8 @@ buildstream 1.1.5
    11 31
    12 32 
       o Added new `remote` source plugin for downloading file blobs
    13 33
    14 
    -  o Failed builds are included in the cache as well.
    15 
    -    `bst checkout` will provide anything in `%{install-root}`.
    16 
    -    A build including cached fails will cause any dependant elements
    17 
    -    to not be scheduled and fail during artifact assembly,
    18 
    -    and display the retry prompt during an interactive session.
    34 
    +  o Add support for the new include '(@)' directive in project.conf and .bst files
    19 35
    20 
    -  o Due to enabling the use of relative workspaces, "Legacy" workspaces
    21 
    -    may need to be closed and remade before the changes will affect them.
    22 
    -    Downgrading after using this feature may result in workspaces
    23 
    -    not functioning correctly
    24 36
    25 37 
     =================
    26 38 
     buildstream 1.1.4

  • buildstream/_loader/loadelement.py
    ... ... @@ -71,6 +71,7 @@ class LoadElement():
    71 71 
                 'kind', 'depends', 'sources', 'sandbox',
    72 72 
                 'variables', 'environment', 'environment-nocache',
    73 73 
                 'config', 'public', 'description',
    74 
    +            'build-depends', 'runtime-depends',
    74 75 
             ])
    75 76
    76 77 
             # Extract the Dependencies
    ... ... @@ -127,28 +128,46 @@ class LoadElement():
    127 128 
     # Returns:
    128 129 
     #    (list): a list of Dependency objects
    129 130 
     #
    130 
    -def _extract_depends_from_node(node):
    131 
    -    depends = _yaml.node_get(node, list, Symbol.DEPENDS, default_value=[])
    131 
    +def _extract_depends_from_node(node, *, key=None):
    132 
    +    if key is None:
    133 
    +        build_depends = _extract_depends_from_node(node, key=Symbol.BUILD_DEPENDS)
    134 
    +        runtime_depends = _extract_depends_from_node(node, key=Symbol.RUNTIME_DEPENDS)
    135 
    +        depends = _extract_depends_from_node(node, key=Symbol.DEPENDS)
    136 
    +        return build_depends + runtime_depends + depends
    137 
    +    elif key == Symbol.BUILD_DEPENDS:
    138 
    +        default_dep_type = Symbol.BUILD
    139 
    +    elif key == Symbol.RUNTIME_DEPENDS:
    140 
    +        default_dep_type = Symbol.RUNTIME
    141 
    +    elif key == Symbol.DEPENDS:
    142 
    +        default_dep_type = None
    143 
    +    else:
    144 
    +        assert False, "Unexpected value of key '{}'".format(key)
    145 
    +
    146 
    +    depends = _yaml.node_get(node, list, key, default_value=[])
    132 147 
         output_deps = []
    133 148
    134 149 
         for dep in depends:
    135 
    -        dep_provenance = _yaml.node_get_provenance(node, key=Symbol.DEPENDS, indices=[depends.index(dep)])
    150 
    +        dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[depends.index(dep)])
    136 151
    137 152 
             if isinstance(dep, str):
    138 
    -            dependency = Dependency(dep, provenance=dep_provenance)
    153 
    +            dependency = Dependency(dep, provenance=dep_provenance, dep_type=default_dep_type)
    139 154
    140 155 
             elif isinstance(dep, Mapping):
    141 
    -            _yaml.node_validate(dep, ['filename', 'type', 'junction'])
    142 
    -
    143 
    -            # Make type optional, for this we set it to None
    144 
    -            dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
    145 
    -            if dep_type is None or dep_type == Symbol.ALL:
    146 
    -                dep_type = None
    147 
    -            elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
    148 
    -                provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE)
    149 
    -                raise LoadError(LoadErrorReason.INVALID_DATA,
    150 
    -                                "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
    151 
    -                                .format(provenance, dep_type))
    156 
    +            if default_dep_type:
    157 
    +                _yaml.node_validate(dep, ['filename', 'junction'])
    158 
    +                dep_type = default_dep_type
    159 
    +            else:
    160 
    +                _yaml.node_validate(dep, ['filename', 'type', 'junction'])
    161 
    +
    162 
    +                # Make type optional, for this we set it to None
    163 
    +                dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
    164 
    +                if dep_type is None or dep_type == Symbol.ALL:
    165 
    +                    dep_type = None
    166 
    +                elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
    167 
    +                    provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE)
    168 
    +                    raise LoadError(LoadErrorReason.INVALID_DATA,
    169 
    +                                    "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
    170 
    +                                    .format(provenance, dep_type))
    152 171
    153 172 
                 filename = _yaml.node_get(dep, str, Symbol.FILENAME)
    154 173 
                 junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value=None)
    ... ... @@ -159,13 +178,13 @@ def _extract_depends_from_node(node):
    159 178
    160 179 
             else:
    161 180 
                 index = depends.index(dep)
    162 
    -            p = _yaml.node_get_provenance(node, key=Symbol.DEPENDS, indices=[index])
    181 
    +            p = _yaml.node_get_provenance(node, key=key, indices=[index])
    163 182 
                 raise LoadError(LoadErrorReason.INVALID_DATA,
    164 183 
                                 "{}: Dependency is not specified as a string or a dictionary".format(p))
    165 184
    166 185 
             output_deps.append(dependency)
    167 186
    168 
    -    # Now delete "depends", we dont want it anymore
    169 
    -    del node[Symbol.DEPENDS]
    187 
    +    # Now delete the field, we dont want it anymore
    188 
    +    del node[key]
    170 189
    171 190 
         return output_deps

  • buildstream/_loader/types.py
    ... ... @@ -26,6 +26,8 @@ class Symbol():
    26 26 
         FILENAME = "filename"
    27 27 
         KIND = "kind"
    28 28 
         DEPENDS = "depends"
    29 
    +    BUILD_DEPENDS = "build-depends"
    30 
    +    RUNTIME_DEPENDS = "runtime-depends"
    29 31 
         SOURCES = "sources"
    30 32 
         CONFIG = "config"
    31 33 
         VARIABLES = "variables"

  • buildstream/_versions.py
    ... ... @@ -23,7 +23,7 @@
    23 23 
     # This version is bumped whenever enhancements are made
    24 24 
     # to the `project.conf` format or the core element format.
    25 25 
     #
    26 
    -BST_FORMAT_VERSION = 13
    26 
    +BST_FORMAT_VERSION = 14
    27 27
    28 28
    29 29 
     # The base BuildStream artifact version

  • buildstream/plugins/sources/git.py
    ... ... @@ -414,8 +414,10 @@ class GitSource(Source):
    414 414 
                         mirror.stage(directory)
    415 415
    416 416 
         def get_source_fetchers(self):
    417 
    +        yield self.mirror
    417 418 
             self.refresh_submodules()
    418 
    -        return [self.mirror] + self.submodules
    419 
    +        for submodule in self.submodules:
    420 
    +            yield submodule
    419 421
    420 422 
         ###########################################################
    421 423 
         #                     Local Functions                     #

  • doc/source/format_declaring.rst
    ... ... @@ -98,6 +98,68 @@ relative filename to the elements they depend on here.
    98 98 
     See :ref:`format_dependencies` for more information on the dependency model.
    99 99
    100 100
    101 
    +.. _format_build_depends:
    102 
    +
    103 
    +Build-Depends
    104 
    +~~~~~~~~~~~~~
    105 
    +
    106 
    +.. code:: yaml
    107 
    +
    108 
    +   # Specify some build-dependencies
    109 
    +   build-depends:
    110 
    +   - element1.bst
    111 
    +   - element2.bst
    112 
    +
    113 
    +Build dependencies between elements can be specified with the ``build-depends`` attribute.
    114 
    +The above code snippet is equivalent to:
    115 
    +
    116 
    +.. code:: yaml
    117 
    +
    118 
    +   # Specify some build-dependencies
    119 
    +   depends:
    120 
    +   - filename: element1.bst
    121 
    +     type: build
    122 
    +   - filename: element2.bst
    123 
    +     type: build
    124 
    +
    125 
    +See :ref:`format_dependencies` for more information on the dependency model.
    126 
    +
    127 
    +.. note::
    128 
    +
    129 
    +   The ``build-depends`` configuration is available since :ref:`format version 14 <project_format_version>`
    130 
    +
    131 
    +
    132 
    +.. _format_runtime_depends:
    133 
    +
    134 
    +Runtime-Depends
    135 
    +~~~~~~~~~~~~~~~
    136 
    +
    137 
    +.. code:: yaml
    138 
    +
    139 
    +   # Specify some runtime-dependencies
    140 
    +   runtime-depends:
    141 
    +   - element1.bst
    142 
    +   - element2.bst
    143 
    +
    144 
    +Runtime dependencies between elements can be specified with the ``runtime-depends`` attribute.
    145 
    +The above code snippet is equivalent to:
    146 
    +
    147 
    +.. code:: yaml
    148 
    +
    149 
    +   # Specify some runtime-dependencies
    150 
    +   depends:
    151 
    +   - filename: element1.bst
    152 
    +     type: runtime
    153 
    +   - filename: element2.bst
    154 
    +     type: runtime
    155 
    +
    156 
    +See :ref:`format_dependencies` for more information on the dependency model.
    157 
    +
    158 
    +.. note::
    159 
    +
    160 
    +   The ``runtime-depends`` configuration is available since :ref:`format version 14 <project_format_version>`
    161 
    +
    162 
    +
    101 163 
     .. _format_sources:
    102 164
    103 165 
     Sources
    ... ... @@ -276,8 +338,8 @@ attributes are suitable.
    276 338
    277 339 
     .. note::
    278 340
    279 
    -   Note the order in which element dependencies are declared in the ``depends``
    280 
    -   list is not meaningful.
    341 
    +   Note the order in which element dependencies are declared in the ``depends``,
    342 
    +   ``build-depends`` and ``runtime-depends`` lists are not meaningful.
    281 343
    282 344 
     Dependency dictionary:
    283 345
    ... ... @@ -299,6 +361,8 @@ Attributes:
    299 361 
     * ``type``
    300 362
    301 363 
       This attribute is used to express the :ref:`dependency type <format_dependencies_types>`.
    364 
    +  This field is not permitted in :ref:`Build-Depends <format_build_depends>` or
    365 
    +  :ref:`Runtime-Depends <format_runtime_depends>`.
    302 366
    303 367 
     * ``junction``
    304 368

  • tests/frontend/mirror.py
    ... ... @@ -139,6 +139,67 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
    139 139 
         result.assert_success()
    140 140
    141 141
    142 
    +@pytest.mark.datafiles(DATA_DIR)
    143 
    +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
    144 
    +def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
    145 
    +    if kind == 'ostree':
    146 
    +        # FIXME: Mirroring fallback fails with ostree
    147 
    +        pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
    148 
    +
    149 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    150 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    151 
    +    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
    152 
    +    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
    153 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    154 
    +    os.makedirs(project_dir)
    155 
    +    element_dir = os.path.join(project_dir, 'elements')
    156 
    +
    157 
    +    # Create repo objects of the upstream and mirror
    158 
    +    upstream_repo = create_repo(kind, upstream_repodir)
    159 
    +    ref = upstream_repo.create(dev_files_path)
    160 
    +    mirror_repo = upstream_repo.copy(mirror_repodir)
    161 
    +
    162 
    +    element = {
    163 
    +        'kind': 'import',
    164 
    +        'sources': [
    165 
    +            upstream_repo.source_config(ref=ref)
    166 
    +        ]
    167 
    +    }
    168 
    +
    169 
    +    element_name = 'test.bst'
    170 
    +    element_path = os.path.join(element_dir, element_name)
    171 
    +    full_repo = element['sources'][0]['url']
    172 
    +    upstream_map, repo_name = os.path.split(full_repo)
    173 
    +    alias = 'foo-' + kind
    174 
    +    aliased_repo = alias + ':' + repo_name
    175 
    +    element['sources'][0]['url'] = aliased_repo
    176 
    +    full_mirror = mirror_repo.source_config()['url']
    177 
    +    mirror_map, _ = os.path.split(full_mirror)
    178 
    +    os.makedirs(element_dir)
    179 
    +    _yaml.dump(element, element_path)
    180 
    +
    181 
    +    project = {
    182 
    +        'name': 'test',
    183 
    +        'element-path': 'elements',
    184 
    +        'aliases': {
    185 
    +            alias: 'http://www.example.com/'
    186 
    +        },
    187 
    +        'mirrors': [
    188 
    +            {
    189 
    +                'name': 'middle-earth',
    190 
    +                'aliases': {
    191 
    +                    alias: [mirror_map + "/"],
    192 
    +                },
    193 
    +            },
    194 
    +        ]
    195 
    +    }
    196 
    +    project_file = os.path.join(project_dir, 'project.conf')
    197 
    +    _yaml.dump(project, project_file)
    198 
    +
    199 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    200 
    +    result.assert_success()
    201 
    +
    202 
    +
    142 203 
     @pytest.mark.datafiles(DATA_DIR)
    143 204 
     def test_mirror_fetch_multi(cli, tmpdir, datafiles):
    144 205 
         output_file = os.path.join(str(tmpdir), "output.txt")
    ... ... @@ -405,10 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
    405 466 
     @pytest.mark.datafiles(DATA_DIR)
    406 467 
     @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
    407 468 
     def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
    408 
    -    if kind == 'git':
    409 
    -        # FIXME: Mirroring fallback does not work with git because it tries to
    410 
    -        # fetch submodules on upstream.
    411 
    -        pytest.skip("Bug #537 - Mirror fallback does not work for git")
    412 469 
         if kind == 'ostree':
    413 470 
             # FIXME: Mirroring fallback fails with ostree
    414 471 
             pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
    ... ... @@ -559,3 +616,276 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
    559 616 
         os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo)
    560 617 
         result = cli.run(project=project_dir, args=['fetch', element_name])
    561 618 
         result.assert_success()
    619 
    +
    620 
    +
    621 
    +@pytest.mark.datafiles(DATA_DIR)
    622 
    +def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles):
    623 
    +    # Test that it behaves as expected with submodules, both defined in config
    624 
    +    # and discovered when fetching.
    625 
    +    foo_file = os.path.join(str(datafiles), 'files', 'foo')
    626 
    +    bar_file = os.path.join(str(datafiles), 'files', 'bar')
    627 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    628 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    629 
    +    mirror_dir = os.path.join(str(datafiles), 'mirror')
    630 
    +
    631 
    +    defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo')
    632 
    +    defined_mirror_ref = defined_subrepo.create(bin_files_path)
    633 
    +    defined_mirror = defined_subrepo.copy(mirror_dir)
    634 
    +    defined_subref = defined_subrepo.add_file(foo_file)
    635 
    +
    636 
    +    found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo')
    637 
    +    found_subref = found_subrepo.create(dev_files_path)
    638 
    +
    639 
    +    main_repo = create_repo('git', str(tmpdir))
    640 
    +    main_mirror_ref = main_repo.create(bin_files_path)
    641 
    +    main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo)
    642 
    +    main_repo.add_submodule('found', 'file://' + found_subrepo.repo)
    643 
    +    main_mirror = main_repo.copy(mirror_dir)
    644 
    +    main_ref = main_repo.add_file(bar_file)
    645 
    +
    646 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    647 
    +    os.makedirs(project_dir)
    648 
    +    element_dir = os.path.join(project_dir, 'elements')
    649 
    +    os.makedirs(element_dir)
    650 
    +    element = {
    651 
    +        'kind': 'import',
    652 
    +        'sources': [
    653 
    +            main_repo.source_config(ref=main_mirror_ref)
    654 
    +        ]
    655 
    +    }
    656 
    +    element_name = 'test.bst'
    657 
    +    element_path = os.path.join(element_dir, element_name)
    658 
    +
    659 
    +    # Alias the main repo
    660 
    +    full_repo = element['sources'][0]['url']
    661 
    +    _, repo_name = os.path.split(full_repo)
    662 
    +    alias = 'foo'
    663 
    +    aliased_repo = alias + ':' + repo_name
    664 
    +    element['sources'][0]['url'] = aliased_repo
    665 
    +
    666 
    +    # Hide the found subrepo
    667 
    +    del element['sources'][0]['submodules']['found']
    668 
    +
    669 
    +    # Alias the defined subrepo
    670 
    +    subrepo = element['sources'][0]['submodules']['defined']['url']
    671 
    +    _, repo_name = os.path.split(subrepo)
    672 
    +    aliased_repo = alias + ':' + repo_name
    673 
    +    element['sources'][0]['submodules']['defined']['url'] = aliased_repo
    674 
    +
    675 
    +    _yaml.dump(element, element_path)
    676 
    +
    677 
    +    full_mirror = main_mirror.source_config()['url']
    678 
    +    mirror_map, _ = os.path.split(full_mirror)
    679 
    +    project = {
    680 
    +        'name': 'test',
    681 
    +        'element-path': 'elements',
    682 
    +        'aliases': {
    683 
    +            alias: 'http://www.example.com/'
    684 
    +        },
    685 
    +        'mirrors': [
    686 
    +            {
    687 
    +                'name': 'middle-earth',
    688 
    +                'aliases': {
    689 
    +                    alias: [mirror_map + "/"],
    690 
    +                },
    691 
    +            },
    692 
    +        ]
    693 
    +    }
    694 
    +    project_file = os.path.join(project_dir, 'project.conf')
    695 
    +    _yaml.dump(project, project_file)
    696 
    +
    697 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    698 
    +    result.assert_success()
    699 
    +
    700 
    +
    701 
    +@pytest.mark.datafiles(DATA_DIR)
    702 
    +def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles):
    703 
    +    # Main repo has no mirror or alias.
    704 
    +    # One submodule is overridden to use a mirror.
    705 
    +    # There is another submodules not overriden.
    706 
    +    # Upstream for overriden submodule is down.
    707 
    +    #
    708 
    +    # We expect:
    709 
    +    #  - overriden submodule is fetched from mirror.
    710 
    +    #  - other submodule is fetched.
    711 
    +
    712 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    713 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    714 
    +
    715 
    +    upstream_bin_repodir = os.path.join(str(tmpdir), 'bin-upstream')
    716 
    +    mirror_bin_repodir = os.path.join(str(tmpdir), 'bin-mirror')
    717 
    +    upstream_bin_repo = create_repo('git', upstream_bin_repodir)
    718 
    +    upstream_bin_repo.create(bin_files_path)
    719 
    +    mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir)
    720 
    +
    721 
    +    dev_repodir = os.path.join(str(tmpdir), 'dev-upstream')
    722 
    +    dev_repo = create_repo('git', dev_repodir)
    723 
    +    dev_repo.create(dev_files_path)
    724 
    +
    725 
    +    main_files = os.path.join(str(tmpdir), 'main-files')
    726 
    +    os.makedirs(main_files)
    727 
    +    with open(os.path.join(main_files, 'README'), 'w') as f:
    728 
    +        f.write("TEST\n")
    729 
    +    main_repodir = os.path.join(str(tmpdir), 'main-upstream')
    730 
    +    main_repo = create_repo('git', main_repodir)
    731 
    +    main_repo.create(main_files)
    732 
    +
    733 
    +    upstream_url = 'file://{}'.format(upstream_bin_repo.repo)
    734 
    +    main_repo.add_submodule('bin', url=upstream_url)
    735 
    +    main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
    736 
    +    # Unlist 'dev'.
    737 
    +    del main_repo.submodules['dev']
    738 
    +
    739 
    +    main_ref = main_repo.latest_commit()
    740 
    +
    741 
    +    upstream_map, repo_name = os.path.split(upstream_url)
    742 
    +    alias = 'foo'
    743 
    +    aliased_repo = '{}:{}'.format(alias, repo_name)
    744 
    +    main_repo.submodules['bin']['url'] = aliased_repo
    745 
    +
    746 
    +    full_mirror = mirror_bin_repo.source_config()['url']
    747 
    +    mirror_map, _ = os.path.split(full_mirror)
    748 
    +
    749 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    750 
    +    os.makedirs(project_dir)
    751 
    +    element_dir = os.path.join(project_dir, 'elements')
    752 
    +
    753 
    +    element = {
    754 
    +        'kind': 'import',
    755 
    +        'sources': [
    756 
    +            main_repo.source_config(ref=main_ref, checkout_submodules=True)
    757 
    +        ]
    758 
    +    }
    759 
    +    element_name = 'test.bst'
    760 
    +    element_path = os.path.join(element_dir, element_name)
    761 
    +    os.makedirs(element_dir)
    762 
    +    _yaml.dump(element, element_path)
    763 
    +
    764 
    +    project = {
    765 
    +        'name': 'test',
    766 
    +        'element-path': 'elements',
    767 
    +        'aliases': {
    768 
    +            alias: upstream_map + "/"
    769 
    +        },
    770 
    +        'mirrors': [
    771 
    +            {
    772 
    +                'name': 'middle-earth',
    773 
    +                'aliases': {
    774 
    +                    alias: [mirror_map + "/"],
    775 
    +                }
    776 
    +            }
    777 
    +        ]
    778 
    +    }
    779 
    +    project_file = os.path.join(project_dir, 'project.conf')
    780 
    +    _yaml.dump(project, project_file)
    781 
    +
    782 
    +    # Now make the upstream unavailable.
    783 
    +    os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo))
    784 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    785 
    +    result.assert_success()
    786 
    +
    787 
    +    result = cli.run(project=project_dir, args=['build', element_name])
    788 
    +    result.assert_success()
    789 
    +
    790 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    791 
    +    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
    792 
    +    result.assert_success()
    793 
    +
    794 
    +    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
    795 
    +    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
    796 
    +
    797 
    +
    798 
    +@pytest.mark.datafiles(DATA_DIR)
    799 
    +def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles):
    800 
    +    # Main repo has mirror. But does not list submodules.
    801 
    +    #
    802 
    +    # We expect:
    803 
    +    #  - we will fetch submodules anyway
    804 
    +
    805 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    806 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    807 
    +
    808 
    +    bin_repodir = os.path.join(str(tmpdir), 'bin-repo')
    809 
    +    bin_repo = create_repo('git', bin_repodir)
    810 
    +    bin_repo.create(bin_files_path)
    811 
    +
    812 
    +    dev_repodir = os.path.join(str(tmpdir), 'dev-repo')
    813 
    +    dev_repo = create_repo('git', dev_repodir)
    814 
    +    dev_repo.create(dev_files_path)
    815 
    +
    816 
    +    main_files = os.path.join(str(tmpdir), 'main-files')
    817 
    +    os.makedirs(main_files)
    818 
    +    with open(os.path.join(main_files, 'README'), 'w') as f:
    819 
    +        f.write("TEST\n")
    820 
    +    upstream_main_repodir = os.path.join(str(tmpdir), 'main-upstream')
    821 
    +    upstream_main_repo = create_repo('git', upstream_main_repodir)
    822 
    +    upstream_main_repo.create(main_files)
    823 
    +
    824 
    +    upstream_main_repo.add_submodule('bin', url='file://{}'.format(bin_repo.repo))
    825 
    +    upstream_main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
    826 
    +    # Unlist submodules.
    827 
    +    del upstream_main_repo.submodules['bin']
    828 
    +    del upstream_main_repo.submodules['dev']
    829 
    +
    830 
    +    upstream_main_ref = upstream_main_repo.latest_commit()
    831 
    +
    832 
    +    mirror_main_repodir = os.path.join(str(tmpdir), 'main-mirror')
    833 
    +    mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir)
    834 
    +
    835 
    +    upstream_url = mirror_main_repo.source_config()['url']
    836 
    +
    837 
    +    upstream_map, repo_name = os.path.split(upstream_url)
    838 
    +    alias = 'foo'
    839 
    +    aliased_repo = '{}:{}'.format(alias, repo_name)
    840 
    +
    841 
    +    full_mirror = mirror_main_repo.source_config()['url']
    842 
    +    mirror_map, _ = os.path.split(full_mirror)
    843 
    +
    844 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    845 
    +    os.makedirs(project_dir)
    846 
    +    element_dir = os.path.join(project_dir, 'elements')
    847 
    +
    848 
    +    element = {
    849 
    +        'kind': 'import',
    850 
    +        'sources': [
    851 
    +            upstream_main_repo.source_config(ref=upstream_main_ref, checkout_submodules=True)
    852 
    +        ]
    853 
    +    }
    854 
    +    element['sources'][0]['url'] = aliased_repo
    855 
    +    element_name = 'test.bst'
    856 
    +    element_path = os.path.join(element_dir, element_name)
    857 
    +    os.makedirs(element_dir)
    858 
    +    _yaml.dump(element, element_path)
    859 
    +
    860 
    +    project = {
    861 
    +        'name': 'test',
    862 
    +        'element-path': 'elements',
    863 
    +        'aliases': {
    864 
    +            alias: upstream_map + "/"
    865 
    +        },
    866 
    +        'mirrors': [
    867 
    +            {
    868 
    +                'name': 'middle-earth',
    869 
    +                'aliases': {
    870 
    +                    alias: [mirror_map + "/"],
    871 
    +                }
    872 
    +            }
    873 
    +        ]
    874 
    +    }
    875 
    +    project_file = os.path.join(project_dir, 'project.conf')
    876 
    +    _yaml.dump(project, project_file)
    877 
    +
    878 
    +    # Now make the upstream unavailable.
    879 
    +    os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo))
    880 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    881 
    +    result.assert_success()
    882 
    +
    883 
    +    result = cli.run(project=project_dir, args=['build', element_name])
    884 
    +    result.assert_success()
    885 
    +
    886 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    887 
    +    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
    888 
    +    result.assert_success()
    889 
    +
    890 
    +    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
    891 
    +    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))

  • tests/frontend/project/files/bar

  • tests/frontend/project/files/foo

  • tests/loader/dependencies.py
    ... ... @@ -3,6 +3,7 @@ import pytest
    3 3
    4 4 
     from buildstream._exceptions import LoadError, LoadErrorReason
    5 5 
     from buildstream._loader import Loader, MetaElement
    6 
    +from tests.testutils import cli
    6 7 
     from . import make_loader
    7 8
    8 9 
     DATA_DIR = os.path.join(
    ... ... @@ -27,7 +28,7 @@ def test_two_files(datafiles):
    27 28 
         assert(len(element.dependencies) == 1)
    28 29 
         firstdep = element.dependencies[0]
    29 30 
         assert(isinstance(firstdep, MetaElement))
    30 
    -    assert(firstdep.kind == 'thefirstdep')
    31 
    +    assert(firstdep.kind == 'manual')
    31 32
    32 33
    33 34 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -47,7 +48,7 @@ def test_shared_dependency(datafiles):
    47 48 
         #
    48 49 
         firstdep = element.dependencies[0]
    49 50 
         assert(isinstance(firstdep, MetaElement))
    50 
    -    assert(firstdep.kind == 'thefirstdep')
    51 
    +    assert(firstdep.kind == 'manual')
    51 52 
         assert(len(firstdep.dependencies) == 0)
    52 53
    53 54 
         # The second specified dependency is 'shareddep'
    ... ... @@ -86,7 +87,7 @@ def test_dependency_dict(datafiles):
    86 87 
         assert(len(element.dependencies) == 1)
    87 88 
         firstdep = element.dependencies[0]
    88 89 
         assert(isinstance(firstdep, MetaElement))
    89 
    -    assert(firstdep.kind == 'thefirstdep')
    90 
    +    assert(firstdep.kind == 'manual')
    90 91
    91 92
    92 93 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -186,3 +187,49 @@ def test_all_dependency(datafiles):
    186 187 
         assert(isinstance(firstdep, MetaElement))
    187 188 
         firstbuilddep = element.build_dependencies[0]
    188 189 
         assert(firstdep == firstbuilddep)
    190 
    +
    191 
    +
    192 
    +@pytest.mark.datafiles(DATA_DIR)
    193 
    +def test_list_build_dependency(cli, datafiles):
    194 
    +    project = str(datafiles)
    195 
    +
    196 
    +    # Check that the pipeline includes the build dependency
    197 
    +    deps = cli.get_pipeline(project, ['elements/builddep-list.bst'], scope="build")
    198 
    +    assert "elements/firstdep.bst" in deps
    199 
    +
    200 
    +
    201 
    +@pytest.mark.datafiles(DATA_DIR)
    202 
    +def test_list_runtime_dependency(cli, datafiles):
    203 
    +    project = str(datafiles)
    204 
    +
    205 
    +    # Check that the pipeline includes the runtime dependency
    206 
    +    deps = cli.get_pipeline(project, ['elements/runtimedep-list.bst'], scope="run")
    207 
    +    assert "elements/firstdep.bst" in deps
    208 
    +
    209 
    +
    210 
    +@pytest.mark.datafiles(DATA_DIR)
    211 
    +def test_list_dependencies_combined(cli, datafiles):
    212 
    +    project = str(datafiles)
    213 
    +
    214 
    +    # Check that runtime deps get combined
    215 
    +    rundeps = cli.get_pipeline(project, ['elements/list-combine.bst'], scope="run")
    216 
    +    assert "elements/firstdep.bst" not in rundeps
    217 
    +    assert "elements/seconddep.bst" in rundeps
    218 
    +    assert "elements/thirddep.bst" in rundeps
    219 
    +
    220 
    +    # Check that build deps get combined
    221 
    +    builddeps = cli.get_pipeline(project, ['elements/list-combine.bst'], scope="build")
    222 
    +    assert "elements/firstdep.bst" in builddeps
    223 
    +    assert "elements/seconddep.bst" not in builddeps
    224 
    +    assert "elements/thirddep.bst" in builddeps
    225 
    +
    226 
    +
    227 
    +@pytest.mark.datafiles(DATA_DIR)
    228 
    +def test_list_overlap(cli, datafiles):
    229 
    +    project = str(datafiles)
    230 
    +
    231 
    +    # Check that dependencies get merged
    232 
    +    rundeps = cli.get_pipeline(project, ['elements/list-overlap.bst'], scope="run")
    233 
    +    assert "elements/firstdep.bst" in rundeps
    234 
    +    builddeps = cli.get_pipeline(project, ['elements/list-overlap.bst'], scope="build")
    235 
    +    assert "elements/firstdep.bst" in builddeps

  • tests/loader/dependencies/elements/builddep-list.bst
    1 
    +kind: stack
    2 
    +description: This element has a build-only dependency specified via build-depends
    3 
    +build-depends:
    4 
    +  - elements/firstdep.bst

  • tests/loader/dependencies/elements/firstdep.bst
    1 
    -kind: thefirstdep
    1 
    +kind: manual
    2 2 
     description: This is the first dependency

  • tests/loader/dependencies/elements/list-combine.bst
    1 
    +kind: stack
    2 
    +description: This element depends on three elements in different ways
    3 
    +build-depends:
    4 
    +- elements/firstdep.bst
    5 
    +runtime-depends:
    6 
    +- elements/seconddep.bst
    7 
    +depends:
    8 
    +- elements/thirddep.bst

  • tests/loader/dependencies/elements/list-overlap.bst
    1 
    +kind: stack
    2 
    +description: This element depends on two elements in different ways
    3 
    +build-depends:
    4 
    +- elements/firstdep.bst
    5 
    +depends:
    6 
    +- filename: elements/firstdep.bst
    7 
    +  type: runtime

  • tests/loader/dependencies/elements/runtimedep-list.bst
    1 
    +kind: stack
    2 
    +description: This element has a runtime-only dependency
    3 
    +runtime-depends:
    4 
    +  - elements/firstdep.bst

  • tests/loader/dependencies/elements/seconddep.bst
    1 
    +kind: manual
    2 
    +description: This is the second dependency

  • tests/loader/dependencies/elements/thirddep.bst
    1 
    +kind: manual
    2 
    +description: This is the third dependency

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]