GNOME.org
 

[Notes] [Git][BuildStream/buildstream][master] 7 commits: buildstream/plugins/sources/git.py: Fix clone side effect in getting fetchers

Title: GitLab

Tristan Van Berkom pushed to branch master at BuildStream / buildstream

Commits:

4 changed files:

Changes:

  • buildstream/plugins/sources/git.py
    ... ... @@ -414,8 +414,10 @@ class GitSource(Source):
    414 414 
                         mirror.stage(directory)
    415 415
    416 416 
         def get_source_fetchers(self):
    417 
    +        yield self.mirror
    417 418 
             self.refresh_submodules()
    418 
    -        return [self.mirror] + self.submodules
    419 
    +        for submodule in self.submodules:
    420 
    +            yield submodule
    419 421
    420 422 
         ###########################################################
    421 423 
         #                     Local Functions                     #

  • tests/frontend/mirror.py
    ... ... @@ -139,6 +139,67 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
    139 139 
         result.assert_success()
    140 140
    141 141
    142 
    +@pytest.mark.datafiles(DATA_DIR)
    143 
    +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
    144 
    +def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
    145 
    +    if kind == 'ostree':
    146 
    +        # FIXME: Mirroring fallback fails with ostree
    147 
    +        pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
    148 
    +
    149 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    150 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    151 
    +    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
    152 
    +    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
    153 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    154 
    +    os.makedirs(project_dir)
    155 
    +    element_dir = os.path.join(project_dir, 'elements')
    156 
    +
    157 
    +    # Create repo objects of the upstream and mirror
    158 
    +    upstream_repo = create_repo(kind, upstream_repodir)
    159 
    +    ref = upstream_repo.create(dev_files_path)
    160 
    +    mirror_repo = upstream_repo.copy(mirror_repodir)
    161 
    +
    162 
    +    element = {
    163 
    +        'kind': 'import',
    164 
    +        'sources': [
    165 
    +            upstream_repo.source_config(ref=ref)
    166 
    +        ]
    167 
    +    }
    168 
    +
    169 
    +    element_name = 'test.bst'
    170 
    +    element_path = os.path.join(element_dir, element_name)
    171 
    +    full_repo = element['sources'][0]['url']
    172 
    +    upstream_map, repo_name = os.path.split(full_repo)
    173 
    +    alias = 'foo-' + kind
    174 
    +    aliased_repo = alias + ':' + repo_name
    175 
    +    element['sources'][0]['url'] = aliased_repo
    176 
    +    full_mirror = mirror_repo.source_config()['url']
    177 
    +    mirror_map, _ = os.path.split(full_mirror)
    178 
    +    os.makedirs(element_dir)
    179 
    +    _yaml.dump(element, element_path)
    180 
    +
    181 
    +    project = {
    182 
    +        'name': 'test',
    183 
    +        'element-path': 'elements',
    184 
    +        'aliases': {
    185 
    +            alias: 'http://www.example.com/'
    186 
    +        },
    187 
    +        'mirrors': [
    188 
    +            {
    189 
    +                'name': 'middle-earth',
    190 
    +                'aliases': {
    191 
    +                    alias: [mirror_map + "/"],
    192 
    +                },
    193 
    +            },
    194 
    +        ]
    195 
    +    }
    196 
    +    project_file = os.path.join(project_dir, 'project.conf')
    197 
    +    _yaml.dump(project, project_file)
    198 
    +
    199 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    200 
    +    result.assert_success()
    201 
    +
    202 
    +
    142 203 
     @pytest.mark.datafiles(DATA_DIR)
    143 204 
     def test_mirror_fetch_multi(cli, tmpdir, datafiles):
    144 205 
         output_file = os.path.join(str(tmpdir), "output.txt")
    ... ... @@ -405,10 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
    405 466 
     @pytest.mark.datafiles(DATA_DIR)
    406 467 
     @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
    407 468 
     def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
    408 
    -    if kind == 'git':
    409 
    -        # FIXME: Mirroring fallback does not work with git because it tries to
    410 
    -        # fetch submodules on upstream.
    411 
    -        pytest.skip("Bug #537 - Mirror fallback does not work for git")
    412 469 
         if kind == 'ostree':
    413 470 
             # FIXME: Mirroring fallback fails with ostree
    414 471 
             pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
    ... ... @@ -559,3 +616,276 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
    559 616 
         os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo)
    560 617 
         result = cli.run(project=project_dir, args=['fetch', element_name])
    561 618 
         result.assert_success()
    619 
    +
    620 
    +
    621 
    +@pytest.mark.datafiles(DATA_DIR)
    622 
    +def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles):
    623 
    +    # Test that it behaves as expected with submodules, both defined in config
    624 
    +    # and discovered when fetching.
    625 
    +    foo_file = os.path.join(str(datafiles), 'files', 'foo')
    626 
    +    bar_file = os.path.join(str(datafiles), 'files', 'bar')
    627 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    628 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    629 
    +    mirror_dir = os.path.join(str(datafiles), 'mirror')
    630 
    +
    631 
    +    defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo')
    632 
    +    defined_mirror_ref = defined_subrepo.create(bin_files_path)
    633 
    +    defined_mirror = defined_subrepo.copy(mirror_dir)
    634 
    +    defined_subref = defined_subrepo.add_file(foo_file)
    635 
    +
    636 
    +    found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo')
    637 
    +    found_subref = found_subrepo.create(dev_files_path)
    638 
    +
    639 
    +    main_repo = create_repo('git', str(tmpdir))
    640 
    +    main_mirror_ref = main_repo.create(bin_files_path)
    641 
    +    main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo)
    642 
    +    main_repo.add_submodule('found', 'file://' + found_subrepo.repo)
    643 
    +    main_mirror = main_repo.copy(mirror_dir)
    644 
    +    main_ref = main_repo.add_file(bar_file)
    645 
    +
    646 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    647 
    +    os.makedirs(project_dir)
    648 
    +    element_dir = os.path.join(project_dir, 'elements')
    649 
    +    os.makedirs(element_dir)
    650 
    +    element = {
    651 
    +        'kind': 'import',
    652 
    +        'sources': [
    653 
    +            main_repo.source_config(ref=main_mirror_ref)
    654 
    +        ]
    655 
    +    }
    656 
    +    element_name = 'test.bst'
    657 
    +    element_path = os.path.join(element_dir, element_name)
    658 
    +
    659 
    +    # Alias the main repo
    660 
    +    full_repo = element['sources'][0]['url']
    661 
    +    _, repo_name = os.path.split(full_repo)
    662 
    +    alias = 'foo'
    663 
    +    aliased_repo = alias + ':' + repo_name
    664 
    +    element['sources'][0]['url'] = aliased_repo
    665 
    +
    666 
    +    # Hide the found subrepo
    667 
    +    del element['sources'][0]['submodules']['found']
    668 
    +
    669 
    +    # Alias the defined subrepo
    670 
    +    subrepo = element['sources'][0]['submodules']['defined']['url']
    671 
    +    _, repo_name = os.path.split(subrepo)
    672 
    +    aliased_repo = alias + ':' + repo_name
    673 
    +    element['sources'][0]['submodules']['defined']['url'] = aliased_repo
    674 
    +
    675 
    +    _yaml.dump(element, element_path)
    676 
    +
    677 
    +    full_mirror = main_mirror.source_config()['url']
    678 
    +    mirror_map, _ = os.path.split(full_mirror)
    679 
    +    project = {
    680 
    +        'name': 'test',
    681 
    +        'element-path': 'elements',
    682 
    +        'aliases': {
    683 
    +            alias: 'http://www.example.com/'
    684 
    +        },
    685 
    +        'mirrors': [
    686 
    +            {
    687 
    +                'name': 'middle-earth',
    688 
    +                'aliases': {
    689 
    +                    alias: [mirror_map + "/"],
    690 
    +                },
    691 
    +            },
    692 
    +        ]
    693 
    +    }
    694 
    +    project_file = os.path.join(project_dir, 'project.conf')
    695 
    +    _yaml.dump(project, project_file)
    696 
    +
    697 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    698 
    +    result.assert_success()
    699 
    +
    700 
    +
    701 
    +@pytest.mark.datafiles(DATA_DIR)
    702 
    +def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles):
    703 
    +    # Main repo has no mirror or alias.
    704 
    +    # One submodule is overridden to use a mirror.
    705 
    +    # There is another submodules not overriden.
    706 
    +    # Upstream for overriden submodule is down.
    707 
    +    #
    708 
    +    # We expect:
    709 
    +    #  - overriden submodule is fetched from mirror.
    710 
    +    #  - other submodule is fetched.
    711 
    +
    712 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    713 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    714 
    +
    715 
    +    upstream_bin_repodir = os.path.join(str(tmpdir), 'bin-upstream')
    716 
    +    mirror_bin_repodir = os.path.join(str(tmpdir), 'bin-mirror')
    717 
    +    upstream_bin_repo = create_repo('git', upstream_bin_repodir)
    718 
    +    upstream_bin_repo.create(bin_files_path)
    719 
    +    mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir)
    720 
    +
    721 
    +    dev_repodir = os.path.join(str(tmpdir), 'dev-upstream')
    722 
    +    dev_repo = create_repo('git', dev_repodir)
    723 
    +    dev_repo.create(dev_files_path)
    724 
    +
    725 
    +    main_files = os.path.join(str(tmpdir), 'main-files')
    726 
    +    os.makedirs(main_files)
    727 
    +    with open(os.path.join(main_files, 'README'), 'w') as f:
    728 
    +        f.write("TEST\n")
    729 
    +    main_repodir = os.path.join(str(tmpdir), 'main-upstream')
    730 
    +    main_repo = create_repo('git', main_repodir)
    731 
    +    main_repo.create(main_files)
    732 
    +
    733 
    +    upstream_url = 'file://{}'.format(upstream_bin_repo.repo)
    734 
    +    main_repo.add_submodule('bin', url=upstream_url)
    735 
    +    main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
    736 
    +    # Unlist 'dev'.
    737 
    +    del main_repo.submodules['dev']
    738 
    +
    739 
    +    main_ref = main_repo.latest_commit()
    740 
    +
    741 
    +    upstream_map, repo_name = os.path.split(upstream_url)
    742 
    +    alias = 'foo'
    743 
    +    aliased_repo = '{}:{}'.format(alias, repo_name)
    744 
    +    main_repo.submodules['bin']['url'] = aliased_repo
    745 
    +
    746 
    +    full_mirror = mirror_bin_repo.source_config()['url']
    747 
    +    mirror_map, _ = os.path.split(full_mirror)
    748 
    +
    749 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    750 
    +    os.makedirs(project_dir)
    751 
    +    element_dir = os.path.join(project_dir, 'elements')
    752 
    +
    753 
    +    element = {
    754 
    +        'kind': 'import',
    755 
    +        'sources': [
    756 
    +            main_repo.source_config(ref=main_ref, checkout_submodules=True)
    757 
    +        ]
    758 
    +    }
    759 
    +    element_name = 'test.bst'
    760 
    +    element_path = os.path.join(element_dir, element_name)
    761 
    +    os.makedirs(element_dir)
    762 
    +    _yaml.dump(element, element_path)
    763 
    +
    764 
    +    project = {
    765 
    +        'name': 'test',
    766 
    +        'element-path': 'elements',
    767 
    +        'aliases': {
    768 
    +            alias: upstream_map + "/"
    769 
    +        },
    770 
    +        'mirrors': [
    771 
    +            {
    772 
    +                'name': 'middle-earth',
    773 
    +                'aliases': {
    774 
    +                    alias: [mirror_map + "/"],
    775 
    +                }
    776 
    +            }
    777 
    +        ]
    778 
    +    }
    779 
    +    project_file = os.path.join(project_dir, 'project.conf')
    780 
    +    _yaml.dump(project, project_file)
    781 
    +
    782 
    +    # Now make the upstream unavailable.
    783 
    +    os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo))
    784 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    785 
    +    result.assert_success()
    786 
    +
    787 
    +    result = cli.run(project=project_dir, args=['build', element_name])
    788 
    +    result.assert_success()
    789 
    +
    790 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    791 
    +    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
    792 
    +    result.assert_success()
    793 
    +
    794 
    +    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
    795 
    +    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
    796 
    +
    797 
    +
    798 
    +@pytest.mark.datafiles(DATA_DIR)
    799 
    +def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles):
    800 
    +    # Main repo has mirror. But does not list submodules.
    801 
    +    #
    802 
    +    # We expect:
    803 
    +    #  - we will fetch submodules anyway
    804 
    +
    805 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    806 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    807 
    +
    808 
    +    bin_repodir = os.path.join(str(tmpdir), 'bin-repo')
    809 
    +    bin_repo = create_repo('git', bin_repodir)
    810 
    +    bin_repo.create(bin_files_path)
    811 
    +
    812 
    +    dev_repodir = os.path.join(str(tmpdir), 'dev-repo')
    813 
    +    dev_repo = create_repo('git', dev_repodir)
    814 
    +    dev_repo.create(dev_files_path)
    815 
    +
    816 
    +    main_files = os.path.join(str(tmpdir), 'main-files')
    817 
    +    os.makedirs(main_files)
    818 
    +    with open(os.path.join(main_files, 'README'), 'w') as f:
    819 
    +        f.write("TEST\n")
    820 
    +    upstream_main_repodir = os.path.join(str(tmpdir), 'main-upstream')
    821 
    +    upstream_main_repo = create_repo('git', upstream_main_repodir)
    822 
    +    upstream_main_repo.create(main_files)
    823 
    +
    824 
    +    upstream_main_repo.add_submodule('bin', url='file://{}'.format(bin_repo.repo))
    825 
    +    upstream_main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
    826 
    +    # Unlist submodules.
    827 
    +    del upstream_main_repo.submodules['bin']
    828 
    +    del upstream_main_repo.submodules['dev']
    829 
    +
    830 
    +    upstream_main_ref = upstream_main_repo.latest_commit()
    831 
    +
    832 
    +    mirror_main_repodir = os.path.join(str(tmpdir), 'main-mirror')
    833 
    +    mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir)
    834 
    +
    835 
    +    upstream_url = mirror_main_repo.source_config()['url']
    836 
    +
    837 
    +    upstream_map, repo_name = os.path.split(upstream_url)
    838 
    +    alias = 'foo'
    839 
    +    aliased_repo = '{}:{}'.format(alias, repo_name)
    840 
    +
    841 
    +    full_mirror = mirror_main_repo.source_config()['url']
    842 
    +    mirror_map, _ = os.path.split(full_mirror)
    843 
    +
    844 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    845 
    +    os.makedirs(project_dir)
    846 
    +    element_dir = os.path.join(project_dir, 'elements')
    847 
    +
    848 
    +    element = {
    849 
    +        'kind': 'import',
    850 
    +        'sources': [
    851 
    +            upstream_main_repo.source_config(ref=upstream_main_ref, checkout_submodules=True)
    852 
    +        ]
    853 
    +    }
    854 
    +    element['sources'][0]['url'] = aliased_repo
    855 
    +    element_name = 'test.bst'
    856 
    +    element_path = os.path.join(element_dir, element_name)
    857 
    +    os.makedirs(element_dir)
    858 
    +    _yaml.dump(element, element_path)
    859 
    +
    860 
    +    project = {
    861 
    +        'name': 'test',
    862 
    +        'element-path': 'elements',
    863 
    +        'aliases': {
    864 
    +            alias: upstream_map + "/"
    865 
    +        },
    866 
    +        'mirrors': [
    867 
    +            {
    868 
    +                'name': 'middle-earth',
    869 
    +                'aliases': {
    870 
    +                    alias: [mirror_map + "/"],
    871 
    +                }
    872 
    +            }
    873 
    +        ]
    874 
    +    }
    875 
    +    project_file = os.path.join(project_dir, 'project.conf')
    876 
    +    _yaml.dump(project, project_file)
    877 
    +
    878 
    +    # Now make the upstream unavailable.
    879 
    +    os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo))
    880 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    881 
    +    result.assert_success()
    882 
    +
    883 
    +    result = cli.run(project=project_dir, args=['build', element_name])
    884 
    +    result.assert_success()
    885 
    +
    886 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    887 
    +    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
    888 
    +    result.assert_success()
    889 
    +
    890 
    +    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
    891 
    +    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))

  • tests/frontend/project/files/bar

  • tests/frontend/project/files/foo

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]