[Notes] [Git][BuildStream/buildstream][master] 7 commits: buildstream/plugins/sources/git.py: Fix clone side effect in getting fetchers



Title: GitLab

Tristan Van Berkom pushed to branch master at BuildStream / buildstream

Commits:

4 changed files:

Changes:

  • buildstream/plugins/sources/git.py
    ... ... @@ -414,8 +414,10 @@ class GitSource(Source):
    414 414
                         mirror.stage(directory)
    
    415 415
     
    
    416 416
         def get_source_fetchers(self):
    
    417
    +        yield self.mirror
    
    417 418
             self.refresh_submodules()
    
    418
    -        return [self.mirror] + self.submodules
    
    419
    +        for submodule in self.submodules:
    
    420
    +            yield submodule
    
    419 421
     
    
    420 422
         ###########################################################
    
    421 423
         #                     Local Functions                     #
    

  • tests/frontend/mirror.py
    ... ... @@ -139,6 +139,67 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
    139 139
         result.assert_success()
    
    140 140
     
    
    141 141
     
    
    142
    +@pytest.mark.datafiles(DATA_DIR)
    
    143
    +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
    
    144
    +def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
    
    145
    +    if kind == 'ostree':
    
    146
    +        # FIXME: Mirroring fallback fails with ostree
    
    147
    +        pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
    
    148
    +
    
    149
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    
    150
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    
    151
    +    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
    
    152
    +    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
    
    153
    +    project_dir = os.path.join(str(tmpdir), 'project')
    
    154
    +    os.makedirs(project_dir)
    
    155
    +    element_dir = os.path.join(project_dir, 'elements')
    
    156
    +
    
    157
    +    # Create repo objects of the upstream and mirror
    
    158
    +    upstream_repo = create_repo(kind, upstream_repodir)
    
    159
    +    ref = upstream_repo.create(dev_files_path)
    
    160
    +    mirror_repo = upstream_repo.copy(mirror_repodir)
    
    161
    +
    
    162
    +    element = {
    
    163
    +        'kind': 'import',
    
    164
    +        'sources': [
    
    165
    +            upstream_repo.source_config(ref=ref)
    
    166
    +        ]
    
    167
    +    }
    
    168
    +
    
    169
    +    element_name = 'test.bst'
    
    170
    +    element_path = os.path.join(element_dir, element_name)
    
    171
    +    full_repo = element['sources'][0]['url']
    
    172
    +    upstream_map, repo_name = os.path.split(full_repo)
    
    173
    +    alias = 'foo-' + kind
    
    174
    +    aliased_repo = alias + ':' + repo_name
    
    175
    +    element['sources'][0]['url'] = aliased_repo
    
    176
    +    full_mirror = mirror_repo.source_config()['url']
    
    177
    +    mirror_map, _ = os.path.split(full_mirror)
    
    178
    +    os.makedirs(element_dir)
    
    179
    +    _yaml.dump(element, element_path)
    
    180
    +
    
    181
    +    project = {
    
    182
    +        'name': 'test',
    
    183
    +        'element-path': 'elements',
    
    184
    +        'aliases': {
    
    185
    +            alias: 'http://www.example.com/'
    
    186
    +        },
    
    187
    +        'mirrors': [
    
    188
    +            {
    
    189
    +                'name': 'middle-earth',
    
    190
    +                'aliases': {
    
    191
    +                    alias: [mirror_map + "/"],
    
    192
    +                },
    
    193
    +            },
    
    194
    +        ]
    
    195
    +    }
    
    196
    +    project_file = os.path.join(project_dir, 'project.conf')
    
    197
    +    _yaml.dump(project, project_file)
    
    198
    +
    
    199
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    
    200
    +    result.assert_success()
    
    201
    +
    
    202
    +
    
    142 203
     @pytest.mark.datafiles(DATA_DIR)
    
    143 204
     def test_mirror_fetch_multi(cli, tmpdir, datafiles):
    
    144 205
         output_file = os.path.join(str(tmpdir), "output.txt")
    
    ... ... @@ -405,10 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
    405 466
     @pytest.mark.datafiles(DATA_DIR)
    
    406 467
     @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
    
    407 468
     def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
    
    408
    -    if kind == 'git':
    
    409
    -        # FIXME: Mirroring fallback does not work with git because it tries to
    
    410
    -        # fetch submodules on upstream.
    
    411
    -        pytest.skip("Bug #537 - Mirror fallback does not work for git")
    
    412 469
         if kind == 'ostree':
    
    413 470
             # FIXME: Mirroring fallback fails with ostree
    
    414 471
             pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
    
    ... ... @@ -559,3 +616,276 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
    559 616
         os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo)
    
    560 617
         result = cli.run(project=project_dir, args=['fetch', element_name])
    
    561 618
         result.assert_success()
    
    619
    +
    
    620
    +
    
    621
    +@pytest.mark.datafiles(DATA_DIR)
    
    622
    +def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles):
    
    623
    +    # Test that it behaves as expected with submodules, both defined in config
    
    624
    +    # and discovered when fetching.
    
    625
    +    foo_file = os.path.join(str(datafiles), 'files', 'foo')
    
    626
    +    bar_file = os.path.join(str(datafiles), 'files', 'bar')
    
    627
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    
    628
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    
    629
    +    mirror_dir = os.path.join(str(datafiles), 'mirror')
    
    630
    +
    
    631
    +    defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo')
    
    632
    +    defined_mirror_ref = defined_subrepo.create(bin_files_path)
    
    633
    +    defined_mirror = defined_subrepo.copy(mirror_dir)
    
    634
    +    defined_subref = defined_subrepo.add_file(foo_file)
    
    635
    +
    
    636
    +    found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo')
    
    637
    +    found_subref = found_subrepo.create(dev_files_path)
    
    638
    +
    
    639
    +    main_repo = create_repo('git', str(tmpdir))
    
    640
    +    main_mirror_ref = main_repo.create(bin_files_path)
    
    641
    +    main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo)
    
    642
    +    main_repo.add_submodule('found', 'file://' + found_subrepo.repo)
    
    643
    +    main_mirror = main_repo.copy(mirror_dir)
    
    644
    +    main_ref = main_repo.add_file(bar_file)
    
    645
    +
    
    646
    +    project_dir = os.path.join(str(tmpdir), 'project')
    
    647
    +    os.makedirs(project_dir)
    
    648
    +    element_dir = os.path.join(project_dir, 'elements')
    
    649
    +    os.makedirs(element_dir)
    
    650
    +    element = {
    
    651
    +        'kind': 'import',
    
    652
    +        'sources': [
    
    653
    +            main_repo.source_config(ref=main_mirror_ref)
    
    654
    +        ]
    
    655
    +    }
    
    656
    +    element_name = 'test.bst'
    
    657
    +    element_path = os.path.join(element_dir, element_name)
    
    658
    +
    
    659
    +    # Alias the main repo
    
    660
    +    full_repo = element['sources'][0]['url']
    
    661
    +    _, repo_name = os.path.split(full_repo)
    
    662
    +    alias = 'foo'
    
    663
    +    aliased_repo = alias + ':' + repo_name
    
    664
    +    element['sources'][0]['url'] = aliased_repo
    
    665
    +
    
    666
    +    # Hide the found subrepo
    
    667
    +    del element['sources'][0]['submodules']['found']
    
    668
    +
    
    669
    +    # Alias the defined subrepo
    
    670
    +    subrepo = element['sources'][0]['submodules']['defined']['url']
    
    671
    +    _, repo_name = os.path.split(subrepo)
    
    672
    +    aliased_repo = alias + ':' + repo_name
    
    673
    +    element['sources'][0]['submodules']['defined']['url'] = aliased_repo
    
    674
    +
    
    675
    +    _yaml.dump(element, element_path)
    
    676
    +
    
    677
    +    full_mirror = main_mirror.source_config()['url']
    
    678
    +    mirror_map, _ = os.path.split(full_mirror)
    
    679
    +    project = {
    
    680
    +        'name': 'test',
    
    681
    +        'element-path': 'elements',
    
    682
    +        'aliases': {
    
    683
    +            alias: 'http://www.example.com/'
    
    684
    +        },
    
    685
    +        'mirrors': [
    
    686
    +            {
    
    687
    +                'name': 'middle-earth',
    
    688
    +                'aliases': {
    
    689
    +                    alias: [mirror_map + "/"],
    
    690
    +                },
    
    691
    +            },
    
    692
    +        ]
    
    693
    +    }
    
    694
    +    project_file = os.path.join(project_dir, 'project.conf')
    
    695
    +    _yaml.dump(project, project_file)
    
    696
    +
    
    697
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    
    698
    +    result.assert_success()
    
    699
    +
    
    700
    +
    
    701
    +@pytest.mark.datafiles(DATA_DIR)
    
    702
    +def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles):
    
    703
    +    # Main repo has no mirror or alias.
    
    704
    +    # One submodule is overridden to use a mirror.
    
    705
    +    # There is another submodules not overriden.
    
    706
    +    # Upstream for overriden submodule is down.
    
    707
    +    #
    
    708
    +    # We expect:
    
    709
    +    #  - overriden submodule is fetched from mirror.
    
    710
    +    #  - other submodule is fetched.
    
    711
    +
    
    712
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    
    713
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    
    714
    +
    
    715
    +    upstream_bin_repodir = os.path.join(str(tmpdir), 'bin-upstream')
    
    716
    +    mirror_bin_repodir = os.path.join(str(tmpdir), 'bin-mirror')
    
    717
    +    upstream_bin_repo = create_repo('git', upstream_bin_repodir)
    
    718
    +    upstream_bin_repo.create(bin_files_path)
    
    719
    +    mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir)
    
    720
    +
    
    721
    +    dev_repodir = os.path.join(str(tmpdir), 'dev-upstream')
    
    722
    +    dev_repo = create_repo('git', dev_repodir)
    
    723
    +    dev_repo.create(dev_files_path)
    
    724
    +
    
    725
    +    main_files = os.path.join(str(tmpdir), 'main-files')
    
    726
    +    os.makedirs(main_files)
    
    727
    +    with open(os.path.join(main_files, 'README'), 'w') as f:
    
    728
    +        f.write("TEST\n")
    
    729
    +    main_repodir = os.path.join(str(tmpdir), 'main-upstream')
    
    730
    +    main_repo = create_repo('git', main_repodir)
    
    731
    +    main_repo.create(main_files)
    
    732
    +
    
    733
    +    upstream_url = 'file://{}'.format(upstream_bin_repo.repo)
    
    734
    +    main_repo.add_submodule('bin', url=upstream_url)
    
    735
    +    main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
    
    736
    +    # Unlist 'dev'.
    
    737
    +    del main_repo.submodules['dev']
    
    738
    +
    
    739
    +    main_ref = main_repo.latest_commit()
    
    740
    +
    
    741
    +    upstream_map, repo_name = os.path.split(upstream_url)
    
    742
    +    alias = 'foo'
    
    743
    +    aliased_repo = '{}:{}'.format(alias, repo_name)
    
    744
    +    main_repo.submodules['bin']['url'] = aliased_repo
    
    745
    +
    
    746
    +    full_mirror = mirror_bin_repo.source_config()['url']
    
    747
    +    mirror_map, _ = os.path.split(full_mirror)
    
    748
    +
    
    749
    +    project_dir = os.path.join(str(tmpdir), 'project')
    
    750
    +    os.makedirs(project_dir)
    
    751
    +    element_dir = os.path.join(project_dir, 'elements')
    
    752
    +
    
    753
    +    element = {
    
    754
    +        'kind': 'import',
    
    755
    +        'sources': [
    
    756
    +            main_repo.source_config(ref=main_ref, checkout_submodules=True)
    
    757
    +        ]
    
    758
    +    }
    
    759
    +    element_name = 'test.bst'
    
    760
    +    element_path = os.path.join(element_dir, element_name)
    
    761
    +    os.makedirs(element_dir)
    
    762
    +    _yaml.dump(element, element_path)
    
    763
    +
    
    764
    +    project = {
    
    765
    +        'name': 'test',
    
    766
    +        'element-path': 'elements',
    
    767
    +        'aliases': {
    
    768
    +            alias: upstream_map + "/"
    
    769
    +        },
    
    770
    +        'mirrors': [
    
    771
    +            {
    
    772
    +                'name': 'middle-earth',
    
    773
    +                'aliases': {
    
    774
    +                    alias: [mirror_map + "/"],
    
    775
    +                }
    
    776
    +            }
    
    777
    +        ]
    
    778
    +    }
    
    779
    +    project_file = os.path.join(project_dir, 'project.conf')
    
    780
    +    _yaml.dump(project, project_file)
    
    781
    +
    
    782
    +    # Now make the upstream unavailable.
    
    783
    +    os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo))
    
    784
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    
    785
    +    result.assert_success()
    
    786
    +
    
    787
    +    result = cli.run(project=project_dir, args=['build', element_name])
    
    788
    +    result.assert_success()
    
    789
    +
    
    790
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    
    791
    +    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
    
    792
    +    result.assert_success()
    
    793
    +
    
    794
    +    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
    
    795
    +    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
    
    796
    +
    
    797
    +
    
    798
    +@pytest.mark.datafiles(DATA_DIR)
    
    799
    +def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles):
    
    800
    +    # Main repo has mirror. But does not list submodules.
    
    801
    +    #
    
    802
    +    # We expect:
    
    803
    +    #  - we will fetch submodules anyway
    
    804
    +
    
    805
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    
    806
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    
    807
    +
    
    808
    +    bin_repodir = os.path.join(str(tmpdir), 'bin-repo')
    
    809
    +    bin_repo = create_repo('git', bin_repodir)
    
    810
    +    bin_repo.create(bin_files_path)
    
    811
    +
    
    812
    +    dev_repodir = os.path.join(str(tmpdir), 'dev-repo')
    
    813
    +    dev_repo = create_repo('git', dev_repodir)
    
    814
    +    dev_repo.create(dev_files_path)
    
    815
    +
    
    816
    +    main_files = os.path.join(str(tmpdir), 'main-files')
    
    817
    +    os.makedirs(main_files)
    
    818
    +    with open(os.path.join(main_files, 'README'), 'w') as f:
    
    819
    +        f.write("TEST\n")
    
    820
    +    upstream_main_repodir = os.path.join(str(tmpdir), 'main-upstream')
    
    821
    +    upstream_main_repo = create_repo('git', upstream_main_repodir)
    
    822
    +    upstream_main_repo.create(main_files)
    
    823
    +
    
    824
    +    upstream_main_repo.add_submodule('bin', url='file://{}'.format(bin_repo.repo))
    
    825
    +    upstream_main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
    
    826
    +    # Unlist submodules.
    
    827
    +    del upstream_main_repo.submodules['bin']
    
    828
    +    del upstream_main_repo.submodules['dev']
    
    829
    +
    
    830
    +    upstream_main_ref = upstream_main_repo.latest_commit()
    
    831
    +
    
    832
    +    mirror_main_repodir = os.path.join(str(tmpdir), 'main-mirror')
    
    833
    +    mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir)
    
    834
    +
    
    835
    +    upstream_url = mirror_main_repo.source_config()['url']
    
    836
    +
    
    837
    +    upstream_map, repo_name = os.path.split(upstream_url)
    
    838
    +    alias = 'foo'
    
    839
    +    aliased_repo = '{}:{}'.format(alias, repo_name)
    
    840
    +
    
    841
    +    full_mirror = mirror_main_repo.source_config()['url']
    
    842
    +    mirror_map, _ = os.path.split(full_mirror)
    
    843
    +
    
    844
    +    project_dir = os.path.join(str(tmpdir), 'project')
    
    845
    +    os.makedirs(project_dir)
    
    846
    +    element_dir = os.path.join(project_dir, 'elements')
    
    847
    +
    
    848
    +    element = {
    
    849
    +        'kind': 'import',
    
    850
    +        'sources': [
    
    851
    +            upstream_main_repo.source_config(ref=upstream_main_ref, checkout_submodules=True)
    
    852
    +        ]
    
    853
    +    }
    
    854
    +    element['sources'][0]['url'] = aliased_repo
    
    855
    +    element_name = 'test.bst'
    
    856
    +    element_path = os.path.join(element_dir, element_name)
    
    857
    +    os.makedirs(element_dir)
    
    858
    +    _yaml.dump(element, element_path)
    
    859
    +
    
    860
    +    project = {
    
    861
    +        'name': 'test',
    
    862
    +        'element-path': 'elements',
    
    863
    +        'aliases': {
    
    864
    +            alias: upstream_map + "/"
    
    865
    +        },
    
    866
    +        'mirrors': [
    
    867
    +            {
    
    868
    +                'name': 'middle-earth',
    
    869
    +                'aliases': {
    
    870
    +                    alias: [mirror_map + "/"],
    
    871
    +                }
    
    872
    +            }
    
    873
    +        ]
    
    874
    +    }
    
    875
    +    project_file = os.path.join(project_dir, 'project.conf')
    
    876
    +    _yaml.dump(project, project_file)
    
    877
    +
    
    878
    +    # Now make the upstream unavailable.
    
    879
    +    os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo))
    
    880
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    
    881
    +    result.assert_success()
    
    882
    +
    
    883
    +    result = cli.run(project=project_dir, args=['build', element_name])
    
    884
    +    result.assert_success()
    
    885
    +
    
    886
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    
    887
    +    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
    
    888
    +    result.assert_success()
    
    889
    +
    
    890
    +    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
    
    891
    +    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))

  • tests/frontend/project/files/bar

  • tests/frontend/project/files/foo



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]