GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/fallback_mirror_ostree-1.2] 8 commits: buildstream/plugins/sources/git.py: Fix clone side effect in getting fetchers

Title: GitLab

Valentin David pushed to branch valentindavid/fallback_mirror_ostree-1.2 at BuildStream / buildstream

Commits:

5 changed files:

Changes:

  • buildstream/plugins/sources/git.py
    ... ... @@ -404,8 +404,10 @@ class GitSource(Source):
    404 404 
                         mirror.stage(directory)
    405 405
    406 406 
         def get_source_fetchers(self):
    407 
    +        yield self.mirror
    407 408 
             self.refresh_submodules()
    408 
    -        return [self.mirror] + self.submodules
    409 
    +        for submodule in self.submodules:
    410 
    +            yield submodule
    409 411
    410 412 
         ###########################################################
    411 413 
         #                     Local Functions                     #

  • buildstream/plugins/sources/ostree.py
    ... ... @@ -71,7 +71,7 @@ class OSTreeSource(Source):
    71 71 
             self.ref = self.node_get_member(node, str, 'ref', None)
    72 72 
             self.tracking = self.node_get_member(node, str, 'track', None)
    73 73 
             self.mirror = os.path.join(self.get_mirror_directory(),
    74 
    -                                   utils.url_directory_name(self.url))
    74 
    +                                   utils.url_directory_name(self.original_url))
    75 75
    76 76 
             # (optional) Not all repos are signed. But if they are, get the gpg key
    77 77 
             self.gpg_key_path = None
    ... ... @@ -104,10 +104,11 @@ class OSTreeSource(Source):
    104 104 
                 return None
    105 105
    106 106 
             self.ensure()
    107 
    +        remote_name = self.ensure_remote(self.url)
    107 108 
             with self.timed_activity("Fetching tracking ref '{}' from origin: {}"
    108 109 
                                      .format(self.tracking, self.url)):
    109 110 
                 try:
    110 
    -                _ostree.fetch(self.repo, ref=self.tracking, progress=self.progress)
    111 
    +                _ostree.fetch(self.repo, remote=remote_name, ref=self.tracking, progress=self.progress)
    111 112 
                 except OSTreeError as e:
    112 113 
                     raise SourceError("{}: Failed to fetch tracking ref '{}' from origin {}\n\n{}"
    113 114 
                                       .format(self, self.tracking, self.url, e)) from e
    ... ... @@ -116,11 +117,12 @@ class OSTreeSource(Source):
    116 117
    117 118 
         def fetch(self):
    118 119 
             self.ensure()
    120 
    +        remote_name = self.ensure_remote(self.url)
    119 121 
             if not _ostree.exists(self.repo, self.ref):
    120 122 
                 with self.timed_activity("Fetching remote ref: {} from origin: {}"
    121 123 
                                          .format(self.ref, self.url)):
    122 124 
                     try:
    123 
    -                    _ostree.fetch(self.repo, ref=self.ref, progress=self.progress)
    125 
    +                    _ostree.fetch(self.repo, remote=remote_name, ref=self.ref, progress=self.progress)
    124 126 
                     except OSTreeError as e:
    125 127 
                         raise SourceError("{}: Failed to fetch ref '{}' from origin: {}\n\n{}"
    126 128 
                                           .format(self, self.ref, self.url, e)) from e
    ... ... @@ -171,14 +173,22 @@ class OSTreeSource(Source):
    171 173 
                 self.status("Creating local mirror for {}".format(self.url))
    172 174
    173 175 
                 self.repo = _ostree.ensure(self.mirror, True)
    174 
    -            gpg_key = None
    175 
    -            if self.gpg_key_path:
    176 
    -                gpg_key = 'file://' + self.gpg_key_path
    177 176
    178 
    -            try:
    179 
    -                _ostree.configure_remote(self.repo, "origin", self.url, key_url=gpg_key)
    180 
    -            except OSTreeError as e:
    181 
    -                raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
    177 
    +    def ensure_remote(self, url):
    178 
    +        if self.original_url == self.url:
    179 
    +            remote_name = 'origin'
    180 
    +        else:
    181 
    +            remote_name = utils.url_directory_name(url)
    182 
    +
    183 
    +        gpg_key = None
    184 
    +        if self.gpg_key_path:
    185 
    +            gpg_key = 'file://' + self.gpg_key_path
    186 
    +
    187 
    +        try:
    188 
    +            _ostree.configure_remote(self.repo, remote_name, url, key_url=gpg_key)
    189 
    +        except OSTreeError as e:
    190 
    +            raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
    191 
    +        return remote_name
    182 192
    183 193 
         def progress(self, percent, message):
    184 194 
             self.status(message)

  • tests/frontend/mirror.py
    ... ... @@ -139,6 +139,67 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
    139 139 
         result.assert_success()
    140 140
    141 141
    142 
    +@pytest.mark.datafiles(DATA_DIR)
    143 
    +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
    144 
    +def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
    145 
    +    if kind == 'ostree':
    146 
    +        # FIXME: Mirroring fallback fails with ostree
    147 
    +        pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
    148 
    +
    149 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    150 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    151 
    +    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
    152 
    +    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
    153 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    154 
    +    os.makedirs(project_dir)
    155 
    +    element_dir = os.path.join(project_dir, 'elements')
    156 
    +
    157 
    +    # Create repo objects of the upstream and mirror
    158 
    +    upstream_repo = create_repo(kind, upstream_repodir)
    159 
    +    ref = upstream_repo.create(dev_files_path)
    160 
    +    mirror_repo = upstream_repo.copy(mirror_repodir)
    161 
    +
    162 
    +    element = {
    163 
    +        'kind': 'import',
    164 
    +        'sources': [
    165 
    +            upstream_repo.source_config(ref=ref)
    166 
    +        ]
    167 
    +    }
    168 
    +
    169 
    +    element_name = 'test.bst'
    170 
    +    element_path = os.path.join(element_dir, element_name)
    171 
    +    full_repo = element['sources'][0]['url']
    172 
    +    upstream_map, repo_name = os.path.split(full_repo)
    173 
    +    alias = 'foo-' + kind
    174 
    +    aliased_repo = alias + ':' + repo_name
    175 
    +    element['sources'][0]['url'] = aliased_repo
    176 
    +    full_mirror = mirror_repo.source_config()['url']
    177 
    +    mirror_map, _ = os.path.split(full_mirror)
    178 
    +    os.makedirs(element_dir)
    179 
    +    _yaml.dump(element, element_path)
    180 
    +
    181 
    +    project = {
    182 
    +        'name': 'test',
    183 
    +        'element-path': 'elements',
    184 
    +        'aliases': {
    185 
    +            alias: 'http://www.example.com/'
    186 
    +        },
    187 
    +        'mirrors': [
    188 
    +            {
    189 
    +                'name': 'middle-earth',
    190 
    +                'aliases': {
    191 
    +                    alias: [mirror_map + "/"],
    192 
    +                },
    193 
    +            },
    194 
    +        ]
    195 
    +    }
    196 
    +    project_file = os.path.join(project_dir, 'project.conf')
    197 
    +    _yaml.dump(project, project_file)
    198 
    +
    199 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    200 
    +    result.assert_success()
    201 
    +
    202 
    +
    142 203 
     @pytest.mark.datafiles(DATA_DIR)
    143 204 
     def test_mirror_fetch_multi(cli, tmpdir, datafiles):
    144 205 
         output_file = os.path.join(str(tmpdir), "output.txt")
    ... ... @@ -405,14 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
    405 466 
     @pytest.mark.datafiles(DATA_DIR)
    406 467 
     @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
    407 468 
     def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
    408 
    -    if kind == 'git':
    409 
    -        # FIXME: Mirroring fallback does not work with git because it tries to
    410 
    -        # fetch submodules on upstream.
    411 
    -        pytest.skip("Bug #537 - Mirror fallback does not work for git")
    412 
    -    if kind == 'ostree':
    413 
    -        # FIXME: Mirroring fallback fails with ostree
    414 
    -        pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
    415 
    -
    416 469 
         bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    417 470 
         upstream_repodir = os.path.join(str(tmpdir), 'upstream')
    418 471 
         mirror_repodir = os.path.join(str(tmpdir), 'mirror')
    ... ... @@ -559,3 +612,276 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
    559 612 
         os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo)
    560 613 
         result = cli.run(project=project_dir, args=['fetch', element_name])
    561 614 
         result.assert_success()
    615 
    +
    616 
    +
    617 
    +@pytest.mark.datafiles(DATA_DIR)
    618 
    +def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles):
    619 
    +    # Test that it behaves as expected with submodules, both defined in config
    620 
    +    # and discovered when fetching.
    621 
    +    foo_file = os.path.join(str(datafiles), 'files', 'foo')
    622 
    +    bar_file = os.path.join(str(datafiles), 'files', 'bar')
    623 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    624 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    625 
    +    mirror_dir = os.path.join(str(datafiles), 'mirror')
    626 
    +
    627 
    +    defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo')
    628 
    +    defined_mirror_ref = defined_subrepo.create(bin_files_path)
    629 
    +    defined_mirror = defined_subrepo.copy(mirror_dir)
    630 
    +    defined_subref = defined_subrepo.add_file(foo_file)
    631 
    +
    632 
    +    found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo')
    633 
    +    found_subref = found_subrepo.create(dev_files_path)
    634 
    +
    635 
    +    main_repo = create_repo('git', str(tmpdir))
    636 
    +    main_mirror_ref = main_repo.create(bin_files_path)
    637 
    +    main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo)
    638 
    +    main_repo.add_submodule('found', 'file://' + found_subrepo.repo)
    639 
    +    main_mirror = main_repo.copy(mirror_dir)
    640 
    +    main_ref = main_repo.add_file(bar_file)
    641 
    +
    642 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    643 
    +    os.makedirs(project_dir)
    644 
    +    element_dir = os.path.join(project_dir, 'elements')
    645 
    +    os.makedirs(element_dir)
    646 
    +    element = {
    647 
    +        'kind': 'import',
    648 
    +        'sources': [
    649 
    +            main_repo.source_config(ref=main_mirror_ref)
    650 
    +        ]
    651 
    +    }
    652 
    +    element_name = 'test.bst'
    653 
    +    element_path = os.path.join(element_dir, element_name)
    654 
    +
    655 
    +    # Alias the main repo
    656 
    +    full_repo = element['sources'][0]['url']
    657 
    +    _, repo_name = os.path.split(full_repo)
    658 
    +    alias = 'foo'
    659 
    +    aliased_repo = alias + ':' + repo_name
    660 
    +    element['sources'][0]['url'] = aliased_repo
    661 
    +
    662 
    +    # Hide the found subrepo
    663 
    +    del element['sources'][0]['submodules']['found']
    664 
    +
    665 
    +    # Alias the defined subrepo
    666 
    +    subrepo = element['sources'][0]['submodules']['defined']['url']
    667 
    +    _, repo_name = os.path.split(subrepo)
    668 
    +    aliased_repo = alias + ':' + repo_name
    669 
    +    element['sources'][0]['submodules']['defined']['url'] = aliased_repo
    670 
    +
    671 
    +    _yaml.dump(element, element_path)
    672 
    +
    673 
    +    full_mirror = main_mirror.source_config()['url']
    674 
    +    mirror_map, _ = os.path.split(full_mirror)
    675 
    +    project = {
    676 
    +        'name': 'test',
    677 
    +        'element-path': 'elements',
    678 
    +        'aliases': {
    679 
    +            alias: 'http://www.example.com/'
    680 
    +        },
    681 
    +        'mirrors': [
    682 
    +            {
    683 
    +                'name': 'middle-earth',
    684 
    +                'aliases': {
    685 
    +                    alias: [mirror_map + "/"],
    686 
    +                },
    687 
    +            },
    688 
    +        ]
    689 
    +    }
    690 
    +    project_file = os.path.join(project_dir, 'project.conf')
    691 
    +    _yaml.dump(project, project_file)
    692 
    +
    693 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    694 
    +    result.assert_success()
    695 
    +
    696 
    +
    697 
    +@pytest.mark.datafiles(DATA_DIR)
    698 
    +def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles):
    699 
    +    # Main repo has no mirror or alias.
    700 
    +    # One submodule is overridden to use a mirror.
    701 
    +    # There is another submodules not overriden.
    702 
    +    # Upstream for overriden submodule is down.
    703 
    +    #
    704 
    +    # We expect:
    705 
    +    #  - overriden submodule is fetched from mirror.
    706 
    +    #  - other submodule is fetched.
    707 
    +
    708 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    709 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    710 
    +
    711 
    +    upstream_bin_repodir = os.path.join(str(tmpdir), 'bin-upstream')
    712 
    +    mirror_bin_repodir = os.path.join(str(tmpdir), 'bin-mirror')
    713 
    +    upstream_bin_repo = create_repo('git', upstream_bin_repodir)
    714 
    +    upstream_bin_repo.create(bin_files_path)
    715 
    +    mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir)
    716 
    +
    717 
    +    dev_repodir = os.path.join(str(tmpdir), 'dev-upstream')
    718 
    +    dev_repo = create_repo('git', dev_repodir)
    719 
    +    dev_repo.create(dev_files_path)
    720 
    +
    721 
    +    main_files = os.path.join(str(tmpdir), 'main-files')
    722 
    +    os.makedirs(main_files)
    723 
    +    with open(os.path.join(main_files, 'README'), 'w') as f:
    724 
    +        f.write("TEST\n")
    725 
    +    main_repodir = os.path.join(str(tmpdir), 'main-upstream')
    726 
    +    main_repo = create_repo('git', main_repodir)
    727 
    +    main_repo.create(main_files)
    728 
    +
    729 
    +    upstream_url = 'file://{}'.format(upstream_bin_repo.repo)
    730 
    +    main_repo.add_submodule('bin', url=upstream_url)
    731 
    +    main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
    732 
    +    # Unlist 'dev'.
    733 
    +    del main_repo.submodules['dev']
    734 
    +
    735 
    +    main_ref = main_repo.latest_commit()
    736 
    +
    737 
    +    upstream_map, repo_name = os.path.split(upstream_url)
    738 
    +    alias = 'foo'
    739 
    +    aliased_repo = '{}:{}'.format(alias, repo_name)
    740 
    +    main_repo.submodules['bin']['url'] = aliased_repo
    741 
    +
    742 
    +    full_mirror = mirror_bin_repo.source_config()['url']
    743 
    +    mirror_map, _ = os.path.split(full_mirror)
    744 
    +
    745 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    746 
    +    os.makedirs(project_dir)
    747 
    +    element_dir = os.path.join(project_dir, 'elements')
    748 
    +
    749 
    +    element = {
    750 
    +        'kind': 'import',
    751 
    +        'sources': [
    752 
    +            main_repo.source_config(ref=main_ref, checkout_submodules=True)
    753 
    +        ]
    754 
    +    }
    755 
    +    element_name = 'test.bst'
    756 
    +    element_path = os.path.join(element_dir, element_name)
    757 
    +    os.makedirs(element_dir)
    758 
    +    _yaml.dump(element, element_path)
    759 
    +
    760 
    +    project = {
    761 
    +        'name': 'test',
    762 
    +        'element-path': 'elements',
    763 
    +        'aliases': {
    764 
    +            alias: upstream_map + "/"
    765 
    +        },
    766 
    +        'mirrors': [
    767 
    +            {
    768 
    +                'name': 'middle-earth',
    769 
    +                'aliases': {
    770 
    +                    alias: [mirror_map + "/"],
    771 
    +                }
    772 
    +            }
    773 
    +        ]
    774 
    +    }
    775 
    +    project_file = os.path.join(project_dir, 'project.conf')
    776 
    +    _yaml.dump(project, project_file)
    777 
    +
    778 
    +    # Now make the upstream unavailable.
    779 
    +    os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo))
    780 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    781 
    +    result.assert_success()
    782 
    +
    783 
    +    result = cli.run(project=project_dir, args=['build', element_name])
    784 
    +    result.assert_success()
    785 
    +
    786 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    787 
    +    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
    788 
    +    result.assert_success()
    789 
    +
    790 
    +    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
    791 
    +    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
    792 
    +
    793 
    +
    794 
    +@pytest.mark.datafiles(DATA_DIR)
    795 
    +def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles):
    796 
    +    # Main repo has mirror. But does not list submodules.
    797 
    +    #
    798 
    +    # We expect:
    799 
    +    #  - we will fetch submodules anyway
    800 
    +
    801 
    +    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
    802 
    +    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
    803 
    +
    804 
    +    bin_repodir = os.path.join(str(tmpdir), 'bin-repo')
    805 
    +    bin_repo = create_repo('git', bin_repodir)
    806 
    +    bin_repo.create(bin_files_path)
    807 
    +
    808 
    +    dev_repodir = os.path.join(str(tmpdir), 'dev-repo')
    809 
    +    dev_repo = create_repo('git', dev_repodir)
    810 
    +    dev_repo.create(dev_files_path)
    811 
    +
    812 
    +    main_files = os.path.join(str(tmpdir), 'main-files')
    813 
    +    os.makedirs(main_files)
    814 
    +    with open(os.path.join(main_files, 'README'), 'w') as f:
    815 
    +        f.write("TEST\n")
    816 
    +    upstream_main_repodir = os.path.join(str(tmpdir), 'main-upstream')
    817 
    +    upstream_main_repo = create_repo('git', upstream_main_repodir)
    818 
    +    upstream_main_repo.create(main_files)
    819 
    +
    820 
    +    upstream_main_repo.add_submodule('bin', url='file://{}'.format(bin_repo.repo))
    821 
    +    upstream_main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
    822 
    +    # Unlist submodules.
    823 
    +    del upstream_main_repo.submodules['bin']
    824 
    +    del upstream_main_repo.submodules['dev']
    825 
    +
    826 
    +    upstream_main_ref = upstream_main_repo.latest_commit()
    827 
    +
    828 
    +    mirror_main_repodir = os.path.join(str(tmpdir), 'main-mirror')
    829 
    +    mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir)
    830 
    +
    831 
    +    upstream_url = mirror_main_repo.source_config()['url']
    832 
    +
    833 
    +    upstream_map, repo_name = os.path.split(upstream_url)
    834 
    +    alias = 'foo'
    835 
    +    aliased_repo = '{}:{}'.format(alias, repo_name)
    836 
    +
    837 
    +    full_mirror = mirror_main_repo.source_config()['url']
    838 
    +    mirror_map, _ = os.path.split(full_mirror)
    839 
    +
    840 
    +    project_dir = os.path.join(str(tmpdir), 'project')
    841 
    +    os.makedirs(project_dir)
    842 
    +    element_dir = os.path.join(project_dir, 'elements')
    843 
    +
    844 
    +    element = {
    845 
    +        'kind': 'import',
    846 
    +        'sources': [
    847 
    +            upstream_main_repo.source_config(ref=upstream_main_ref, checkout_submodules=True)
    848 
    +        ]
    849 
    +    }
    850 
    +    element['sources'][0]['url'] = aliased_repo
    851 
    +    element_name = 'test.bst'
    852 
    +    element_path = os.path.join(element_dir, element_name)
    853 
    +    os.makedirs(element_dir)
    854 
    +    _yaml.dump(element, element_path)
    855 
    +
    856 
    +    project = {
    857 
    +        'name': 'test',
    858 
    +        'element-path': 'elements',
    859 
    +        'aliases': {
    860 
    +            alias: upstream_map + "/"
    861 
    +        },
    862 
    +        'mirrors': [
    863 
    +            {
    864 
    +                'name': 'middle-earth',
    865 
    +                'aliases': {
    866 
    +                    alias: [mirror_map + "/"],
    867 
    +                }
    868 
    +            }
    869 
    +        ]
    870 
    +    }
    871 
    +    project_file = os.path.join(project_dir, 'project.conf')
    872 
    +    _yaml.dump(project, project_file)
    873 
    +
    874 
    +    # Now make the upstream unavailable.
    875 
    +    os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo))
    876 
    +    result = cli.run(project=project_dir, args=['fetch', element_name])
    877 
    +    result.assert_success()
    878 
    +
    879 
    +    result = cli.run(project=project_dir, args=['build', element_name])
    880 
    +    result.assert_success()
    881 
    +
    882 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    883 
    +    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
    884 
    +    result.assert_success()
    885 
    +
    886 
    +    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
    887 
    +    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))

  • tests/frontend/project/files/bar

  • tests/frontend/project/files/foo

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]