[Notes] [Git][BuildStream/buildstream][valentindavid/git_shallow_fetch] Fetch git shallow clone when possible



Title: GitLab

Valentin David pushed to branch valentindavid/git_shallow_fetch at BuildStream / buildstream

Commits:

2 changed files:

Changes:

  • buildstream/plugins/sources/git.py
    ... ... @@ -183,7 +183,7 @@ WARN_INVALID_SUBMODULE = "invalid-submodule"
    183 183
     #
    
    184 184
     class GitMirror(SourceFetcher):
    
    185 185
     
    
    186
    -    def __init__(self, source, path, url, ref, *, primary=False, tags=[]):
    
    186
    +    def __init__(self, source, path, url, ref, *, primary=False, tags=[], tracking=None):
    
    187 187
     
    
    188 188
             super().__init__()
    
    189 189
             self.source = source
    
    ... ... @@ -192,11 +192,101 @@ class GitMirror(SourceFetcher):
    192 192
             self.ref = ref
    
    193 193
             self.tags = tags
    
    194 194
             self.primary = primary
    
    195
    +        dirname = utils.url_directory_name(url)
    
    195 196
             self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
    
    197
    +        self.fetch_mirror = os.path.join(source.get_mirror_directory(), '{}-{}'.format(dirname, ref))
    
    196 198
             self.mark_download_url(url)
    
    199
    +        self.tracking = tracking
    
    200
    +
    
    201
    +    def mirror_path(self):
    
    202
    +        if os.path.exists(self.mirror):
    
    203
    +            return self.mirror
    
    204
    +        else:
    
    205
    +            assert os.path.exists(self.fetch_mirror)
    
    206
    +            return self.fetch_mirror
    
    207
    +
    
    208
    +    def ensure_fetchable(self, alias_override=None):
    
    209
    +
    
    210
    +        if os.path.exists(self.mirror):
    
    211
    +            return
    
    212
    +
    
    213
    +        if self.tags:
    
    214
    +            for tag, commit, _ in self.tags:
    
    215
    +                if commit != self.ref:
    
    216
    +                    self.source.status("{}: tag '{}' is not on commit '{}', so a full clone is required"
    
    217
    +                                       .format(self.source, tag, commit))
    
    218
    +                    self.ensure_trackable(alias_override=alias_override)
    
    219
    +                    return
    
    220
    +
    
    221
    +        if os.path.exists(self.fetch_mirror):
    
    222
    +            return
    
    223
    +
    
    224
    +        with self.source.tempdir() as tmpdir:
    
    225
    +            self.source.call([self.source.host_git, 'init', '--bare', tmpdir],
    
    226
    +                             fail="Failed to init git repository",
    
    227
    +                             fail_temporarily=True)
    
    228
    +
    
    229
    +            url = self.source.translate_url(self.url, alias_override=alias_override,
    
    230
    +                                            primary=self.primary)
    
    231
    +
    
    232
    +            self.source.call([self.source.host_git, 'remote', 'add', '--mirror=fetch', 'origin', url],
    
    233
    +                             cwd=tmpdir,
    
    234
    +                             fail="Failed to init git repository",
    
    235
    +                             fail_temporarily=True)
    
    236
    +
    
    237
    +            _, refs = self.source.check_output([self.source.host_git, 'ls-remote', 'origin'],
    
    238
    +                                               cwd=tmpdir,
    
    239
    +                                               fail="Failed to clone git repository {}".format(url),
    
    240
    +                                               fail_temporarily=True)
    
    241
    +
    
    242
    +            advertised = None
    
    243
    +            for ref_line in refs.splitlines():
    
    244
    +                commit, ref = ref_line.split('\t', 1)
    
    245
    +                if ref == 'HEAD':
    
    246
    +                    continue
    
    247
    +                if self.tracking:
    
    248
    +                    # For validate_cache to work
    
    249
    +                    if ref not in ['refs/heads/{}'.format(self.tracking),
    
    250
    +                                   'refs/tags/{}'.format(self.tracking),
    
    251
    +                                   'refs/tags/{}{}'.format(self.tracking, '^{}')]:
    
    252
    +                        continue
    
    253
    +                if self.ref == commit:
    
    254
    +                    if ref.endswith('^{}'):
    
    255
    +                        ref = ref[:-3]
    
    256
    +                    advertised = ref
    
    257
    +                    break
    
    258
    +
    
    259
    +            if advertised is None:
    
    260
    +                self.source.status("{}: {} is not advertised on {}, so a full clone is required"
    
    261
    +                                   .format(self.source, self.ref, url))
    
    262
    +
    
    263
    +                self.ensure_trackable(alias_override=alias_override)
    
    264
    +                return
    
    265
    +
    
    266
    +            self.source.call([self.source.host_git, 'fetch', '--depth=1', 'origin', advertised],
    
    267
    +                             cwd=tmpdir,
    
    268
    +                             fail="Failed to fetch repository",
    
    269
    +                             fail_temporarily=True)
    
    270
    +
    
    271
    +            # We need to have a ref to make it clonable
    
    272
    +            self.source.call([self.source.host_git, 'update-ref', 'HEAD', self.ref],
    
    273
    +                             cwd=tmpdir,
    
    274
    +                             fail="Failed to tag HEAD",
    
    275
    +                             fail_temporarily=True)
    
    276
    +
    
    277
    +            try:
    
    278
    +                move_atomic(tmpdir, self.fetch_mirror)
    
    279
    +            except DirectoryExistsError:
    
    280
    +                # Another process was quicker to download this repository.
    
    281
    +                # Let's discard our own
    
    282
    +                self.source.status("{}: Discarding duplicate clone of {}"
    
    283
    +                                   .format(self.source, url))
    
    284
    +            except OSError as e:
    
    285
    +                raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}': {}"
    
    286
    +                                  .format(self.source, url, tmpdir, self.fetch_mirror, e)) from e
    
    197 287
     
    
    198 288
         # Ensures that the mirror exists
    
    199
    -    def ensure(self, alias_override=None):
    
    289
    +    def ensure_trackable(self, alias_override=None):
    
    200 290
     
    
    201 291
             # Unfortunately, git does not know how to only clone just a specific ref,
    
    202 292
             # so we have to download all of those gigs even if we only need a couple
    
    ... ... @@ -231,18 +321,20 @@ class GitMirror(SourceFetcher):
    231 321
                                             alias_override=alias_override,
    
    232 322
                                             primary=self.primary)
    
    233 323
     
    
    324
    +        mirror = self.mirror_path()
    
    325
    +
    
    234 326
             if alias_override:
    
    235 327
                 remote_name = utils.url_directory_name(alias_override)
    
    236 328
                 _, remotes = self.source.check_output(
    
    237 329
                     [self.source.host_git, 'remote'],
    
    238
    -                fail="Failed to retrieve list of remotes in {}".format(self.mirror),
    
    239
    -                cwd=self.mirror
    
    330
    +                fail="Failed to retrieve list of remotes in {}".format(mirror),
    
    331
    +                cwd=mirror
    
    240 332
                 )
    
    241 333
                 if remote_name not in remotes:
    
    242 334
                     self.source.call(
    
    243 335
                         [self.source.host_git, 'remote', 'add', remote_name, url],
    
    244 336
                         fail="Failed to add remote {} with url {}".format(remote_name, url),
    
    245
    -                    cwd=self.mirror
    
    337
    +                    cwd=mirror
    
    246 338
                     )
    
    247 339
             else:
    
    248 340
                 remote_name = "origin"
    
    ... ... @@ -250,7 +342,7 @@ class GitMirror(SourceFetcher):
    250 342
             self.source.call([self.source.host_git, 'fetch', remote_name, '--prune', '--force', '--tags'],
    
    251 343
                              fail="Failed to fetch from remote git repository: {}".format(url),
    
    252 344
                              fail_temporarily=True,
    
    253
    -                         cwd=self.mirror)
    
    345
    +                         cwd=mirror)
    
    254 346
     
    
    255 347
         def fetch(self, alias_override=None):
    
    256 348
             # Resolve the URL for the message
    
    ... ... @@ -261,7 +353,7 @@ class GitMirror(SourceFetcher):
    261 353
             with self.source.timed_activity("Fetching from {}"
    
    262 354
                                             .format(resolved_url),
    
    263 355
                                             silent_nested=True):
    
    264
    -            self.ensure(alias_override)
    
    356
    +            self.ensure_fetchable(alias_override)
    
    265 357
                 if not self.has_ref():
    
    266 358
                     self._fetch(alias_override)
    
    267 359
                 self.assert_ref()
    
    ... ... @@ -270,12 +362,14 @@ class GitMirror(SourceFetcher):
    270 362
             if not self.ref:
    
    271 363
                 return False
    
    272 364
     
    
    273
    -        # If the mirror doesnt exist, we also dont have the ref
    
    274
    -        if not os.path.exists(self.mirror):
    
    365
    +        if not os.path.exists(self.mirror) and not os.path.exists(self.fetch_mirror):
    
    366
    +            # If the mirror doesnt exist, we also dont have the ref
    
    275 367
                 return False
    
    276 368
     
    
    369
    +        mirror = self.mirror_path()
    
    370
    +
    
    277 371
             # Check if the ref is really there
    
    278
    -        rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=self.mirror)
    
    372
    +        rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=mirror)
    
    279 373
             return rc == 0
    
    280 374
     
    
    281 375
         def assert_ref(self):
    
    ... ... @@ -325,11 +419,13 @@ class GitMirror(SourceFetcher):
    325 419
         def stage(self, directory):
    
    326 420
             fullpath = os.path.join(directory, self.path)
    
    327 421
     
    
    422
    +        mirror = self.mirror_path()
    
    423
    +
    
    328 424
             # Using --shared here avoids copying the objects into the checkout, in any
    
    329 425
             # case we're just checking out a specific commit and then removing the .git/
    
    330 426
             # directory.
    
    331
    -        self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', self.mirror, fullpath],
    
    332
    -                         fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
    
    427
    +        self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', mirror, fullpath],
    
    428
    +                         fail="Failed to create git mirror {} in directory: {}".format(mirror, fullpath),
    
    333 429
                              fail_temporarily=True)
    
    334 430
     
    
    335 431
             self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
    
    ... ... @@ -359,9 +455,11 @@ class GitMirror(SourceFetcher):
    359 455
     
    
    360 456
         # List the submodules (path/url tuples) present at the given ref of this repo
    
    361 457
         def submodule_list(self):
    
    458
    +        mirror = self.mirror_path()
    
    459
    +
    
    362 460
             modules = "{}:{}".format(self.ref, GIT_MODULES)
    
    363 461
             exit_code, output = self.source.check_output(
    
    364
    -            [self.source.host_git, 'show', modules], cwd=self.mirror)
    
    462
    +            [self.source.host_git, 'show', modules], cwd=mirror)
    
    365 463
     
    
    366 464
             # If git show reports error code 128 here, we take it to mean there is
    
    367 465
             # no .gitmodules file to display for the given revision.
    
    ... ... @@ -389,6 +487,8 @@ class GitMirror(SourceFetcher):
    389 487
         # Fetch the ref which this mirror requires its submodule to have,
    
    390 488
         # at the given ref of this mirror.
    
    391 489
         def submodule_ref(self, submodule, ref=None):
    
    490
    +        mirror = self.mirror_path()
    
    491
    +
    
    392 492
             if not ref:
    
    393 493
                 ref = self.ref
    
    394 494
     
    
    ... ... @@ -397,7 +497,7 @@ class GitMirror(SourceFetcher):
    397 497
             _, output = self.source.check_output([self.source.host_git, 'ls-tree', ref, submodule],
    
    398 498
                                                  fail="ls-tree failed for commit {} and submodule: {}".format(
    
    399 499
                                                      ref, submodule),
    
    400
    -                                             cwd=self.mirror)
    
    500
    +                                             cwd=mirror)
    
    401 501
     
    
    402 502
             # read the commit hash from the output
    
    403 503
             fields = output.split()
    
    ... ... @@ -514,8 +614,8 @@ class GitSource(Source):
    514 614
             self.track_tags = self.node_get_member(node, bool, 'track-tags', False)
    
    515 615
     
    
    516 616
             self.original_url = self.node_get_member(node, str, 'url')
    
    517
    -        self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True)
    
    518 617
             self.tracking = self.node_get_member(node, str, 'track', None)
    
    618
    +        self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True, tracking=self.tracking)
    
    519 619
     
    
    520 620
             self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1')
    
    521 621
             if self.ref_format not in ['sha1', 'git-describe']:
    
    ... ... @@ -633,7 +733,7 @@ class GitSource(Source):
    633 733
             with self.timed_activity("Tracking {} from {}"
    
    634 734
                                      .format(self.tracking, resolved_url),
    
    635 735
                                      silent_nested=True):
    
    636
    -            self.mirror.ensure()
    
    736
    +            self.mirror.ensure_trackable()
    
    637 737
                 self.mirror._fetch()
    
    638 738
     
    
    639 739
                 # Update self.mirror.ref and node.ref from the self.tracking branch
    
    ... ... @@ -643,6 +743,7 @@ class GitSource(Source):
    643 743
     
    
    644 744
         def init_workspace(self, directory):
    
    645 745
             # XXX: may wish to refactor this as some code dupe with stage()
    
    746
    +        self.mirror.ensure_trackable()
    
    646 747
             self.refresh_submodules()
    
    647 748
     
    
    648 749
             with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
    
    ... ... @@ -717,15 +818,16 @@ class GitSource(Source):
    717 818
             # Assert that the ref exists in the track tag/branch, if track has been specified.
    
    718 819
             ref_in_track = False
    
    719 820
             if self.tracking:
    
    821
    +            mirror = self.mirror.mirror_path()
    
    720 822
                 _, branch = self.check_output([self.host_git, 'branch', '--list', self.tracking,
    
    721 823
                                                '--contains', self.mirror.ref],
    
    722
    -                                          cwd=self.mirror.mirror)
    
    824
    +                                          cwd=mirror)
    
    723 825
                 if branch:
    
    724 826
                     ref_in_track = True
    
    725 827
                 else:
    
    726 828
                     _, tag = self.check_output([self.host_git, 'tag', '--list', self.tracking,
    
    727 829
                                                 '--contains', self.mirror.ref],
    
    728
    -                                           cwd=self.mirror.mirror)
    
    830
    +                                           cwd=mirror)
    
    729 831
                     if tag:
    
    730 832
                         ref_in_track = True
    
    731 833
     
    
    ... ... @@ -749,7 +851,7 @@ class GitSource(Source):
    749 851
     
    
    750 852
             self.refresh_submodules()
    
    751 853
             for mirror in self.submodules:
    
    752
    -            if not os.path.exists(mirror.mirror):
    
    854
    +            if not os.path.exists(mirror.mirror) and not os.path.exists(mirror.fetch_mirror):
    
    753 855
                     return False
    
    754 856
                 if not mirror.has_ref():
    
    755 857
                     return False
    
    ... ... @@ -761,7 +863,7 @@ class GitSource(Source):
    761 863
         # Assumes that we have our mirror and we have the ref which we point to
    
    762 864
         #
    
    763 865
         def refresh_submodules(self):
    
    764
    -        self.mirror.ensure()
    
    866
    +        self.mirror.ensure_fetchable()
    
    765 867
             submodules = []
    
    766 868
     
    
    767 869
             for path, url in self.mirror.submodule_list():
    

  • tests/sources/git.py
    ... ... @@ -28,6 +28,7 @@ import shutil
    28 28
     from buildstream._exceptions import ErrorDomain
    
    29 29
     from buildstream import _yaml
    
    30 30
     from buildstream.plugin import CoreWarnings
    
    31
    +from buildstream.utils import url_directory_name
    
    31 32
     
    
    32 33
     from tests.testutils import cli, create_repo
    
    33 34
     from tests.testutils.site import HAVE_GIT
    
    ... ... @@ -1018,3 +1019,249 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles):
    1018 1019
     
    
    1019 1020
         result = cli.run(project=project, args=['build', 'target.bst'])
    
    1020 1021
         result.assert_success()
    
    1022
    +
    
    1023
    +
    
    1024
    +@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
    
    1025
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
    
    1026
    +def test_fetch_shallow(cli, tmpdir, datafiles):
    
    1027
    +    project = str(datafiles)
    
    1028
    +
    
    1029
    +    repo = create_repo('git', str(tmpdir))
    
    1030
    +    previous_ref = repo.create(os.path.join(project, 'repofiles'))
    
    1031
    +
    
    1032
    +    file1 = os.path.join(str(tmpdir), 'file1')
    
    1033
    +    with open(file1, 'w') as f:
    
    1034
    +        f.write('test\n')
    
    1035
    +    ref = repo.add_file(file1)
    
    1036
    +
    
    1037
    +    source_config = repo.source_config(ref=ref)
    
    1038
    +
    
    1039
    +    # Write out our test target with a bad ref
    
    1040
    +    element = {
    
    1041
    +        'kind': 'import',
    
    1042
    +        'sources': [
    
    1043
    +            source_config
    
    1044
    +        ]
    
    1045
    +    }
    
    1046
    +    _yaml.dump(element, os.path.join(project, 'target.bst'))
    
    1047
    +
    
    1048
    +    sources_dir = os.path.join(str(tmpdir), 'sources')
    
    1049
    +    os.makedirs(sources_dir, exist_ok=True)
    
    1050
    +    config = {
    
    1051
    +        'sourcedir': sources_dir
    
    1052
    +    }
    
    1053
    +    cli.configure(config)
    
    1054
    +
    
    1055
    +    result = cli.run(project=project, args=[
    
    1056
    +        'source', 'fetch', 'target.bst'
    
    1057
    +    ])
    
    1058
    +    result.assert_success()
    
    1059
    +
    
    1060
    +    cache_dir_name = url_directory_name(source_config['url'])
    
    1061
    +    full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
    
    1062
    +    shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, ref))
    
    1063
    +
    
    1064
    +    assert os.path.exists(shallow_cache_path)
    
    1065
    +    assert not os.path.exists(full_cache_path)
    
    1066
    +
    
    1067
    +    output = subprocess.run(['git', 'log', '--format=format:%H'],
    
    1068
    +                            cwd=shallow_cache_path,
    
    1069
    +                            stdout=subprocess.PIPE).stdout.decode('ascii')
    
    1070
    +    assert output.splitlines() == [ref]
    
    1071
    +
    
    1072
    +    result = cli.run(project=project, args=[
    
    1073
    +        'build', 'target.bst'
    
    1074
    +    ])
    
    1075
    +    result.assert_success()
    
    1076
    +
    
    1077
    +    output = subprocess.run(['git', 'log', '--format=format:%H'],
    
    1078
    +                            cwd=shallow_cache_path,
    
    1079
    +                            stdout=subprocess.PIPE).stdout.decode('ascii')
    
    1080
    +    assert output.splitlines() == [ref]
    
    1081
    +
    
    1082
    +    assert os.path.exists(shallow_cache_path)
    
    1083
    +    assert not os.path.exists(full_cache_path)
    
    1084
    +
    
    1085
    +    result = cli.run(project=project, args=[
    
    1086
    +        'track', 'target.bst'
    
    1087
    +    ])
    
    1088
    +    result.assert_success()
    
    1089
    +
    
    1090
    +    assert os.path.exists(full_cache_path)
    
    1091
    +    output = subprocess.run(['git', 'log', '--format=format:%H'],
    
    1092
    +                            cwd=full_cache_path,
    
    1093
    +                            stdout=subprocess.PIPE).stdout.decode('ascii')
    
    1094
    +    assert output.splitlines() == [ref, previous_ref]
    
    1095
    +
    
    1096
    +
    
    1097
    +@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
    
    1098
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
    
    1099
    +def test_fetch_shallow_not_tagged(cli, tmpdir, datafiles):
    
    1100
    +    """When a ref is not tagged and not head of branch on remote we cannot
    
    1101
    +    get a shallow clone.  It should automatically get a full clone.
    
    1102
    +    """
    
    1103
    +
    
    1104
    +    project = str(datafiles)
    
    1105
    +
    
    1106
    +    repo = create_repo('git', str(tmpdir))
    
    1107
    +    previous_ref = repo.create(os.path.join(project, 'repofiles'))
    
    1108
    +
    
    1109
    +    file1 = os.path.join(str(tmpdir), 'file1')
    
    1110
    +    with open(file1, 'w') as f:
    
    1111
    +        f.write('test\n')
    
    1112
    +    ref = repo.add_file(file1)
    
    1113
    +
    
    1114
    +    source_config = repo.source_config(ref=previous_ref)
    
    1115
    +
    
    1116
    +    # Write out our test target with a bad ref
    
    1117
    +    element = {
    
    1118
    +        'kind': 'import',
    
    1119
    +        'sources': [
    
    1120
    +            source_config
    
    1121
    +        ]
    
    1122
    +    }
    
    1123
    +    _yaml.dump(element, os.path.join(project, 'target.bst'))
    
    1124
    +
    
    1125
    +    sources_dir = os.path.join(str(tmpdir), 'sources')
    
    1126
    +    os.makedirs(sources_dir, exist_ok=True)
    
    1127
    +    config = {
    
    1128
    +        'sourcedir': sources_dir
    
    1129
    +    }
    
    1130
    +    cli.configure(config)
    
    1131
    +
    
    1132
    +    result = cli.run(project=project, args=[
    
    1133
    +        'source', 'fetch', 'target.bst'
    
    1134
    +    ])
    
    1135
    +    result.assert_success()
    
    1136
    +
    
    1137
    +    cache_dir_name = url_directory_name(source_config['url'])
    
    1138
    +    full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
    
    1139
    +    shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, previous_ref))
    
    1140
    +
    
    1141
    +    assert not os.path.exists(shallow_cache_path)
    
    1142
    +    assert os.path.exists(full_cache_path)
    
    1143
    +
    
    1144
    +    output = subprocess.run(['git', 'log', '--format=format:%H'],
    
    1145
    +                            cwd=full_cache_path,
    
    1146
    +                            stdout=subprocess.PIPE).stdout.decode('ascii')
    
    1147
    +    assert output.splitlines() == [ref, previous_ref]
    
    1148
    +
    
    1149
    +
    
    1150
    +@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
    
    1151
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
    
    1152
    +def test_fetch_shallow_annotated_tag(cli, tmpdir, datafiles):
    
    1153
    +    """When a ref is not tagged and not head of branch on remote we cannot
    
    1154
    +    get a shallow clone.  It should automatically get a full clone.
    
    1155
    +    """
    
    1156
    +
    
    1157
    +    project = str(datafiles)
    
    1158
    +
    
    1159
    +    repo = create_repo('git', str(tmpdir))
    
    1160
    +    previous_ref = repo.create(os.path.join(project, 'repofiles'))
    
    1161
    +
    
    1162
    +    repo.add_annotated_tag('tag', 'tag')
    
    1163
    +
    
    1164
    +    file1 = os.path.join(str(tmpdir), 'file1')
    
    1165
    +    with open(file1, 'w') as f:
    
    1166
    +        f.write('test\n')
    
    1167
    +    ref = repo.add_file(file1)
    
    1168
    +
    
    1169
    +    source_config = repo.source_config(ref=previous_ref)
    
    1170
    +    del source_config['track']
    
    1171
    +
    
    1172
    +    # Write out our test target with a bad ref
    
    1173
    +    element = {
    
    1174
    +        'kind': 'import',
    
    1175
    +        'sources': [
    
    1176
    +            source_config
    
    1177
    +        ]
    
    1178
    +    }
    
    1179
    +    _yaml.dump(element, os.path.join(project, 'target.bst'))
    
    1180
    +
    
    1181
    +    sources_dir = os.path.join(str(tmpdir), 'sources')
    
    1182
    +    os.makedirs(sources_dir, exist_ok=True)
    
    1183
    +    config = {
    
    1184
    +        'sourcedir': sources_dir
    
    1185
    +    }
    
    1186
    +    cli.configure(config)
    
    1187
    +
    
    1188
    +    result = cli.run(project=project, args=[
    
    1189
    +        'source', 'fetch', 'target.bst'
    
    1190
    +    ])
    
    1191
    +    result.assert_success()
    
    1192
    +
    
    1193
    +    cache_dir_name = url_directory_name(source_config['url'])
    
    1194
    +    full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
    
    1195
    +    shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, previous_ref))
    
    1196
    +
    
    1197
    +    assert os.path.exists(shallow_cache_path)
    
    1198
    +    assert not os.path.exists(full_cache_path)
    
    1199
    +
    
    1200
    +    output = subprocess.run(['git', 'log', '--format=format:%H'],
    
    1201
    +                            cwd=shallow_cache_path,
    
    1202
    +                            stdout=subprocess.PIPE).stdout.decode('ascii')
    
    1203
    +    assert output.splitlines() == [previous_ref]
    
    1204
    +
    
    1205
    +
    
    1206
    +@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
    
    1207
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
    
    1208
    +def test_fetch_shallow_workspace_open(cli, tmpdir, datafiles):
    
    1209
    +    """
    
    1210
    +    Workspaces should get a full clone.
    
    1211
    +    """
    
    1212
    +    project = str(datafiles)
    
    1213
    +
    
    1214
    +    repo = create_repo('git', str(tmpdir))
    
    1215
    +    previous_ref = repo.create(os.path.join(project, 'repofiles'))
    
    1216
    +
    
    1217
    +    file1 = os.path.join(str(tmpdir), 'file1')
    
    1218
    +    with open(file1, 'w') as f:
    
    1219
    +        f.write('test\n')
    
    1220
    +    ref = repo.add_file(file1)
    
    1221
    +
    
    1222
    +    source_config = repo.source_config(ref=ref)
    
    1223
    +
    
    1224
    +    # Write out our test target with a bad ref
    
    1225
    +    element = {
    
    1226
    +        'kind': 'import',
    
    1227
    +        'sources': [
    
    1228
    +            source_config
    
    1229
    +        ]
    
    1230
    +    }
    
    1231
    +    _yaml.dump(element, os.path.join(project, 'target.bst'))
    
    1232
    +
    
    1233
    +    sources_dir = os.path.join(str(tmpdir), 'sources')
    
    1234
    +    os.makedirs(sources_dir, exist_ok=True)
    
    1235
    +    config = {
    
    1236
    +        'sourcedir': sources_dir
    
    1237
    +    }
    
    1238
    +    cli.configure(config)
    
    1239
    +
    
    1240
    +    result = cli.run(project=project, args=[
    
    1241
    +        'source', 'fetch', 'target.bst'
    
    1242
    +    ])
    
    1243
    +    result.assert_success()
    
    1244
    +
    
    1245
    +    cache_dir_name = url_directory_name(source_config['url'])
    
    1246
    +    full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
    
    1247
    +    shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, ref))
    
    1248
    +
    
    1249
    +    assert os.path.exists(shallow_cache_path)
    
    1250
    +    assert not os.path.exists(full_cache_path)
    
    1251
    +
    
    1252
    +    output = subprocess.run(['git', 'log', '--format=format:%H'],
    
    1253
    +                            cwd=shallow_cache_path,
    
    1254
    +                            stdout=subprocess.PIPE).stdout.decode('ascii')
    
    1255
    +    assert output.splitlines() == [ref]
    
    1256
    +
    
    1257
    +    workspace = os.path.join(str(tmpdir), 'workspace')
    
    1258
    +
    
    1259
    +    result = cli.run(project=project, args=[
    
    1260
    +        'workspace', 'open', 'target.bst', '--directory', workspace
    
    1261
    +    ])
    
    1262
    +    result.assert_success()
    
    1263
    +
    
    1264
    +    output = subprocess.run(['git', 'log', '--format=format:%H'],
    
    1265
    +                            cwd=workspace,
    
    1266
    +                            stdout=subprocess.PIPE).stdout.decode('ascii')
    
    1267
    +    assert output.splitlines() == [ref, previous_ref]



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]