Valentin David pushed to branch valentindavid/git_shallow_fetch at BuildStream / buildstream
Commits:
-
052c4c3d
by Valentin David at 2018-12-17T09:37:44Z
2 changed files:
Changes:
... | ... | @@ -183,7 +183,7 @@ WARN_INVALID_SUBMODULE = "invalid-submodule" |
183 | 183 |
#
|
184 | 184 |
class GitMirror(SourceFetcher):
|
185 | 185 |
|
186 |
- def __init__(self, source, path, url, ref, *, primary=False, tags=[]):
|
|
186 |
+ def __init__(self, source, path, url, ref, *, primary=False, tags=[], tracking=None):
|
|
187 | 187 |
|
188 | 188 |
super().__init__()
|
189 | 189 |
self.source = source
|
... | ... | @@ -192,11 +192,101 @@ class GitMirror(SourceFetcher): |
192 | 192 |
self.ref = ref
|
193 | 193 |
self.tags = tags
|
194 | 194 |
self.primary = primary
|
195 |
+ dirname = utils.url_directory_name(url)
|
|
195 | 196 |
self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
|
197 |
+ self.fetch_mirror = os.path.join(source.get_mirror_directory(), '{}-{}'.format(dirname, ref))
|
|
196 | 198 |
self.mark_download_url(url)
|
199 |
+ self.tracking = tracking
|
|
200 |
+ |
|
201 |
+ def mirror_path(self):
|
|
202 |
+ if os.path.exists(self.mirror):
|
|
203 |
+ return self.mirror
|
|
204 |
+ else:
|
|
205 |
+ assert os.path.exists(self.fetch_mirror)
|
|
206 |
+ return self.fetch_mirror
|
|
207 |
+ |
|
208 |
+ def ensure_fetchable(self, alias_override=None):
|
|
209 |
+ |
|
210 |
+ if os.path.exists(self.mirror):
|
|
211 |
+ return
|
|
212 |
+ |
|
213 |
+ if self.tags:
|
|
214 |
+ for tag, commit, _ in self.tags:
|
|
215 |
+ if commit != self.ref:
|
|
216 |
+ self.source.status("{}: tag '{}' is not on commit '{}', so a full clone is required"
|
|
217 |
+ .format(self.source, tag, commit))
|
|
218 |
+ self.ensure_trackable(alias_override=alias_override)
|
|
219 |
+ return
|
|
220 |
+ |
|
221 |
+ if os.path.exists(self.fetch_mirror):
|
|
222 |
+ return
|
|
223 |
+ |
|
224 |
+ with self.source.tempdir() as tmpdir:
|
|
225 |
+ self.source.call([self.source.host_git, 'init', '--bare', tmpdir],
|
|
226 |
+ fail="Failed to init git repository",
|
|
227 |
+ fail_temporarily=True)
|
|
228 |
+ |
|
229 |
+ url = self.source.translate_url(self.url, alias_override=alias_override,
|
|
230 |
+ primary=self.primary)
|
|
231 |
+ |
|
232 |
+ self.source.call([self.source.host_git, 'remote', 'add', '--mirror=fetch', 'origin', url],
|
|
233 |
+ cwd=tmpdir,
|
|
234 |
+ fail="Failed to init git repository",
|
|
235 |
+ fail_temporarily=True)
|
|
236 |
+ |
|
237 |
+ _, refs = self.source.check_output([self.source.host_git, 'ls-remote', 'origin'],
|
|
238 |
+ cwd=tmpdir,
|
|
239 |
+ fail="Failed to clone git repository {}".format(url),
|
|
240 |
+ fail_temporarily=True)
|
|
241 |
+ |
|
242 |
+ advertised = None
|
|
243 |
+ for ref_line in refs.splitlines():
|
|
244 |
+ commit, ref = ref_line.split('\t', 1)
|
|
245 |
+ if ref == 'HEAD':
|
|
246 |
+ continue
|
|
247 |
+ if self.tracking:
|
|
248 |
+ # For validate_cache to work
|
|
249 |
+ if ref not in ['refs/heads/{}'.format(self.tracking),
|
|
250 |
+ 'refs/tags/{}'.format(self.tracking),
|
|
251 |
+ 'refs/tags/{}{}'.format(self.tracking, '^{}')]:
|
|
252 |
+ continue
|
|
253 |
+ if self.ref == commit:
|
|
254 |
+ if ref.endswith('^{}'):
|
|
255 |
+ ref = ref[:-3]
|
|
256 |
+ advertised = ref
|
|
257 |
+ break
|
|
258 |
+ |
|
259 |
+ if advertised is None:
|
|
260 |
+ self.source.status("{}: {} is not advertised on {}, so a full clone is required"
|
|
261 |
+ .format(self.source, self.ref, url))
|
|
262 |
+ |
|
263 |
+ self.ensure_trackable(alias_override=alias_override)
|
|
264 |
+ return
|
|
265 |
+ |
|
266 |
+ self.source.call([self.source.host_git, 'fetch', '--depth=1', 'origin', advertised],
|
|
267 |
+ cwd=tmpdir,
|
|
268 |
+ fail="Failed to fetch repository",
|
|
269 |
+ fail_temporarily=True)
|
|
270 |
+ |
|
271 |
+ # We need to have a ref to make it clonable
|
|
272 |
+ self.source.call([self.source.host_git, 'update-ref', 'HEAD', self.ref],
|
|
273 |
+ cwd=tmpdir,
|
|
274 |
+ fail="Failed to tag HEAD",
|
|
275 |
+ fail_temporarily=True)
|
|
276 |
+ |
|
277 |
+ try:
|
|
278 |
+ move_atomic(tmpdir, self.fetch_mirror)
|
|
279 |
+ except DirectoryExistsError:
|
|
280 |
+ # Another process was quicker to download this repository.
|
|
281 |
+ # Let's discard our own
|
|
282 |
+ self.source.status("{}: Discarding duplicate clone of {}"
|
|
283 |
+ .format(self.source, url))
|
|
284 |
+ except OSError as e:
|
|
285 |
+ raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}': {}"
|
|
286 |
+ .format(self.source, url, tmpdir, self.fetch_mirror, e)) from e
|
|
197 | 287 |
|
198 | 288 |
# Ensures that the mirror exists
|
199 |
- def ensure(self, alias_override=None):
|
|
289 |
+ def ensure_trackable(self, alias_override=None):
|
|
200 | 290 |
|
201 | 291 |
# Unfortunately, git does not know how to only clone just a specific ref,
|
202 | 292 |
# so we have to download all of those gigs even if we only need a couple
|
... | ... | @@ -231,18 +321,20 @@ class GitMirror(SourceFetcher): |
231 | 321 |
alias_override=alias_override,
|
232 | 322 |
primary=self.primary)
|
233 | 323 |
|
324 |
+ mirror = self.mirror_path()
|
|
325 |
+ |
|
234 | 326 |
if alias_override:
|
235 | 327 |
remote_name = utils.url_directory_name(alias_override)
|
236 | 328 |
_, remotes = self.source.check_output(
|
237 | 329 |
[self.source.host_git, 'remote'],
|
238 |
- fail="Failed to retrieve list of remotes in {}".format(self.mirror),
|
|
239 |
- cwd=self.mirror
|
|
330 |
+ fail="Failed to retrieve list of remotes in {}".format(mirror),
|
|
331 |
+ cwd=mirror
|
|
240 | 332 |
)
|
241 | 333 |
if remote_name not in remotes:
|
242 | 334 |
self.source.call(
|
243 | 335 |
[self.source.host_git, 'remote', 'add', remote_name, url],
|
244 | 336 |
fail="Failed to add remote {} with url {}".format(remote_name, url),
|
245 |
- cwd=self.mirror
|
|
337 |
+ cwd=mirror
|
|
246 | 338 |
)
|
247 | 339 |
else:
|
248 | 340 |
remote_name = "origin"
|
... | ... | @@ -250,7 +342,7 @@ class GitMirror(SourceFetcher): |
250 | 342 |
self.source.call([self.source.host_git, 'fetch', remote_name, '--prune', '--force', '--tags'],
|
251 | 343 |
fail="Failed to fetch from remote git repository: {}".format(url),
|
252 | 344 |
fail_temporarily=True,
|
253 |
- cwd=self.mirror)
|
|
345 |
+ cwd=mirror)
|
|
254 | 346 |
|
255 | 347 |
def fetch(self, alias_override=None):
|
256 | 348 |
# Resolve the URL for the message
|
... | ... | @@ -261,7 +353,7 @@ class GitMirror(SourceFetcher): |
261 | 353 |
with self.source.timed_activity("Fetching from {}"
|
262 | 354 |
.format(resolved_url),
|
263 | 355 |
silent_nested=True):
|
264 |
- self.ensure(alias_override)
|
|
356 |
+ self.ensure_fetchable(alias_override)
|
|
265 | 357 |
if not self.has_ref():
|
266 | 358 |
self._fetch(alias_override)
|
267 | 359 |
self.assert_ref()
|
... | ... | @@ -270,12 +362,14 @@ class GitMirror(SourceFetcher): |
270 | 362 |
if not self.ref:
|
271 | 363 |
return False
|
272 | 364 |
|
273 |
- # If the mirror doesnt exist, we also dont have the ref
|
|
274 |
- if not os.path.exists(self.mirror):
|
|
365 |
+ if not os.path.exists(self.mirror) and not os.path.exists(self.fetch_mirror):
|
|
366 |
+ # If the mirror doesnt exist, we also dont have the ref
|
|
275 | 367 |
return False
|
276 | 368 |
|
369 |
+ mirror = self.mirror_path()
|
|
370 |
+ |
|
277 | 371 |
# Check if the ref is really there
|
278 |
- rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=self.mirror)
|
|
372 |
+ rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=mirror)
|
|
279 | 373 |
return rc == 0
|
280 | 374 |
|
281 | 375 |
def assert_ref(self):
|
... | ... | @@ -325,11 +419,13 @@ class GitMirror(SourceFetcher): |
325 | 419 |
def stage(self, directory):
|
326 | 420 |
fullpath = os.path.join(directory, self.path)
|
327 | 421 |
|
422 |
+ mirror = self.mirror_path()
|
|
423 |
+ |
|
328 | 424 |
# Using --shared here avoids copying the objects into the checkout, in any
|
329 | 425 |
# case we're just checking out a specific commit and then removing the .git/
|
330 | 426 |
# directory.
|
331 |
- self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', self.mirror, fullpath],
|
|
332 |
- fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
|
|
427 |
+ self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', mirror, fullpath],
|
|
428 |
+ fail="Failed to create git mirror {} in directory: {}".format(mirror, fullpath),
|
|
333 | 429 |
fail_temporarily=True)
|
334 | 430 |
|
335 | 431 |
self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
|
... | ... | @@ -359,9 +455,11 @@ class GitMirror(SourceFetcher): |
359 | 455 |
|
360 | 456 |
# List the submodules (path/url tuples) present at the given ref of this repo
|
361 | 457 |
def submodule_list(self):
|
458 |
+ mirror = self.mirror_path()
|
|
459 |
+ |
|
362 | 460 |
modules = "{}:{}".format(self.ref, GIT_MODULES)
|
363 | 461 |
exit_code, output = self.source.check_output(
|
364 |
- [self.source.host_git, 'show', modules], cwd=self.mirror)
|
|
462 |
+ [self.source.host_git, 'show', modules], cwd=mirror)
|
|
365 | 463 |
|
366 | 464 |
# If git show reports error code 128 here, we take it to mean there is
|
367 | 465 |
# no .gitmodules file to display for the given revision.
|
... | ... | @@ -389,6 +487,8 @@ class GitMirror(SourceFetcher): |
389 | 487 |
# Fetch the ref which this mirror requires its submodule to have,
|
390 | 488 |
# at the given ref of this mirror.
|
391 | 489 |
def submodule_ref(self, submodule, ref=None):
|
490 |
+ mirror = self.mirror_path()
|
|
491 |
+ |
|
392 | 492 |
if not ref:
|
393 | 493 |
ref = self.ref
|
394 | 494 |
|
... | ... | @@ -397,7 +497,7 @@ class GitMirror(SourceFetcher): |
397 | 497 |
_, output = self.source.check_output([self.source.host_git, 'ls-tree', ref, submodule],
|
398 | 498 |
fail="ls-tree failed for commit {} and submodule: {}".format(
|
399 | 499 |
ref, submodule),
|
400 |
- cwd=self.mirror)
|
|
500 |
+ cwd=mirror)
|
|
401 | 501 |
|
402 | 502 |
# read the commit hash from the output
|
403 | 503 |
fields = output.split()
|
... | ... | @@ -514,8 +614,8 @@ class GitSource(Source): |
514 | 614 |
self.track_tags = self.node_get_member(node, bool, 'track-tags', False)
|
515 | 615 |
|
516 | 616 |
self.original_url = self.node_get_member(node, str, 'url')
|
517 |
- self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True)
|
|
518 | 617 |
self.tracking = self.node_get_member(node, str, 'track', None)
|
618 |
+ self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True, tracking=self.tracking)
|
|
519 | 619 |
|
520 | 620 |
self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1')
|
521 | 621 |
if self.ref_format not in ['sha1', 'git-describe']:
|
... | ... | @@ -633,7 +733,7 @@ class GitSource(Source): |
633 | 733 |
with self.timed_activity("Tracking {} from {}"
|
634 | 734 |
.format(self.tracking, resolved_url),
|
635 | 735 |
silent_nested=True):
|
636 |
- self.mirror.ensure()
|
|
736 |
+ self.mirror.ensure_trackable()
|
|
637 | 737 |
self.mirror._fetch()
|
638 | 738 |
|
639 | 739 |
# Update self.mirror.ref and node.ref from the self.tracking branch
|
... | ... | @@ -643,6 +743,7 @@ class GitSource(Source): |
643 | 743 |
|
644 | 744 |
def init_workspace(self, directory):
|
645 | 745 |
# XXX: may wish to refactor this as some code dupe with stage()
|
746 |
+ self.mirror.ensure_trackable()
|
|
646 | 747 |
self.refresh_submodules()
|
647 | 748 |
|
648 | 749 |
with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
|
... | ... | @@ -717,15 +818,16 @@ class GitSource(Source): |
717 | 818 |
# Assert that the ref exists in the track tag/branch, if track has been specified.
|
718 | 819 |
ref_in_track = False
|
719 | 820 |
if self.tracking:
|
821 |
+ mirror = self.mirror.mirror_path()
|
|
720 | 822 |
_, branch = self.check_output([self.host_git, 'branch', '--list', self.tracking,
|
721 | 823 |
'--contains', self.mirror.ref],
|
722 |
- cwd=self.mirror.mirror)
|
|
824 |
+ cwd=mirror)
|
|
723 | 825 |
if branch:
|
724 | 826 |
ref_in_track = True
|
725 | 827 |
else:
|
726 | 828 |
_, tag = self.check_output([self.host_git, 'tag', '--list', self.tracking,
|
727 | 829 |
'--contains', self.mirror.ref],
|
728 |
- cwd=self.mirror.mirror)
|
|
830 |
+ cwd=mirror)
|
|
729 | 831 |
if tag:
|
730 | 832 |
ref_in_track = True
|
731 | 833 |
|
... | ... | @@ -749,7 +851,7 @@ class GitSource(Source): |
749 | 851 |
|
750 | 852 |
self.refresh_submodules()
|
751 | 853 |
for mirror in self.submodules:
|
752 |
- if not os.path.exists(mirror.mirror):
|
|
854 |
+ if not os.path.exists(mirror.mirror) and not os.path.exists(mirror.fetch_mirror):
|
|
753 | 855 |
return False
|
754 | 856 |
if not mirror.has_ref():
|
755 | 857 |
return False
|
... | ... | @@ -761,7 +863,7 @@ class GitSource(Source): |
761 | 863 |
# Assumes that we have our mirror and we have the ref which we point to
|
762 | 864 |
#
|
763 | 865 |
def refresh_submodules(self):
|
764 |
- self.mirror.ensure()
|
|
866 |
+ self.mirror.ensure_fetchable()
|
|
765 | 867 |
submodules = []
|
766 | 868 |
|
767 | 869 |
for path, url in self.mirror.submodule_list():
|
... | ... | @@ -28,6 +28,7 @@ import shutil |
28 | 28 |
from buildstream._exceptions import ErrorDomain
|
29 | 29 |
from buildstream import _yaml
|
30 | 30 |
from buildstream.plugin import CoreWarnings
|
31 |
+from buildstream.utils import url_directory_name
|
|
31 | 32 |
|
32 | 33 |
from tests.testutils import cli, create_repo
|
33 | 34 |
from tests.testutils.site import HAVE_GIT
|
... | ... | @@ -1018,3 +1019,249 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles): |
1018 | 1019 |
|
1019 | 1020 |
result = cli.run(project=project, args=['build', 'target.bst'])
|
1020 | 1021 |
result.assert_success()
|
1022 |
+ |
|
1023 |
+ |
|
1024 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
1025 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
1026 |
+def test_fetch_shallow(cli, tmpdir, datafiles):
|
|
1027 |
+ project = str(datafiles)
|
|
1028 |
+ |
|
1029 |
+ repo = create_repo('git', str(tmpdir))
|
|
1030 |
+ previous_ref = repo.create(os.path.join(project, 'repofiles'))
|
|
1031 |
+ |
|
1032 |
+ file1 = os.path.join(str(tmpdir), 'file1')
|
|
1033 |
+ with open(file1, 'w') as f:
|
|
1034 |
+ f.write('test\n')
|
|
1035 |
+ ref = repo.add_file(file1)
|
|
1036 |
+ |
|
1037 |
+ source_config = repo.source_config(ref=ref)
|
|
1038 |
+ |
|
1039 |
+ # Write out our test target with a bad ref
|
|
1040 |
+ element = {
|
|
1041 |
+ 'kind': 'import',
|
|
1042 |
+ 'sources': [
|
|
1043 |
+ source_config
|
|
1044 |
+ ]
|
|
1045 |
+ }
|
|
1046 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
1047 |
+ |
|
1048 |
+ sources_dir = os.path.join(str(tmpdir), 'sources')
|
|
1049 |
+ os.makedirs(sources_dir, exist_ok=True)
|
|
1050 |
+ config = {
|
|
1051 |
+ 'sourcedir': sources_dir
|
|
1052 |
+ }
|
|
1053 |
+ cli.configure(config)
|
|
1054 |
+ |
|
1055 |
+ result = cli.run(project=project, args=[
|
|
1056 |
+ 'source', 'fetch', 'target.bst'
|
|
1057 |
+ ])
|
|
1058 |
+ result.assert_success()
|
|
1059 |
+ |
|
1060 |
+ cache_dir_name = url_directory_name(source_config['url'])
|
|
1061 |
+ full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
|
|
1062 |
+ shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, ref))
|
|
1063 |
+ |
|
1064 |
+ assert os.path.exists(shallow_cache_path)
|
|
1065 |
+ assert not os.path.exists(full_cache_path)
|
|
1066 |
+ |
|
1067 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
1068 |
+ cwd=shallow_cache_path,
|
|
1069 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
1070 |
+ assert output.splitlines() == [ref]
|
|
1071 |
+ |
|
1072 |
+ result = cli.run(project=project, args=[
|
|
1073 |
+ 'build', 'target.bst'
|
|
1074 |
+ ])
|
|
1075 |
+ result.assert_success()
|
|
1076 |
+ |
|
1077 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
1078 |
+ cwd=shallow_cache_path,
|
|
1079 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
1080 |
+ assert output.splitlines() == [ref]
|
|
1081 |
+ |
|
1082 |
+ assert os.path.exists(shallow_cache_path)
|
|
1083 |
+ assert not os.path.exists(full_cache_path)
|
|
1084 |
+ |
|
1085 |
+ result = cli.run(project=project, args=[
|
|
1086 |
+ 'track', 'target.bst'
|
|
1087 |
+ ])
|
|
1088 |
+ result.assert_success()
|
|
1089 |
+ |
|
1090 |
+ assert os.path.exists(full_cache_path)
|
|
1091 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
1092 |
+ cwd=full_cache_path,
|
|
1093 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
1094 |
+ assert output.splitlines() == [ref, previous_ref]
|
|
1095 |
+ |
|
1096 |
+ |
|
1097 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
1098 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
1099 |
+def test_fetch_shallow_not_tagged(cli, tmpdir, datafiles):
|
|
1100 |
+ """When a ref is not tagged and not head of branch on remote we cannot
|
|
1101 |
+ get a shallow clone. It should automatically get a full clone.
|
|
1102 |
+ """
|
|
1103 |
+ |
|
1104 |
+ project = str(datafiles)
|
|
1105 |
+ |
|
1106 |
+ repo = create_repo('git', str(tmpdir))
|
|
1107 |
+ previous_ref = repo.create(os.path.join(project, 'repofiles'))
|
|
1108 |
+ |
|
1109 |
+ file1 = os.path.join(str(tmpdir), 'file1')
|
|
1110 |
+ with open(file1, 'w') as f:
|
|
1111 |
+ f.write('test\n')
|
|
1112 |
+ ref = repo.add_file(file1)
|
|
1113 |
+ |
|
1114 |
+ source_config = repo.source_config(ref=previous_ref)
|
|
1115 |
+ |
|
1116 |
+ # Write out our test target with a bad ref
|
|
1117 |
+ element = {
|
|
1118 |
+ 'kind': 'import',
|
|
1119 |
+ 'sources': [
|
|
1120 |
+ source_config
|
|
1121 |
+ ]
|
|
1122 |
+ }
|
|
1123 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
1124 |
+ |
|
1125 |
+ sources_dir = os.path.join(str(tmpdir), 'sources')
|
|
1126 |
+ os.makedirs(sources_dir, exist_ok=True)
|
|
1127 |
+ config = {
|
|
1128 |
+ 'sourcedir': sources_dir
|
|
1129 |
+ }
|
|
1130 |
+ cli.configure(config)
|
|
1131 |
+ |
|
1132 |
+ result = cli.run(project=project, args=[
|
|
1133 |
+ 'source', 'fetch', 'target.bst'
|
|
1134 |
+ ])
|
|
1135 |
+ result.assert_success()
|
|
1136 |
+ |
|
1137 |
+ cache_dir_name = url_directory_name(source_config['url'])
|
|
1138 |
+ full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
|
|
1139 |
+ shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, previous_ref))
|
|
1140 |
+ |
|
1141 |
+ assert not os.path.exists(shallow_cache_path)
|
|
1142 |
+ assert os.path.exists(full_cache_path)
|
|
1143 |
+ |
|
1144 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
1145 |
+ cwd=full_cache_path,
|
|
1146 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
1147 |
+ assert output.splitlines() == [ref, previous_ref]
|
|
1148 |
+ |
|
1149 |
+ |
|
1150 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
1151 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
1152 |
+def test_fetch_shallow_annotated_tag(cli, tmpdir, datafiles):
|
|
1153 |
+ """When a ref is not tagged and not head of branch on remote we cannot
|
|
1154 |
+ get a shallow clone. It should automatically get a full clone.
|
|
1155 |
+ """
|
|
1156 |
+ |
|
1157 |
+ project = str(datafiles)
|
|
1158 |
+ |
|
1159 |
+ repo = create_repo('git', str(tmpdir))
|
|
1160 |
+ previous_ref = repo.create(os.path.join(project, 'repofiles'))
|
|
1161 |
+ |
|
1162 |
+ repo.add_annotated_tag('tag', 'tag')
|
|
1163 |
+ |
|
1164 |
+ file1 = os.path.join(str(tmpdir), 'file1')
|
|
1165 |
+ with open(file1, 'w') as f:
|
|
1166 |
+ f.write('test\n')
|
|
1167 |
+ ref = repo.add_file(file1)
|
|
1168 |
+ |
|
1169 |
+ source_config = repo.source_config(ref=previous_ref)
|
|
1170 |
+ del source_config['track']
|
|
1171 |
+ |
|
1172 |
+ # Write out our test target with a bad ref
|
|
1173 |
+ element = {
|
|
1174 |
+ 'kind': 'import',
|
|
1175 |
+ 'sources': [
|
|
1176 |
+ source_config
|
|
1177 |
+ ]
|
|
1178 |
+ }
|
|
1179 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
1180 |
+ |
|
1181 |
+ sources_dir = os.path.join(str(tmpdir), 'sources')
|
|
1182 |
+ os.makedirs(sources_dir, exist_ok=True)
|
|
1183 |
+ config = {
|
|
1184 |
+ 'sourcedir': sources_dir
|
|
1185 |
+ }
|
|
1186 |
+ cli.configure(config)
|
|
1187 |
+ |
|
1188 |
+ result = cli.run(project=project, args=[
|
|
1189 |
+ 'source', 'fetch', 'target.bst'
|
|
1190 |
+ ])
|
|
1191 |
+ result.assert_success()
|
|
1192 |
+ |
|
1193 |
+ cache_dir_name = url_directory_name(source_config['url'])
|
|
1194 |
+ full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
|
|
1195 |
+ shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, previous_ref))
|
|
1196 |
+ |
|
1197 |
+ assert os.path.exists(shallow_cache_path)
|
|
1198 |
+ assert not os.path.exists(full_cache_path)
|
|
1199 |
+ |
|
1200 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
1201 |
+ cwd=shallow_cache_path,
|
|
1202 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
1203 |
+ assert output.splitlines() == [previous_ref]
|
|
1204 |
+ |
|
1205 |
+ |
|
1206 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
1207 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
1208 |
+def test_fetch_shallow_workspace_open(cli, tmpdir, datafiles):
|
|
1209 |
+ """
|
|
1210 |
+ Workspaces should get a full clone.
|
|
1211 |
+ """
|
|
1212 |
+ project = str(datafiles)
|
|
1213 |
+ |
|
1214 |
+ repo = create_repo('git', str(tmpdir))
|
|
1215 |
+ previous_ref = repo.create(os.path.join(project, 'repofiles'))
|
|
1216 |
+ |
|
1217 |
+ file1 = os.path.join(str(tmpdir), 'file1')
|
|
1218 |
+ with open(file1, 'w') as f:
|
|
1219 |
+ f.write('test\n')
|
|
1220 |
+ ref = repo.add_file(file1)
|
|
1221 |
+ |
|
1222 |
+ source_config = repo.source_config(ref=ref)
|
|
1223 |
+ |
|
1224 |
+ # Write out our test target with a bad ref
|
|
1225 |
+ element = {
|
|
1226 |
+ 'kind': 'import',
|
|
1227 |
+ 'sources': [
|
|
1228 |
+ source_config
|
|
1229 |
+ ]
|
|
1230 |
+ }
|
|
1231 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
1232 |
+ |
|
1233 |
+ sources_dir = os.path.join(str(tmpdir), 'sources')
|
|
1234 |
+ os.makedirs(sources_dir, exist_ok=True)
|
|
1235 |
+ config = {
|
|
1236 |
+ 'sourcedir': sources_dir
|
|
1237 |
+ }
|
|
1238 |
+ cli.configure(config)
|
|
1239 |
+ |
|
1240 |
+ result = cli.run(project=project, args=[
|
|
1241 |
+ 'source', 'fetch', 'target.bst'
|
|
1242 |
+ ])
|
|
1243 |
+ result.assert_success()
|
|
1244 |
+ |
|
1245 |
+ cache_dir_name = url_directory_name(source_config['url'])
|
|
1246 |
+ full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
|
|
1247 |
+ shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, ref))
|
|
1248 |
+ |
|
1249 |
+ assert os.path.exists(shallow_cache_path)
|
|
1250 |
+ assert not os.path.exists(full_cache_path)
|
|
1251 |
+ |
|
1252 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
1253 |
+ cwd=shallow_cache_path,
|
|
1254 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
1255 |
+ assert output.splitlines() == [ref]
|
|
1256 |
+ |
|
1257 |
+ workspace = os.path.join(str(tmpdir), 'workspace')
|
|
1258 |
+ |
|
1259 |
+ result = cli.run(project=project, args=[
|
|
1260 |
+ 'workspace', 'open', 'target.bst', '--directory', workspace
|
|
1261 |
+ ])
|
|
1262 |
+ result.assert_success()
|
|
1263 |
+ |
|
1264 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
1265 |
+ cwd=workspace,
|
|
1266 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
1267 |
+ assert output.splitlines() == [ref, previous_ref]
|