Valentin David pushed to branch valentindavid/git_shallow_fetch at BuildStream / buildstream
Commits:
-
be7939e6
by Valentin David at 2018-12-17T09:36:17Z
2 changed files:
Changes:
| ... | ... | @@ -183,7 +183,7 @@ WARN_INVALID_SUBMODULE = "invalid-submodule" |
| 183 | 183 |
#
|
| 184 | 184 |
class GitMirror(SourceFetcher):
|
| 185 | 185 |
|
| 186 |
- def __init__(self, source, path, url, ref, *, primary=False, tags=[]):
|
|
| 186 |
+ def __init__(self, source, path, url, ref, *, primary=False, tags=[], tracking=None):
|
|
| 187 | 187 |
|
| 188 | 188 |
super().__init__()
|
| 189 | 189 |
self.source = source
|
| ... | ... | @@ -192,11 +192,101 @@ class GitMirror(SourceFetcher): |
| 192 | 192 |
self.ref = ref
|
| 193 | 193 |
self.tags = tags
|
| 194 | 194 |
self.primary = primary
|
| 195 |
+ dirname = utils.url_directory_name(url)
|
|
| 195 | 196 |
self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
|
| 197 |
+ self.fetch_mirror = os.path.join(source.get_mirror_directory(), '{}-{}'.format(dirname, ref))
|
|
| 196 | 198 |
self.mark_download_url(url)
|
| 199 |
+ self.tracking = tracking
|
|
| 200 |
+ |
|
| 201 |
+ def mirror_path(self):
|
|
| 202 |
+ if os.path.exists(self.mirror):
|
|
| 203 |
+ return self.mirror
|
|
| 204 |
+ else:
|
|
| 205 |
+ assert os.path.exists(self.fetch_mirror)
|
|
| 206 |
+ return self.fetch_mirror
|
|
| 207 |
+ |
|
| 208 |
+ def ensure_fetchable(self, alias_override=None):
|
|
| 209 |
+ |
|
| 210 |
+ if os.path.exists(self.mirror):
|
|
| 211 |
+ return
|
|
| 212 |
+ |
|
| 213 |
+ if self.tags:
|
|
| 214 |
+ for tag, commit, _ in self.tags:
|
|
| 215 |
+ if commit != self.ref:
|
|
| 216 |
+ self.source.status("{}: tag '{}' is not on commit '{}', so a full clone is required"
|
|
| 217 |
+ .format(self.source, tag, commit))
|
|
| 218 |
+ self.ensure_trackable(alias_override=alias_override)
|
|
| 219 |
+ return
|
|
| 220 |
+ |
|
| 221 |
+ if os.path.exists(self.fetch_mirror):
|
|
| 222 |
+ return
|
|
| 223 |
+ |
|
| 224 |
+ with self.source.tempdir() as tmpdir:
|
|
| 225 |
+ self.source.call([self.source.host_git, 'init', '--bare', tmpdir],
|
|
| 226 |
+ fail="Failed to init git repository",
|
|
| 227 |
+ fail_temporarily=True)
|
|
| 228 |
+ |
|
| 229 |
+ url = self.source.translate_url(self.url, alias_override=alias_override,
|
|
| 230 |
+ primary=self.primary)
|
|
| 231 |
+ |
|
| 232 |
+ self.source.call([self.source.host_git, 'remote', 'add', '--mirror=fetch', 'origin', url],
|
|
| 233 |
+ cwd=tmpdir,
|
|
| 234 |
+ fail="Failed to init git repository",
|
|
| 235 |
+ fail_temporarily=True)
|
|
| 236 |
+ |
|
| 237 |
+ _, refs = self.source.check_output([self.source.host_git, 'ls-remote', 'origin'],
|
|
| 238 |
+ cwd=tmpdir,
|
|
| 239 |
+ fail="Failed to clone git repository {}".format(url),
|
|
| 240 |
+ fail_temporarily=True)
|
|
| 241 |
+ |
|
| 242 |
+ advertised = None
|
|
| 243 |
+ for ref_line in refs.splitlines():
|
|
| 244 |
+ commit, ref = ref_line.split('\t', 1)
|
|
| 245 |
+ if ref == 'HEAD':
|
|
| 246 |
+ continue
|
|
| 247 |
+ if self.tracking:
|
|
| 248 |
+ # For validate_cache to work
|
|
| 249 |
+ if ref not in ['refs/heads/{}'.format(self.tracking),
|
|
| 250 |
+ 'refs/tags/{}'.format(self.tracking),
|
|
| 251 |
+ 'refs/tags/{}{}'.format(self.tracking, '^{}')]:
|
|
| 252 |
+ continue
|
|
| 253 |
+ if self.ref == commit:
|
|
| 254 |
+ if ref.endswith('^{}'):
|
|
| 255 |
+ ref = ref[:-3]
|
|
| 256 |
+ advertised = ref
|
|
| 257 |
+ break
|
|
| 258 |
+ |
|
| 259 |
+ if advertised is None:
|
|
| 260 |
+ self.source.status("{}: {} is not advertised on {}, so a full clone is required"
|
|
| 261 |
+ .format(self.source, self.ref, url))
|
|
| 262 |
+ |
|
| 263 |
+ self.ensure_trackable(alias_override=alias_override)
|
|
| 264 |
+ return
|
|
| 265 |
+ |
|
| 266 |
+ self.source.call([self.source.host_git, 'fetch', '--depth=1', 'origin', advertised],
|
|
| 267 |
+ cwd=tmpdir,
|
|
| 268 |
+ fail="Failed to fetch repository",
|
|
| 269 |
+ fail_temporarily=True)
|
|
| 270 |
+ |
|
| 271 |
+ # We need to have a ref to make it clonable
|
|
| 272 |
+ self.source.call([self.source.host_git, 'update-ref', 'HEAD', self.ref],
|
|
| 273 |
+ cwd=tmpdir,
|
|
| 274 |
+ fail="Failed to tag HEAD",
|
|
| 275 |
+ fail_temporarily=True)
|
|
| 276 |
+ |
|
| 277 |
+ try:
|
|
| 278 |
+ move_atomic(tmpdir, self.fetch_mirror)
|
|
| 279 |
+ except DirectoryExistsError:
|
|
| 280 |
+ # Another process was quicker to download this repository.
|
|
| 281 |
+ # Let's discard our own
|
|
| 282 |
+ self.source.status("{}: Discarding duplicate clone of {}"
|
|
| 283 |
+ .format(self.source, url))
|
|
| 284 |
+ except OSError as e:
|
|
| 285 |
+ raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}': {}"
|
|
| 286 |
+ .format(self.source, url, tmpdir, self.fetch_mirror, e)) from e
|
|
| 197 | 287 |
|
| 198 | 288 |
# Ensures that the mirror exists
|
| 199 |
- def ensure(self, alias_override=None):
|
|
| 289 |
+ def ensure_trackable(self, alias_override=None):
|
|
| 200 | 290 |
|
| 201 | 291 |
# Unfortunately, git does not know how to only clone just a specific ref,
|
| 202 | 292 |
# so we have to download all of those gigs even if we only need a couple
|
| ... | ... | @@ -231,18 +321,20 @@ class GitMirror(SourceFetcher): |
| 231 | 321 |
alias_override=alias_override,
|
| 232 | 322 |
primary=self.primary)
|
| 233 | 323 |
|
| 324 |
+ mirror = self.mirror_path()
|
|
| 325 |
+ |
|
| 234 | 326 |
if alias_override:
|
| 235 | 327 |
remote_name = utils.url_directory_name(alias_override)
|
| 236 | 328 |
_, remotes = self.source.check_output(
|
| 237 | 329 |
[self.source.host_git, 'remote'],
|
| 238 |
- fail="Failed to retrieve list of remotes in {}".format(self.mirror),
|
|
| 239 |
- cwd=self.mirror
|
|
| 330 |
+ fail="Failed to retrieve list of remotes in {}".format(mirror),
|
|
| 331 |
+ cwd=mirror
|
|
| 240 | 332 |
)
|
| 241 | 333 |
if remote_name not in remotes:
|
| 242 | 334 |
self.source.call(
|
| 243 | 335 |
[self.source.host_git, 'remote', 'add', remote_name, url],
|
| 244 | 336 |
fail="Failed to add remote {} with url {}".format(remote_name, url),
|
| 245 |
- cwd=self.mirror
|
|
| 337 |
+ cwd=mirror
|
|
| 246 | 338 |
)
|
| 247 | 339 |
else:
|
| 248 | 340 |
remote_name = "origin"
|
| ... | ... | @@ -250,7 +342,7 @@ class GitMirror(SourceFetcher): |
| 250 | 342 |
self.source.call([self.source.host_git, 'fetch', remote_name, '--prune', '--force', '--tags'],
|
| 251 | 343 |
fail="Failed to fetch from remote git repository: {}".format(url),
|
| 252 | 344 |
fail_temporarily=True,
|
| 253 |
- cwd=self.mirror)
|
|
| 345 |
+ cwd=mirror)
|
|
| 254 | 346 |
|
| 255 | 347 |
def fetch(self, alias_override=None):
|
| 256 | 348 |
# Resolve the URL for the message
|
| ... | ... | @@ -261,7 +353,7 @@ class GitMirror(SourceFetcher): |
| 261 | 353 |
with self.source.timed_activity("Fetching from {}"
|
| 262 | 354 |
.format(resolved_url),
|
| 263 | 355 |
silent_nested=True):
|
| 264 |
- self.ensure(alias_override)
|
|
| 356 |
+ self.ensure_fetchable(alias_override)
|
|
| 265 | 357 |
if not self.has_ref():
|
| 266 | 358 |
self._fetch(alias_override)
|
| 267 | 359 |
self.assert_ref()
|
| ... | ... | @@ -270,12 +362,14 @@ class GitMirror(SourceFetcher): |
| 270 | 362 |
if not self.ref:
|
| 271 | 363 |
return False
|
| 272 | 364 |
|
| 273 |
- # If the mirror doesnt exist, we also dont have the ref
|
|
| 274 |
- if not os.path.exists(self.mirror):
|
|
| 365 |
+ if not os.path.exists(self.mirror) and not os.path.exists(self.fetch_mirror):
|
|
| 366 |
+ # If the mirror doesnt exist, we also dont have the ref
|
|
| 275 | 367 |
return False
|
| 276 | 368 |
|
| 369 |
+ mirror = self.mirror_path()
|
|
| 370 |
+ |
|
| 277 | 371 |
# Check if the ref is really there
|
| 278 |
- rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=self.mirror)
|
|
| 372 |
+ rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=mirror)
|
|
| 279 | 373 |
return rc == 0
|
| 280 | 374 |
|
| 281 | 375 |
def assert_ref(self):
|
| ... | ... | @@ -325,11 +419,13 @@ class GitMirror(SourceFetcher): |
| 325 | 419 |
def stage(self, directory):
|
| 326 | 420 |
fullpath = os.path.join(directory, self.path)
|
| 327 | 421 |
|
| 422 |
+ mirror = self.mirror_path()
|
|
| 423 |
+ |
|
| 328 | 424 |
# Using --shared here avoids copying the objects into the checkout, in any
|
| 329 | 425 |
# case we're just checking out a specific commit and then removing the .git/
|
| 330 | 426 |
# directory.
|
| 331 |
- self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', self.mirror, fullpath],
|
|
| 332 |
- fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
|
|
| 427 |
+ self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', mirror, fullpath],
|
|
| 428 |
+ fail="Failed to create git mirror {} in directory: {}".format(mirror, fullpath),
|
|
| 333 | 429 |
fail_temporarily=True)
|
| 334 | 430 |
|
| 335 | 431 |
self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
|
| ... | ... | @@ -359,9 +455,11 @@ class GitMirror(SourceFetcher): |
| 359 | 455 |
|
| 360 | 456 |
# List the submodules (path/url tuples) present at the given ref of this repo
|
| 361 | 457 |
def submodule_list(self):
|
| 458 |
+ mirror = self.mirror_path()
|
|
| 459 |
+ |
|
| 362 | 460 |
modules = "{}:{}".format(self.ref, GIT_MODULES)
|
| 363 | 461 |
exit_code, output = self.source.check_output(
|
| 364 |
- [self.source.host_git, 'show', modules], cwd=self.mirror)
|
|
| 462 |
+ [self.source.host_git, 'show', modules], cwd=mirror)
|
|
| 365 | 463 |
|
| 366 | 464 |
# If git show reports error code 128 here, we take it to mean there is
|
| 367 | 465 |
# no .gitmodules file to display for the given revision.
|
| ... | ... | @@ -389,6 +487,8 @@ class GitMirror(SourceFetcher): |
| 389 | 487 |
# Fetch the ref which this mirror requires its submodule to have,
|
| 390 | 488 |
# at the given ref of this mirror.
|
| 391 | 489 |
def submodule_ref(self, submodule, ref=None):
|
| 490 |
+ mirror = self.mirror_path()
|
|
| 491 |
+ |
|
| 392 | 492 |
if not ref:
|
| 393 | 493 |
ref = self.ref
|
| 394 | 494 |
|
| ... | ... | @@ -397,7 +497,7 @@ class GitMirror(SourceFetcher): |
| 397 | 497 |
_, output = self.source.check_output([self.source.host_git, 'ls-tree', ref, submodule],
|
| 398 | 498 |
fail="ls-tree failed for commit {} and submodule: {}".format(
|
| 399 | 499 |
ref, submodule),
|
| 400 |
- cwd=self.mirror)
|
|
| 500 |
+ cwd=mirror)
|
|
| 401 | 501 |
|
| 402 | 502 |
# read the commit hash from the output
|
| 403 | 503 |
fields = output.split()
|
| ... | ... | @@ -514,8 +614,8 @@ class GitSource(Source): |
| 514 | 614 |
self.track_tags = self.node_get_member(node, bool, 'track-tags', False)
|
| 515 | 615 |
|
| 516 | 616 |
self.original_url = self.node_get_member(node, str, 'url')
|
| 517 |
- self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True)
|
|
| 518 | 617 |
self.tracking = self.node_get_member(node, str, 'track', None)
|
| 618 |
+ self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True, tracking=self.tracking)
|
|
| 519 | 619 |
|
| 520 | 620 |
self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1')
|
| 521 | 621 |
if self.ref_format not in ['sha1', 'git-describe']:
|
| ... | ... | @@ -633,7 +733,7 @@ class GitSource(Source): |
| 633 | 733 |
with self.timed_activity("Tracking {} from {}"
|
| 634 | 734 |
.format(self.tracking, resolved_url),
|
| 635 | 735 |
silent_nested=True):
|
| 636 |
- self.mirror.ensure()
|
|
| 736 |
+ self.mirror.ensure_trackable()
|
|
| 637 | 737 |
self.mirror._fetch()
|
| 638 | 738 |
|
| 639 | 739 |
# Update self.mirror.ref and node.ref from the self.tracking branch
|
| ... | ... | @@ -643,6 +743,7 @@ class GitSource(Source): |
| 643 | 743 |
|
| 644 | 744 |
def init_workspace(self, directory):
|
| 645 | 745 |
# XXX: may wish to refactor this as some code dupe with stage()
|
| 746 |
+ self.mirror.ensure_trackable()
|
|
| 646 | 747 |
self.refresh_submodules()
|
| 647 | 748 |
|
| 648 | 749 |
with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
|
| ... | ... | @@ -717,15 +818,16 @@ class GitSource(Source): |
| 717 | 818 |
# Assert that the ref exists in the track tag/branch, if track has been specified.
|
| 718 | 819 |
ref_in_track = False
|
| 719 | 820 |
if self.tracking:
|
| 821 |
+ mirror = self.mirror.mirror_path()
|
|
| 720 | 822 |
_, branch = self.check_output([self.host_git, 'branch', '--list', self.tracking,
|
| 721 | 823 |
'--contains', self.mirror.ref],
|
| 722 |
- cwd=self.mirror.mirror)
|
|
| 824 |
+ cwd=mirror)
|
|
| 723 | 825 |
if branch:
|
| 724 | 826 |
ref_in_track = True
|
| 725 | 827 |
else:
|
| 726 | 828 |
_, tag = self.check_output([self.host_git, 'tag', '--list', self.tracking,
|
| 727 | 829 |
'--contains', self.mirror.ref],
|
| 728 |
- cwd=self.mirror.mirror)
|
|
| 830 |
+ cwd=mirror)
|
|
| 729 | 831 |
if tag:
|
| 730 | 832 |
ref_in_track = True
|
| 731 | 833 |
|
| ... | ... | @@ -749,7 +851,7 @@ class GitSource(Source): |
| 749 | 851 |
|
| 750 | 852 |
self.refresh_submodules()
|
| 751 | 853 |
for mirror in self.submodules:
|
| 752 |
- if not os.path.exists(mirror.mirror):
|
|
| 854 |
+ if not os.path.exists(mirror.mirror) and not os.path.exists(mirror.fetch_mirror):
|
|
| 753 | 855 |
return False
|
| 754 | 856 |
if not mirror.has_ref():
|
| 755 | 857 |
return False
|
| ... | ... | @@ -761,7 +863,7 @@ class GitSource(Source): |
| 761 | 863 |
# Assumes that we have our mirror and we have the ref which we point to
|
| 762 | 864 |
#
|
| 763 | 865 |
def refresh_submodules(self):
|
| 764 |
- self.mirror.ensure()
|
|
| 866 |
+ self.mirror.ensure_fetchable()
|
|
| 765 | 867 |
submodules = []
|
| 766 | 868 |
|
| 767 | 869 |
for path, url in self.mirror.submodule_list():
|
| ... | ... | @@ -28,6 +28,7 @@ import shutil |
| 28 | 28 |
from buildstream._exceptions import ErrorDomain
|
| 29 | 29 |
from buildstream import _yaml
|
| 30 | 30 |
from buildstream.plugin import CoreWarnings
|
| 31 |
+from buildstream.utils import url_directory_name
|
|
| 31 | 32 |
|
| 32 | 33 |
from tests.testutils import cli, create_repo
|
| 33 | 34 |
from tests.testutils.site import HAVE_GIT
|
| ... | ... | @@ -1018,3 +1019,249 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles): |
| 1018 | 1019 |
|
| 1019 | 1020 |
result = cli.run(project=project, args=['build', 'target.bst'])
|
| 1020 | 1021 |
result.assert_success()
|
| 1022 |
+ |
|
| 1023 |
+ |
|
| 1024 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
| 1025 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
| 1026 |
+def test_fetch_shallow(cli, tmpdir, datafiles):
|
|
| 1027 |
+ project = str(datafiles)
|
|
| 1028 |
+ |
|
| 1029 |
+ repo = create_repo('git', str(tmpdir))
|
|
| 1030 |
+ previous_ref = repo.create(os.path.join(project, 'repofiles'))
|
|
| 1031 |
+ |
|
| 1032 |
+ file1 = os.path.join(str(tmpdir), 'file1')
|
|
| 1033 |
+ with open(file1, 'w') as f:
|
|
| 1034 |
+ f.write('test\n')
|
|
| 1035 |
+ ref = repo.add_file(file1)
|
|
| 1036 |
+ |
|
| 1037 |
+ source_config = repo.source_config(ref=ref)
|
|
| 1038 |
+ |
|
| 1039 |
+ # Write out our test target with a bad ref
|
|
| 1040 |
+ element = {
|
|
| 1041 |
+ 'kind': 'import',
|
|
| 1042 |
+ 'sources': [
|
|
| 1043 |
+ source_config
|
|
| 1044 |
+ ]
|
|
| 1045 |
+ }
|
|
| 1046 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
| 1047 |
+ |
|
| 1048 |
+ sources_dir = os.path.join(str(tmpdir), 'sources')
|
|
| 1049 |
+ os.makedirs(sources_dir, exist_ok=True)
|
|
| 1050 |
+ config = {
|
|
| 1051 |
+ 'sourcedir': sources_dir
|
|
| 1052 |
+ }
|
|
| 1053 |
+ cli.configure(config)
|
|
| 1054 |
+ |
|
| 1055 |
+ result = cli.run(project=project, args=[
|
|
| 1056 |
+ 'fetch', 'target.bst'
|
|
| 1057 |
+ ])
|
|
| 1058 |
+ result.assert_success()
|
|
| 1059 |
+ |
|
| 1060 |
+ cache_dir_name = url_directory_name(source_config['url'])
|
|
| 1061 |
+ full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
|
|
| 1062 |
+ shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, ref))
|
|
| 1063 |
+ |
|
| 1064 |
+ assert os.path.exists(shallow_cache_path)
|
|
| 1065 |
+ assert not os.path.exists(full_cache_path)
|
|
| 1066 |
+ |
|
| 1067 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
| 1068 |
+ cwd=shallow_cache_path,
|
|
| 1069 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
| 1070 |
+ assert output.splitlines() == [ref]
|
|
| 1071 |
+ |
|
| 1072 |
+ result = cli.run(project=project, args=[
|
|
| 1073 |
+ 'build', 'target.bst'
|
|
| 1074 |
+ ])
|
|
| 1075 |
+ result.assert_success()
|
|
| 1076 |
+ |
|
| 1077 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
| 1078 |
+ cwd=shallow_cache_path,
|
|
| 1079 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
| 1080 |
+ assert output.splitlines() == [ref]
|
|
| 1081 |
+ |
|
| 1082 |
+ assert os.path.exists(shallow_cache_path)
|
|
| 1083 |
+ assert not os.path.exists(full_cache_path)
|
|
| 1084 |
+ |
|
| 1085 |
+ result = cli.run(project=project, args=[
|
|
| 1086 |
+ 'track', 'target.bst'
|
|
| 1087 |
+ ])
|
|
| 1088 |
+ result.assert_success()
|
|
| 1089 |
+ |
|
| 1090 |
+ assert os.path.exists(full_cache_path)
|
|
| 1091 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
| 1092 |
+ cwd=full_cache_path,
|
|
| 1093 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
| 1094 |
+ assert output.splitlines() == [ref, previous_ref]
|
|
| 1095 |
+ |
|
| 1096 |
+ |
|
| 1097 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
| 1098 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
| 1099 |
+def test_fetch_shallow_not_tagged(cli, tmpdir, datafiles):
|
|
| 1100 |
+ """When a ref is not tagged and not head of branch on remote we cannot
|
|
| 1101 |
+ get a shallow clone. It should automatically get a full clone.
|
|
| 1102 |
+ """
|
|
| 1103 |
+ |
|
| 1104 |
+ project = str(datafiles)
|
|
| 1105 |
+ |
|
| 1106 |
+ repo = create_repo('git', str(tmpdir))
|
|
| 1107 |
+ previous_ref = repo.create(os.path.join(project, 'repofiles'))
|
|
| 1108 |
+ |
|
| 1109 |
+ file1 = os.path.join(str(tmpdir), 'file1')
|
|
| 1110 |
+ with open(file1, 'w') as f:
|
|
| 1111 |
+ f.write('test\n')
|
|
| 1112 |
+ ref = repo.add_file(file1)
|
|
| 1113 |
+ |
|
| 1114 |
+ source_config = repo.source_config(ref=previous_ref)
|
|
| 1115 |
+ |
|
| 1116 |
+ # Write out our test target with a bad ref
|
|
| 1117 |
+ element = {
|
|
| 1118 |
+ 'kind': 'import',
|
|
| 1119 |
+ 'sources': [
|
|
| 1120 |
+ source_config
|
|
| 1121 |
+ ]
|
|
| 1122 |
+ }
|
|
| 1123 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
| 1124 |
+ |
|
| 1125 |
+ sources_dir = os.path.join(str(tmpdir), 'sources')
|
|
| 1126 |
+ os.makedirs(sources_dir, exist_ok=True)
|
|
| 1127 |
+ config = {
|
|
| 1128 |
+ 'sourcedir': sources_dir
|
|
| 1129 |
+ }
|
|
| 1130 |
+ cli.configure(config)
|
|
| 1131 |
+ |
|
| 1132 |
+ result = cli.run(project=project, args=[
|
|
| 1133 |
+ 'source', 'fetch', 'target.bst'
|
|
| 1134 |
+ ])
|
|
| 1135 |
+ result.assert_success()
|
|
| 1136 |
+ |
|
| 1137 |
+ cache_dir_name = url_directory_name(source_config['url'])
|
|
| 1138 |
+ full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
|
|
| 1139 |
+ shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, previous_ref))
|
|
| 1140 |
+ |
|
| 1141 |
+ assert not os.path.exists(shallow_cache_path)
|
|
| 1142 |
+ assert os.path.exists(full_cache_path)
|
|
| 1143 |
+ |
|
| 1144 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
| 1145 |
+ cwd=full_cache_path,
|
|
| 1146 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
| 1147 |
+ assert output.splitlines() == [ref, previous_ref]
|
|
| 1148 |
+ |
|
| 1149 |
+ |
|
| 1150 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
| 1151 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
| 1152 |
+def test_fetch_shallow_annotated_tag(cli, tmpdir, datafiles):
|
|
| 1153 |
+ """When a ref is not tagged and not head of branch on remote we cannot
|
|
| 1154 |
+ get a shallow clone. It should automatically get a full clone.
|
|
| 1155 |
+ """
|
|
| 1156 |
+ |
|
| 1157 |
+ project = str(datafiles)
|
|
| 1158 |
+ |
|
| 1159 |
+ repo = create_repo('git', str(tmpdir))
|
|
| 1160 |
+ previous_ref = repo.create(os.path.join(project, 'repofiles'))
|
|
| 1161 |
+ |
|
| 1162 |
+ repo.add_annotated_tag('tag', 'tag')
|
|
| 1163 |
+ |
|
| 1164 |
+ file1 = os.path.join(str(tmpdir), 'file1')
|
|
| 1165 |
+ with open(file1, 'w') as f:
|
|
| 1166 |
+ f.write('test\n')
|
|
| 1167 |
+ ref = repo.add_file(file1)
|
|
| 1168 |
+ |
|
| 1169 |
+ source_config = repo.source_config(ref=previous_ref)
|
|
| 1170 |
+ del source_config['track']
|
|
| 1171 |
+ |
|
| 1172 |
+ # Write out our test target with a bad ref
|
|
| 1173 |
+ element = {
|
|
| 1174 |
+ 'kind': 'import',
|
|
| 1175 |
+ 'sources': [
|
|
| 1176 |
+ source_config
|
|
| 1177 |
+ ]
|
|
| 1178 |
+ }
|
|
| 1179 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
| 1180 |
+ |
|
| 1181 |
+ sources_dir = os.path.join(str(tmpdir), 'sources')
|
|
| 1182 |
+ os.makedirs(sources_dir, exist_ok=True)
|
|
| 1183 |
+ config = {
|
|
| 1184 |
+ 'sourcedir': sources_dir
|
|
| 1185 |
+ }
|
|
| 1186 |
+ cli.configure(config)
|
|
| 1187 |
+ |
|
| 1188 |
+ result = cli.run(project=project, args=[
|
|
| 1189 |
+ 'source', 'fetch', 'target.bst'
|
|
| 1190 |
+ ])
|
|
| 1191 |
+ result.assert_success()
|
|
| 1192 |
+ |
|
| 1193 |
+ cache_dir_name = url_directory_name(source_config['url'])
|
|
| 1194 |
+ full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
|
|
| 1195 |
+ shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, previous_ref))
|
|
| 1196 |
+ |
|
| 1197 |
+ assert os.path.exists(shallow_cache_path)
|
|
| 1198 |
+ assert not os.path.exists(full_cache_path)
|
|
| 1199 |
+ |
|
| 1200 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
| 1201 |
+ cwd=shallow_cache_path,
|
|
| 1202 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
| 1203 |
+ assert output.splitlines() == [previous_ref]
|
|
| 1204 |
+ |
|
| 1205 |
+ |
|
| 1206 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
| 1207 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
|
|
| 1208 |
+def test_fetch_shallow_workspace_open(cli, tmpdir, datafiles):
|
|
| 1209 |
+ """
|
|
| 1210 |
+ Workspaces should get a full clone.
|
|
| 1211 |
+ """
|
|
| 1212 |
+ project = str(datafiles)
|
|
| 1213 |
+ |
|
| 1214 |
+ repo = create_repo('git', str(tmpdir))
|
|
| 1215 |
+ previous_ref = repo.create(os.path.join(project, 'repofiles'))
|
|
| 1216 |
+ |
|
| 1217 |
+ file1 = os.path.join(str(tmpdir), 'file1')
|
|
| 1218 |
+ with open(file1, 'w') as f:
|
|
| 1219 |
+ f.write('test\n')
|
|
| 1220 |
+ ref = repo.add_file(file1)
|
|
| 1221 |
+ |
|
| 1222 |
+ source_config = repo.source_config(ref=ref)
|
|
| 1223 |
+ |
|
| 1224 |
+ # Write out our test target with a bad ref
|
|
| 1225 |
+ element = {
|
|
| 1226 |
+ 'kind': 'import',
|
|
| 1227 |
+ 'sources': [
|
|
| 1228 |
+ source_config
|
|
| 1229 |
+ ]
|
|
| 1230 |
+ }
|
|
| 1231 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
| 1232 |
+ |
|
| 1233 |
+ sources_dir = os.path.join(str(tmpdir), 'sources')
|
|
| 1234 |
+ os.makedirs(sources_dir, exist_ok=True)
|
|
| 1235 |
+ config = {
|
|
| 1236 |
+ 'sourcedir': sources_dir
|
|
| 1237 |
+ }
|
|
| 1238 |
+ cli.configure(config)
|
|
| 1239 |
+ |
|
| 1240 |
+ result = cli.run(project=project, args=[
|
|
| 1241 |
+ 'source', 'fetch', 'target.bst'
|
|
| 1242 |
+ ])
|
|
| 1243 |
+ result.assert_success()
|
|
| 1244 |
+ |
|
| 1245 |
+ cache_dir_name = url_directory_name(source_config['url'])
|
|
| 1246 |
+ full_cache_path = os.path.join(sources_dir, 'git', cache_dir_name)
|
|
| 1247 |
+ shallow_cache_path = os.path.join(sources_dir, 'git', '{}-{}'.format(cache_dir_name, ref))
|
|
| 1248 |
+ |
|
| 1249 |
+ assert os.path.exists(shallow_cache_path)
|
|
| 1250 |
+ assert not os.path.exists(full_cache_path)
|
|
| 1251 |
+ |
|
| 1252 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
| 1253 |
+ cwd=shallow_cache_path,
|
|
| 1254 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
| 1255 |
+ assert output.splitlines() == [ref]
|
|
| 1256 |
+ |
|
| 1257 |
+ workspace = os.path.join(str(tmpdir), 'workspace')
|
|
| 1258 |
+ |
|
| 1259 |
+ result = cli.run(project=project, args=[
|
|
| 1260 |
+ 'workspace', 'open', 'target.bst', '--directory', workspace
|
|
| 1261 |
+ ])
|
|
| 1262 |
+ result.assert_success()
|
|
| 1263 |
+ |
|
| 1264 |
+ output = subprocess.run(['git', 'log', '--format=format:%H'],
|
|
| 1265 |
+ cwd=workspace,
|
|
| 1266 |
+ stdout=subprocess.PIPE).stdout.decode('ascii')
|
|
| 1267 |
+ assert output.splitlines() == [ref, previous_ref]
|
