[Notes] [Git][BuildStream/buildstream][valentindavid/git_describe_tracking] Track of git tags and save them to reproduce minimum shallow repository

[Valentin David <gitlab mg gitlab com>]

Title: GitLab

Valentin David pushed to branch valentindavid/git_describe_tracking at BuildStream / buildstream

Commits:

• 55854c32
  by Valentin David at 2018-11-15T09:17:07Z

  Track of git tags and save them to reproduce minimum shallow repository
  
  Instead of tag information being fetched which can change with time,
  they are tracked and saved in the projects.refs/.bst. Then we re-tag
  automatically the closest tag so that `git describe` works and is
  reproducible.
  

3 changed files:

• buildstream/plugins/sources/git.py
• tests/sources/git.py
• tests/testutils/repo/git.py

Changes:

buildstream/plugins/sources/git.py

---

...	...	@@ -91,6 +91,7 @@ import re
91	91	import shutil
92	92	from collections.abc import Mapping
93	93	from io import StringIO
	94	+from tempfile import TemporaryFile
94	95
95	96	from configparser import RawConfigParser
96	97
...	...	@@ -110,13 +111,14 @@ INCONSISTENT_SUBMODULE = "inconsistent-submodules"
110	111	#
111	112	class GitMirror(SourceFetcher):
112	113
113		- def __init__(self, source, path, url, ref, *, primary=False):
	114	+ def __init__(self, source, path, url, ref, *, primary=False, tags=[]):
114	115
115	116	super().__init__()
116	117	self.source = source
117	118	self.path = path
118	119	self.url = url
119	120	self.ref = ref
	121	+ self.tags = tags
120	122	self.primary = primary
121	123	self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
122	124	self.mark_download_url(url)
...	...	@@ -230,7 +232,33 @@ class GitMirror(SourceFetcher):
230	232	if exit_code == 0:
231	233	ref = output.rstrip('\n')
232	234
233		- return ref
	235	+ tags = set()
	236	+ for options in [[], ['--first-parent'], ['--tags'], ['--tags', '--first-parent']]:
	237	+ exit_code, output = self.source.check_output(
	238	+ [self.source.host_git, 'describe', '--abbrev=0', ref] + options,
	239	+ cwd=self.mirror)
	240	+ if exit_code == 0:
	241	+ tag = output.strip()
	242	+ _, commit_ref = self.source.check_output(
	243	+ [self.source.host_git, 'rev-parse', tag + '^{commit}'],
	244	+ fail="Unable to resolve tag '{}'".format(tag),
	245	+ cwd=self.mirror)
	246	+ exit_code, tag_data = self.source.check_output(
	247	+ [self.source.host_git, 'cat-file', 'tag', tag],
	248	+ cwd=self.mirror)
	249	+ if exit_code != 0:
	250	+ tag_data = None
	251	+ tag_ref = None
	252	+ else:
	253	+ _, tag_ref = self.source.check_output(
	254	+ [self.source.host_git, 'rev-parse', tag],
	255	+ fail="Unable to resolve tag '{}'".format(tag),
	256	+ cwd=self.mirror)
	257	+ tag_ref = tag_ref.strip()
	258	+
	259	+ tags.add((tag, tag_ref, commit_ref.strip(), tag_data))
	260	+
	261	+ return ref, list(tags)
234	262
235	263	def stage(self, directory, track=None):
236	264	fullpath = os.path.join(directory, self.path)
...	...	@@ -246,13 +274,79 @@ class GitMirror(SourceFetcher):
246	274	fail="Failed to checkout git ref {}".format(self.ref),
247	275	cwd=fullpath)
248	276
	277	+ # Remove .git dir
	278	+ shutil.rmtree(os.path.join(fullpath, ".git"))
	279	+
	280	+ if self.tags:
	281	+ included = set()
	282	+ shallow = set()
	283	+ for _, _, commit_ref, _ in self.tags:
	284	+
	285	+ _, out = self.source.check_output([self.source.host_git, 'rev-list',
	286	+ '--boundary', '{}..{}'.format(commit_ref, self.ref)],
	287	+ fail="Failed to get git history {}..{} in directory: {}"
	288	+ .format(commit_ref, self.ref, fullpath),
	289	+ fail_temporarily=True,
	290	+ cwd=self.mirror)
	291	+ for line in out.splitlines():
	292	+ rev = line.lstrip('-')
	293	+ if line[0] == '-':
	294	+ shallow.add(rev)
	295	+ else:
	296	+ included.add(rev)
	297	+
	298	+ shallow -= included
	299	+ included |= shallow
	300	+
	301	+ self.source.call([self.source.host_git, 'init'],
	302	+ fail="Cannot initialize git repository: {}".format(fullpath),
	303	+ cwd=fullpath)
	304	+
	305	+ for rev in included:
	306	+ with TemporaryFile() as commit_file:
	307	+ self.source.call([self.source.host_git, 'cat-file', 'commit', rev],
	308	+ stdout=commit_file,
	309	+ fail="Failed to get commit {}".format(rev),
	310	+ cwd=self.mirror)
	311	+ commit_file.seek(0, 0)
	312	+ self.source.call([self.source.host_git, 'hash-object', '-w', '-t', 'commit', '--stdin'],
	313	+ stdin=commit_file,
	314	+ fail="Failed to add commit object {}".format(rev),
	315	+ cwd=fullpath)
	316	+
	317	+ with open(os.path.join(fullpath, '.git', 'shallow'), 'w') as shallow_file:
	318	+ for rev in shallow:
	319	+ shallow_file.write('{}\n'.format(rev))
	320	+
	321	+ for tag, tag_ref, commit_ref, tag_data in self.tags:
	322	+ if tag_ref and tag_data:
	323	+ with TemporaryFile() as tag_file:
	324	+ tag_file.write(tag_data.encode('UTF-8'))
	325	+ tag_file.write('\n'.encode('UTF-8'))
	326	+ tag_file.seek(0, 0)
	327	+ self.source.call([self.source.host_git, 'hash-object', '-w', '-t', 'tag', '--stdin'],
	328	+ stdin=tag_file,
	329	+ fail="Failed to add tag object {}".format(tag_ref),
	330	+ cwd=fullpath)
	331	+
	332	+ self.source.call([self.source.host_git, 'tag', tag, tag_ref],
	333	+ fail="Failed to tag: {}".format(tag),
	334	+ cwd=fullpath)
	335	+ else:
	336	+ self.source.call([self.source.host_git, 'tag', tag, commit_ref],
	337	+ fail="Failed to tag: {}".format(tag),
	338	+ cwd=fullpath)
	339	+
	340	+ with open(os.path.join(fullpath, '.git', 'HEAD'), 'w') as head:
	341	+ self.source.call([self.source.host_git, 'rev-parse', self.ref],
	342	+ stdout=head,
	343	+ fail="Failed to parse commit {}".format(self.ref),
	344	+ cwd=self.mirror)
	345	+
249	346	# Check that the user specified ref exists in the track if provided & not already tracked
250	347	if track:
251	348	self.assert_ref_in_track(fullpath, track)
252	349
253		- # Remove .git dir
254		- shutil.rmtree(os.path.join(fullpath, ".git"))
255		-
256	350	def init_workspace(self, directory, track=None):
257	351	fullpath = os.path.join(directory, self.path)
258	352	url = self.source.translate_url(self.url)
...	...	@@ -365,12 +459,23 @@ class GitSource(Source):
365	459
366	460	def configure(self, node):
367	461	ref = self.node_get_member(node, str, 'ref', None)
368		-
369		- config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules', 'ref-format']
	462	+ tags = []
	463	+ tags_node = self.node_get_member(node, list, 'tags', [])
	464	+ for tag_node in tags_node:
	465	+ tag = self.node_get_member(tag_node, str, 'tag')
	466	+ tag_ref = self.node_get_member(tag_node, str, 'ref', None)
	467	+ commit_ref = self.node_get_member(tag_node, str, 'commit')
	468	+ tag_data = self.node_get_member(tag_node, str, 'data', None)
	469	+ tags.append((tag, tag_ref, commit_ref, tag_data))
	470	+
	471	+ config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules', 'ref-format',
	472	+ 'tags']
370	473	self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
	474	+ for tag_node in tags_node:
	475	+ self.node_validate(tag_node, ['tag', 'ref', 'commit', 'data'])
371	476
372	477	self.original_url = self.node_get_member(node, str, 'url')
373		- self.mirror = GitMirror(self, '', self.original_url, ref, primary=True)
	478	+ self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True)
374	479	self.tracking = self.node_get_member(node, str, 'track', None)
375	480
376	481	self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1')
...	...	@@ -417,6 +522,8 @@ class GitSource(Source):
417	522	# the ref, if the user changes the alias to fetch the same sources
418	523	# from another location, it should not affect the cache key.
419	524	key = [self.original_url, self.mirror.ref]
	525	+ if self.mirror.tags:
	526	+ key.extend(self.mirror.tags)
420	527
421	528	# Only modify the cache key with checkout_submodules if it's something
422	529	# other than the default behaviour.
...	...	@@ -442,12 +549,43 @@ class GitSource(Source):
442	549
443	550	def load_ref(self, node):
444	551	self.mirror.ref = self.node_get_member(node, str, 'ref', None)
	552	+ tags_node = self.node_get_member(node, list, 'tags', [])
	553	+ self.mirror.tags = []
	554	+ for tag_node in tags_node:
	555	+ tag = self.node_get_member(tag_node, str, 'tag')
	556	+ tag_ref = self.node_get_member(tag_node, str, 'ref', None)
	557	+ commit_ref = self.node_get_member(tag_node, str, 'commit')
	558	+ tag_data = self.node_get_member(tag_node, str, 'data', None)
	559	+ self.mirror.tags.append((tag, tag_ref, commit_ref, tag_data))
445	560
446	561	def get_ref(self):
447		- return self.mirror.ref
448		-
449		- def set_ref(self, ref, node):
450		- node['ref'] = self.mirror.ref = ref
	562	+ return self.mirror.ref, self.mirror.tags
	563	+
	564	+ def set_ref(self, ref_data, node):
	565	+ if not ref_data:
	566	+ self.mirror.ref = None
	567	+ if 'ref' in node:
	568	+ del node['ref']
	569	+ self.mirror.tags = []
	570	+ if 'tags' in node:
	571	+ del node['tags']
	572	+ else:
	573	+ ref, tags = ref_data
	574	+ node['ref'] = self.mirror.ref = ref
	575	+ self.mirror.tags = tags
	576	+ if tags:
	577	+ node['tags'] = []
	578	+ for tag, tag_ref, commit_ref, tag_data in tags:
	579	+ data = {'tag': tag,
	580	+ 'commit': commit_ref}
	581	+ if tag_ref:
	582	+ data['ref'] = tag_ref
	583	+ if tag_data:
	584	+ data['data'] = tag_data
	585	+ node['tags'].append(data)
	586	+ else:
	587	+ if 'tags' in node:
	588	+ del node['tags']
451	589
452	590	def track(self):
453	591

tests/sources/git.py

---

...	...	@@ -22,6 +22,7 @@
22	22
23	23	import os
24	24	import pytest
	25	+import subprocess
25	26
26	27	from buildstream._exceptions import ErrorDomain
27	28	from buildstream import _yaml
...	...	@@ -523,3 +524,118 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
523	524	# Fetch it
524	525	result = cli.run(project=project, args=['fetch', 'target.bst'])
525	526	result.assert_success()
	527	+
	528	+
	529	+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
	530	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
	531	+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
	532	+@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')])
	533	+def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type):
	534	+ project = str(datafiles)
	535	+
	536	+ project_config = _yaml.load(os.path.join(project, 'project.conf'))
	537	+ project_config['ref-storage'] = ref_storage
	538	+ _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf'))
	539	+
	540	+ repofiles = os.path.join(str(tmpdir), 'repofiles')
	541	+ os.makedirs(repofiles, exist_ok=True)
	542	+ file0 = os.path.join(repofiles, 'file0')
	543	+ with open(file0, 'w') as f:
	544	+ f.write('test\n')
	545	+
	546	+ repo = create_repo('git', str(tmpdir))
	547	+
	548	+ def tag(name):
	549	+ if tag_type == 'annotated':
	550	+ repo.add_annotated_tag(name, name)
	551	+ else:
	552	+ repo.add_tag(name)
	553	+
	554	+ ref = repo.create(repofiles)
	555	+ tag('uselesstag')
	556	+
	557	+ file1 = os.path.join(str(tmpdir), 'file1')
	558	+ with open(file1, 'w') as f:
	559	+ f.write('test\n')
	560	+ repo.add_file(file1)
	561	+ tag('tag1')
	562	+
	563	+ file2 = os.path.join(str(tmpdir), 'file2')
	564	+ with open(file2, 'w') as f:
	565	+ f.write('test\n')
	566	+ repo.branch('branch2')
	567	+ repo.add_file(file2)
	568	+ tag('tag2')
	569	+
	570	+ repo.checkout('master')
	571	+ file3 = os.path.join(str(tmpdir), 'file3')
	572	+ with open(file3, 'w') as f:
	573	+ f.write('test\n')
	574	+ repo.add_file(file3)
	575	+
	576	+ repo.merge('branch2')
	577	+
	578	+ config = repo.source_config()
	579	+ config['track'] = repo.latest_commit()
	580	+
	581	+ # Write out our test target
	582	+ element = {
	583	+ 'kind': 'import',
	584	+ 'sources': [
	585	+ config
	586	+ ],
	587	+ }
	588	+ element_path = os.path.join(project, 'target.bst')
	589	+ _yaml.dump(element, element_path)
	590	+
	591	+ if ref_storage == 'inline':
	592	+ result = cli.run(project=project, args=['track', 'target.bst'])
	593	+ result.assert_success()
	594	+ else:
	595	+ result = cli.run(project=project, args=['track', 'target.bst', '--deps', 'all'])
	596	+ result.assert_success()
	597	+
	598	+ if ref_storage == 'inline':
	599	+ element = _yaml.load(element_path)
	600	+ tags = _yaml.node_sanitize(element['sources'][0]['tags'])
	601	+ assert len(tags) == 2
	602	+ for tag in tags:
	603	+ assert 'tag' in tag
	604	+ assert 'commit' in tag
	605	+ if tag_type == 'annotated':
	606	+ assert 'ref' in tag
	607	+ assert 'data' in tag
	608	+ else:
	609	+ assert 'ref' not in tag
	610	+ assert 'data' not in tag
	611	+
	612	+ assert set([(tag['tag'], tag['commit']) for tag in tags]) == set([('tag1', repo.rev_parse('tag1^{commit}')),
	613	+ ('tag2', repo.rev_parse('tag2^{commit}'))])
	614	+
	615	+ checkout = os.path.join(str(tmpdir), 'checkout')
	616	+
	617	+ result = cli.run(project=project, args=['build', 'target.bst'])
	618	+ result.assert_success()
	619	+ result = cli.run(project=project, args=['checkout', 'target.bst', checkout])
	620	+ result.assert_success()
	621	+
	622	+ if tag_type == 'annotated':
	623	+ options = []
	624	+ else:
	625	+ options = ['--tags']
	626	+ describe = subprocess.check_output(['git', 'describe'] + options,
	627	+ cwd=checkout)
	628	+ assert describe.startswith('tag2-2-')
	629	+
	630	+ describe_fp = subprocess.check_output(['git', 'describe', '--first-parent'] + options,
	631	+ cwd=checkout)
	632	+ assert describe_fp.startswith('tag1-2-')
	633	+
	634	+ tags = subprocess.check_output(['git', 'tag'],
	635	+ cwd=checkout)
	636	+ tags = set(tags.splitlines())
	637	+ assert tags == set(['tag1', 'tag2'])
	638	+
	639	+ p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')],
	640	+ cwd=checkout)
	641	+ assert p.returncode != 0

tests/testutils/repo/git.py

---

...	...	@@ -45,6 +45,9 @@ class Git(Repo):
45	45	def add_tag(self, tag):
46	46	self._run_git('tag', tag)
47	47
	48	+ def add_annotated_tag(self, tag, message):
	49	+ self._run_git('tag', '-a', tag, '-m', message)
	50	+
48	51	def add_commit(self):
49	52	self._run_git('commit', '--allow-empty', '-m', 'Additional commit')
50	53	return self.latest_commit()
...	...	@@ -95,3 +98,14 @@ class Git(Repo):
95	98
96	99	def branch(self, branch_name):
97	100	self._run_git('checkout', '-b', branch_name)
	101	+
	102	+ def checkout(self, commit):
	103	+ self._run_git('checkout', commit)
	104	+
	105	+ def merge(self, commit):
	106	+ self._run_git('merge', '-m', 'Merge', commit)
	107	+ return self.latest_commit()
	108	+
	109	+ def rev_parse(self, rev):
	110	+ output = self._run_git('rev-parse', rev, stdout=subprocess.PIPE).stdout
	111	+ return output.decode('UTF-8').strip()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.