[Notes] [Git][BuildStream/buildstream][valentindavid/git_describe_tracking] 2 commits: Track of git tags and save them to reproduce minimum shallow repository

[Valentin David <gitlab mg gitlab com>]

Title: GitLab

Valentin David pushed to branch valentindavid/git_describe_tracking at BuildStream / buildstream

Commits:

• bccb335a
  by Valentin David at 2018-12-04T13:13:35Z

  Track of git tags and save them to reproduce minimum shallow repository
  
  Instead of tag information being fetched which can change with time,
  they are tracked and saved in the projects.refs/.bst. Then we re-tag
  automatically the closest tag so that `git describe` works and is
  reproducible.
  

• 0503cbd1
  by Valentin David at 2018-12-04T13:13:35Z

  NEWS: Add news about broken 'git describe' by default.
  

9 changed files:

• NEWS
• buildstream/_versions.py
• buildstream/plugins/sources/git.py
• + tests/cachekey/project/sources/git3.bst
• + tests/cachekey/project/sources/git3.expected
• tests/cachekey/project/target.bst
• tests/cachekey/project/target.expected
• tests/sources/git.py
• tests/testutils/repo/git.py

Changes:

NEWS

---

...	...	@@ -74,6 +74,10 @@ buildstream 1.3.1
74	74	o Add sandbox API for command batching and use it for build, script, and
75	75	compose elements.
76	76
	77	+ o BREAKING CHANGE: The `git` plugin does not create a local `.git`
	78	+ repository by default. If `git describe` is required to work, the
	79	+ plugin has now a tag tracking feature instead. This can be enabled
	80	+ by setting 'track-tags'.
77	81
78	82	=================
79	83	buildstream 1.1.5

buildstream/_versions.py

---

...	...	@@ -23,7 +23,7 @@
23	23	# This version is bumped whenever enhancements are made
24	24	# to the `project.conf` format or the core element format.
25	25	#
26		-BST_FORMAT_VERSION = 18
	26	+BST_FORMAT_VERSION = 19
27	27
28	28
29	29	# The base BuildStream artifact version

buildstream/plugins/sources/git.py

---

...	...	@@ -76,6 +76,24 @@ git - stage files from a git repository
76	76	url: upstream:baz.git
77	77	checkout: False
78	78
	79	+ # Enable tag tracking. This allows to create a dummy shallow
	80	+ # repository with necessary tag informations required to run 'git
	81	+ # describe'. Default is 'False'.
	82	+ track-tags: True
	83	+
	84	+ # A dummy shallow repository is created with the following tags.
	85	+ # Tags should not be edited by hand but rather be populated by
	86	+ # 'bst track' with 'track-tags' set to True. These tags can
	87	+ # also be stored in 'project.refs'. If you do not want to track
	88	+ # the 'ref' at the same time, set 'track' to the wanted commit.
	89	+ tags:
	90	+ - tag: lightweight-example
	91	+ commit: 04ad0dc656cb7cc6feb781aa13bdbf1d67d0af78
	92	+ annotated: false
	93	+ - tag: annotated-example
	94	+ commit: 10abe77fe8d77385d86f225b503d9185f4ef7f3a
	95	+ annotated: true
	96	+
79	97	See :ref:`built-in functionality doumentation <core_source_builtins>` for
80	98	details on common configuration options for sources.
81	99
...	...	@@ -95,6 +113,7 @@ import re
95	113	import shutil
96	114	from collections.abc import Mapping
97	115	from io import StringIO
	116	+from tempfile import TemporaryFile
98	117
99	118	from configparser import RawConfigParser
100	119
...	...	@@ -115,13 +134,14 @@ INCONSISTENT_SUBMODULE = "inconsistent-submodules"
115	134	#
116	135	class GitMirror(SourceFetcher):
117	136
118		- def __init__(self, source, path, url, ref, *, primary=False):
	137	+ def __init__(self, source, path, url, ref, *, primary=False, tags=[]):
119	138
120	139	super().__init__()
121	140	self.source = source
122	141	self.path = path
123	142	self.url = url
124	143	self.ref = ref
	144	+ self.tags = tags
125	145	self.primary = primary
126	146	self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
127	147	self.mark_download_url(url)
...	...	@@ -214,7 +234,7 @@ class GitMirror(SourceFetcher):
214	234	raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'"
215	235	.format(self.source, self.ref, self.url))
216	236
217		- def latest_commit(self, tracking):
	237	+ def latest_commit_with_tags(self, tracking, track_tags=False):
218	238	_, output = self.source.check_output(
219	239	[self.source.host_git, 'rev-parse', tracking],
220	240	fail="Unable to find commit for specified branch name '{}'".format(tracking),
...	...	@@ -230,7 +250,28 @@ class GitMirror(SourceFetcher):
230	250	if exit_code == 0:
231	251	ref = output.rstrip('\n')
232	252
233		- return ref
	253	+ if not track_tags:
	254	+ return ref, []
	255	+
	256	+ tags = set()
	257	+ for options in [[], ['--first-parent'], ['--tags'], ['--tags', '--first-parent']]:
	258	+ exit_code, output = self.source.check_output(
	259	+ [self.source.host_git, 'describe', '--abbrev=0', ref] + options,
	260	+ cwd=self.mirror)
	261	+ if exit_code == 0:
	262	+ tag = output.strip()
	263	+ _, commit_ref = self.source.check_output(
	264	+ [self.source.host_git, 'rev-parse', tag + '^{commit}'],
	265	+ fail="Unable to resolve tag '{}'".format(tag),
	266	+ cwd=self.mirror)
	267	+ exit_code = self.source.call(
	268	+ [self.source.host_git, 'cat-file', 'tag', tag],
	269	+ cwd=self.mirror)
	270	+ annotated = (exit_code == 0)
	271	+
	272	+ tags.add((tag, commit_ref.strip(), annotated))
	273	+
	274	+ return ref, list(tags)
234	275
235	276	def stage(self, directory, track=None):
236	277	fullpath = os.path.join(directory, self.path)
...	...	@@ -246,13 +287,15 @@ class GitMirror(SourceFetcher):
246	287	fail="Failed to checkout git ref {}".format(self.ref),
247	288	cwd=fullpath)
248	289
	290	+ # Remove .git dir
	291	+ shutil.rmtree(os.path.join(fullpath, ".git"))
	292	+
	293	+ self._rebuild_git(fullpath)
	294	+
249	295	# Check that the user specified ref exists in the track if provided & not already tracked
250	296	if track:
251	297	self.assert_ref_in_track(fullpath, track)
252	298
253		- # Remove .git dir
254		- shutil.rmtree(os.path.join(fullpath, ".git"))
255		-
256	299	def init_workspace(self, directory, track=None):
257	300	fullpath = os.path.join(directory, self.path)
258	301	url = self.source.translate_url(self.url)
...	...	@@ -359,6 +402,78 @@ class GitMirror(SourceFetcher):
359	402	.format(self.source, self.ref, track, self.url),
360	403	detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
361	404
	405	+ def _rebuild_git(self, fullpath):
	406	+ if not self.tags:
	407	+ return
	408	+
	409	+ with self.source.tempdir() as tmpdir:
	410	+ included = set()
	411	+ shallow = set()
	412	+ for _, commit_ref, _ in self.tags:
	413	+
	414	+ _, out = self.source.check_output([self.source.host_git, 'rev-list',
	415	+ '--boundary', '{}..{}'.format(commit_ref, self.ref)],
	416	+ fail="Failed to get git history {}..{} in directory: {}"
	417	+ .format(commit_ref, self.ref, fullpath),
	418	+ fail_temporarily=True,
	419	+ cwd=self.mirror)
	420	+ for line in out.splitlines():
	421	+ rev = line.lstrip('-')
	422	+ if line[0] == '-':
	423	+ shallow.add(rev)
	424	+ else:
	425	+ included.add(rev)
	426	+
	427	+ shallow -= included
	428	+ included |= shallow
	429	+
	430	+ self.source.call([self.source.host_git, 'init'],
	431	+ fail="Cannot initialize git repository: {}".format(fullpath),
	432	+ cwd=fullpath)
	433	+
	434	+ for rev in included:
	435	+ with TemporaryFile(dir=tmpdir) as commit_file:
	436	+ self.source.call([self.source.host_git, 'cat-file', 'commit', rev],
	437	+ stdout=commit_file,
	438	+ fail="Failed to get commit {}".format(rev),
	439	+ cwd=self.mirror)
	440	+ commit_file.seek(0, 0)
	441	+ self.source.call([self.source.host_git, 'hash-object', '-w', '-t', 'commit', '--stdin'],
	442	+ stdin=commit_file,
	443	+ fail="Failed to add commit object {}".format(rev),
	444	+ cwd=fullpath)
	445	+
	446	+ with open(os.path.join(fullpath, '.git', 'shallow'), 'w') as shallow_file:
	447	+ for rev in shallow:
	448	+ shallow_file.write('{}\n'.format(rev))
	449	+
	450	+ for tag, commit_ref, annotated in self.tags:
	451	+ if annotated:
	452	+ with TemporaryFile(dir=tmpdir) as tag_file:
	453	+ tag_data = 'object {}\ntype commit\ntag {}\n'.format(commit_ref, tag)
	454	+ tag_file.write(tag_data.encode('ascii'))
	455	+ tag_file.seek(0, 0)
	456	+ _, tag_ref = self.source.check_output(
	457	+ [self.source.host_git, 'hash-object', '-w', '-t',
	458	+ 'tag', '--stdin'],
	459	+ stdin=tag_file,
	460	+ fail="Failed to add tag object {}".format(tag),
	461	+ cwd=fullpath)
	462	+
	463	+ self.source.call([self.source.host_git, 'tag', tag, tag_ref.strip()],
	464	+ fail="Failed to tag: {}".format(tag),
	465	+ cwd=fullpath)
	466	+ else:
	467	+ self.source.call([self.source.host_git, 'tag', tag, commit_ref],
	468	+ fail="Failed to tag: {}".format(tag),
	469	+ cwd=fullpath)
	470	+
	471	+ with open(os.path.join(fullpath, '.git', 'HEAD'), 'w') as head:
	472	+ self.source.call([self.source.host_git, 'rev-parse', self.ref],
	473	+ stdout=head,
	474	+ fail="Failed to parse commit {}".format(self.ref),
	475	+ cwd=self.mirror)
	476	+
362	477
363	478	class GitSource(Source):
364	479	# pylint: disable=attribute-defined-outside-init
...	...	@@ -366,11 +481,20 @@ class GitSource(Source):
366	481	def configure(self, node):
367	482	ref = self.node_get_member(node, str, 'ref', None)
368	483
369		- config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules', 'ref-format']
	484	+ config_keys = ['url', 'track', 'ref', 'submodules',
	485	+ 'checkout-submodules', 'ref-format',
	486	+ 'track-tags', 'tags']
370	487	self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
371	488
	489	+ tags_node = self.node_get_member(node, list, 'tags', [])
	490	+ for tag_node in tags_node:
	491	+ self.node_validate(tag_node, ['tag', 'commit', 'annotated'])
	492	+
	493	+ tags = self._load_tags(node)
	494	+ self.track_tags = self.node_get_member(node, bool, 'track-tags', False)
	495	+
372	496	self.original_url = self.node_get_member(node, str, 'url')
373		- self.mirror = GitMirror(self, '', self.original_url, ref, primary=True)
	497	+ self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True)
374	498	self.tracking = self.node_get_member(node, str, 'track', None)
375	499
376	500	self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1')
...	...	@@ -417,6 +541,9 @@ class GitSource(Source):
417	541	# the ref, if the user changes the alias to fetch the same sources
418	542	# from another location, it should not affect the cache key.
419	543	key = [self.original_url, self.mirror.ref]
	544	+ if self.mirror.tags:
	545	+ tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags}
	546	+ key.append({'tags': tags})
420	547
421	548	# Only modify the cache key with checkout_submodules if it's something
422	549	# other than the default behaviour.
...	...	@@ -442,12 +569,33 @@ class GitSource(Source):
442	569
443	570	def load_ref(self, node):
444	571	self.mirror.ref = self.node_get_member(node, str, 'ref', None)
	572	+ self.mirror.tags = self._load_tags(node)
445	573
446	574	def get_ref(self):
447		- return self.mirror.ref
448		-
449		- def set_ref(self, ref, node):
450		- node['ref'] = self.mirror.ref = ref
	575	+ return self.mirror.ref, self.mirror.tags
	576	+
	577	+ def set_ref(self, ref_data, node):
	578	+ if not ref_data:
	579	+ self.mirror.ref = None
	580	+ if 'ref' in node:
	581	+ del node['ref']
	582	+ self.mirror.tags = []
	583	+ if 'tags' in node:
	584	+ del node['tags']
	585	+ else:
	586	+ ref, tags = ref_data
	587	+ node['ref'] = self.mirror.ref = ref
	588	+ self.mirror.tags = tags
	589	+ if tags:
	590	+ node['tags'] = []
	591	+ for tag, commit_ref, annotated in tags:
	592	+ data = {'tag': tag,
	593	+ 'commit': commit_ref,
	594	+ 'annotated': annotated}
	595	+ node['tags'].append(data)
	596	+ else:
	597	+ if 'tags' in node:
	598	+ del node['tags']
451	599
452	600	def track(self):
453	601
...	...	@@ -470,7 +618,7 @@ class GitSource(Source):
470	618	self.mirror._fetch()
471	619
472	620	# Update self.mirror.ref and node.ref from the self.tracking branch
473		- ret = self.mirror.latest_commit(self.tracking)
	621	+ ret = self.mirror.latest_commit_with_tags(self.tracking, self.track_tags)
474	622
475	623	# Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed
476	624	self.tracked = True
...	...	@@ -556,6 +704,16 @@ class GitSource(Source):
556	704
557	705	self.submodules = submodules
558	706
	707	+ def _load_tags(self, node):
	708	+ tags = []
	709	+ tags_node = self.node_get_member(node, list, 'tags', [])
	710	+ for tag_node in tags_node:
	711	+ tag = self.node_get_member(tag_node, str, 'tag')
	712	+ commit_ref = self.node_get_member(tag_node, str, 'commit')
	713	+ annotated = self.node_get_member(tag_node, bool, 'annotated')
	714	+ tags.append((tag, commit_ref, annotated))
	715	+ return tags
	716	+
559	717
560	718	# Plugin entry point
561	719	def setup():

tests/cachekey/project/sources/git3.bst

---

	1	+kind: import
	2	+sources:
	3	+- kind: git
	4	+ url: https://example.com/git/repo.git
	5	+ ref: 6ac68af3e80b7b17c23a3c65233043550a7fa685
	6	+ tags:
	7	+ - tag: lightweight
	8	+ commit: 0a3917d57477ee9afe7be49a0e8a76f56d176df1
	9	+ annotated: false
	10	+ - tag: annotated
	11	+ commit: 68c7f0bd386684742c41ec2a54ce2325e3922f6c
	12	+ annotated: true

tests/cachekey/project/sources/git3.expected

---

	1	+6a25f539bd8629a36399c58efd2f5c9c117feb845076a37dc321b55d456932b6
		\ No newline at end of file

tests/cachekey/project/target.bst

---

...	...	@@ -7,6 +7,7 @@ depends:
7	7	- sources/bzr1.bst
8	8	- sources/git1.bst
9	9	- sources/git2.bst
	10	+- sources/git3.bst
10	11	- sources/local1.bst
11	12	- sources/local2.bst
12	13	- sources/ostree1.bst

tests/cachekey/project/target.expected

---

1		-125d9e7dcf4f49e5f80d85b7f144b43ed43186064afc2e596e57f26cce679cf5
		\ No newline at end of file
	1	+bc99c288f855ac2619787f0067223f7812d2e10a9d2c7f2bf47de7113c0fd25c
		\ No newline at end of file

tests/sources/git.py

---

...	...	@@ -22,6 +22,7 @@
22	22
23	23	import os
24	24	import pytest
	25	+import subprocess
25	26
26	27	from buildstream._exceptions import ErrorDomain
27	28	from buildstream import _yaml
...	...	@@ -523,3 +524,155 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
523	524	# Fetch it
524	525	result = cli.run(project=project, args=['fetch', 'target.bst'])
525	526	result.assert_success()
	527	+
	528	+
	529	+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
	530	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
	531	+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
	532	+@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')])
	533	+def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type):
	534	+ project = str(datafiles)
	535	+
	536	+ project_config = _yaml.load(os.path.join(project, 'project.conf'))
	537	+ project_config['ref-storage'] = ref_storage
	538	+ _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf'))
	539	+
	540	+ repofiles = os.path.join(str(tmpdir), 'repofiles')
	541	+ os.makedirs(repofiles, exist_ok=True)
	542	+ file0 = os.path.join(repofiles, 'file0')
	543	+ with open(file0, 'w') as f:
	544	+ f.write('test\n')
	545	+
	546	+ repo = create_repo('git', str(tmpdir))
	547	+
	548	+ def tag(name):
	549	+ if tag_type == 'annotated':
	550	+ repo.add_annotated_tag(name, name)
	551	+ else:
	552	+ repo.add_tag(name)
	553	+
	554	+ ref = repo.create(repofiles)
	555	+ tag('uselesstag')
	556	+
	557	+ file1 = os.path.join(str(tmpdir), 'file1')
	558	+ with open(file1, 'w') as f:
	559	+ f.write('test\n')
	560	+ repo.add_file(file1)
	561	+ tag('tag1')
	562	+
	563	+ file2 = os.path.join(str(tmpdir), 'file2')
	564	+ with open(file2, 'w') as f:
	565	+ f.write('test\n')
	566	+ repo.branch('branch2')
	567	+ repo.add_file(file2)
	568	+ tag('tag2')
	569	+
	570	+ repo.checkout('master')
	571	+ file3 = os.path.join(str(tmpdir), 'file3')
	572	+ with open(file3, 'w') as f:
	573	+ f.write('test\n')
	574	+ repo.add_file(file3)
	575	+
	576	+ repo.merge('branch2')
	577	+
	578	+ config = repo.source_config()
	579	+ config['track'] = repo.latest_commit()
	580	+ config['track-tags'] = True
	581	+
	582	+ # Write out our test target
	583	+ element = {
	584	+ 'kind': 'import',
	585	+ 'sources': [
	586	+ config
	587	+ ],
	588	+ }
	589	+ element_path = os.path.join(project, 'target.bst')
	590	+ _yaml.dump(element, element_path)
	591	+
	592	+ if ref_storage == 'inline':
	593	+ result = cli.run(project=project, args=['track', 'target.bst'])
	594	+ result.assert_success()
	595	+ else:
	596	+ result = cli.run(project=project, args=['track', 'target.bst', '--deps', 'all'])
	597	+ result.assert_success()
	598	+
	599	+ if ref_storage == 'inline':
	600	+ element = _yaml.load(element_path)
	601	+ tags = _yaml.node_sanitize(element['sources'][0]['tags'])
	602	+ assert len(tags) == 2
	603	+ for tag in tags:
	604	+ assert 'tag' in tag
	605	+ assert 'commit' in tag
	606	+ assert 'annotated' in tag
	607	+ assert tag['annotated'] == (tag_type == 'annotated')
	608	+
	609	+ assert set([(tag['tag'], tag['commit']) for tag in tags]) == set([('tag1', repo.rev_parse('tag1^{commit}')),
	610	+ ('tag2', repo.rev_parse('tag2^{commit}'))])
	611	+
	612	+ checkout = os.path.join(str(tmpdir), 'checkout')
	613	+
	614	+ result = cli.run(project=project, args=['build', 'target.bst'])
	615	+ result.assert_success()
	616	+ result = cli.run(project=project, args=['checkout', 'target.bst', checkout])
	617	+ result.assert_success()
	618	+
	619	+ if tag_type == 'annotated':
	620	+ options = []
	621	+ else:
	622	+ options = ['--tags']
	623	+ describe = subprocess.check_output(['git', 'describe'] + options,
	624	+ cwd=checkout).decode('ascii')
	625	+ assert describe.startswith('tag2-2-')
	626	+
	627	+ describe_fp = subprocess.check_output(['git', 'describe', '--first-parent'] + options,
	628	+ cwd=checkout).decode('ascii')
	629	+ assert describe_fp.startswith('tag1-2-')
	630	+
	631	+ tags = subprocess.check_output(['git', 'tag'],
	632	+ cwd=checkout).decode('ascii')
	633	+ tags = set(tags.splitlines())
	634	+ assert tags == set(['tag1', 'tag2'])
	635	+
	636	+ p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')],
	637	+ cwd=checkout)
	638	+ assert p.returncode != 0
	639	+
	640	+
	641	+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
	642	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
	643	+def test_default_do_not_track_tags(cli, tmpdir, datafiles):
	644	+ project = str(datafiles)
	645	+
	646	+ project_config = _yaml.load(os.path.join(project, 'project.conf'))
	647	+ project_config['ref-storage'] = 'inline'
	648	+ _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf'))
	649	+
	650	+ repofiles = os.path.join(str(tmpdir), 'repofiles')
	651	+ os.makedirs(repofiles, exist_ok=True)
	652	+ file0 = os.path.join(repofiles, 'file0')
	653	+ with open(file0, 'w') as f:
	654	+ f.write('test\n')
	655	+
	656	+ repo = create_repo('git', str(tmpdir))
	657	+
	658	+ ref = repo.create(repofiles)
	659	+ repo.add_tag('tag')
	660	+
	661	+ config = repo.source_config()
	662	+ config['track'] = repo.latest_commit()
	663	+
	664	+ # Write out our test target
	665	+ element = {
	666	+ 'kind': 'import',
	667	+ 'sources': [
	668	+ config
	669	+ ],
	670	+ }
	671	+ element_path = os.path.join(project, 'target.bst')
	672	+ _yaml.dump(element, element_path)
	673	+
	674	+ result = cli.run(project=project, args=['track', 'target.bst'])
	675	+ result.assert_success()
	676	+
	677	+ element = _yaml.load(element_path)
	678	+ assert 'tags' not in element['sources'][0]

tests/testutils/repo/git.py

---

...	...	@@ -45,6 +45,9 @@ class Git(Repo):
45	45	def add_tag(self, tag):
46	46	self._run_git('tag', tag)
47	47
	48	+ def add_annotated_tag(self, tag, message):
	49	+ self._run_git('tag', '-a', tag, '-m', message)
	50	+
48	51	def add_commit(self):
49	52	self._run_git('commit', '--allow-empty', '-m', 'Additional commit')
50	53	return self.latest_commit()
...	...	@@ -95,3 +98,14 @@ class Git(Repo):
95	98
96	99	def branch(self, branch_name):
97	100	self._run_git('checkout', '-b', branch_name)
	101	+
	102	+ def checkout(self, commit):
	103	+ self._run_git('checkout', commit)
	104	+
	105	+ def merge(self, commit):
	106	+ self._run_git('merge', '-m', 'Merge', commit)
	107	+ return self.latest_commit()
	108	+
	109	+ def rev_parse(self, rev):
	110	+ output = self._run_git('rev-parse', rev, stdout=subprocess.PIPE).stdout
	111	+ return output.decode('UTF-8').strip()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.