[Notes] [Git][BuildStream/buildstream][valentindavid/git_describe_tracking] Track of git tags and save them to reproduce minimum shallow repository

[Valentin David <gitlab mg gitlab com>]

Title: GitLab

Valentin David pushed to branch valentindavid/git_describe_tracking at BuildStream / buildstream

Commits:

• 0fdc3d12
  by Valentin David at 2018-11-01T10:20:07Z

  Track of git tags and save them to reproduce minimum shallow repository
  
  Instead of tag information being fetched which can change with time,
  they are tracked and saved in the projects.refs/.bst. Then we re-tag
  automatically the closest tag so that `git describe` works and is
  reproducible.
  

1 changed file:

• buildstream/plugins/sources/git.py

Changes:

buildstream/plugins/sources/git.py

---

...	...	@@ -110,13 +110,15 @@ INCONSISTENT_SUBMODULE = "inconsistent-submodules"
110	110	#
111	111	class GitMirror(SourceFetcher):
112	112
113		- def __init__(self, source, path, url, ref, *, primary=False):
	113	+ def __init__(self, source, path, url, ref, *, primary=False, tag=None, tag_ref=None):
114	114
115	115	super().__init__()
116	116	self.source = source
117	117	self.path = path
118	118	self.url = url
119	119	self.ref = ref
	120	+ self.tag = tag
	121	+ self.tag_ref = tag_ref
120	122	self.primary = primary
121	123	self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
122	124	self.mark_download_url(url)
...	...	@@ -230,28 +232,71 @@ class GitMirror(SourceFetcher):
230	232	if exit_code == 0:
231	233	ref = output.rstrip('\n')
232	234
233		- return ref
	235	+ exit_code, output = self.source.check_output(
	236	+ [self.source.host_git, 'describe', '--abbrev=0', ref],
	237	+ cwd=self.mirror)
	238	+ if exit_code == 0:
	239	+ tag = output.strip()
	240	+ exit_code, output = self.source.check_output(
	241	+ [self.source.host_git, 'rev-parse', ref],
	242	+ cwd=self.mirror)
	243	+ tag_ref = output.strip()
	244	+ else:
	245	+ tag_ref = None
	246	+ tag = None
	247	+
	248	+ return ref, tag, tag_ref
234	249
235	250	def stage(self, directory, track=None):
236	251	fullpath = os.path.join(directory, self.path)
237	252
238		- # Using --shared here avoids copying the objects into the checkout, in any
239		- # case we're just checking out a specific commit and then removing the .git/
240		- # directory.
241		- self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', self.mirror, fullpath],
242		- fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
243		- fail_temporarily=True)
	253	+ if self.tag and self.tag_ref:
	254	+ _, out = self.source.check_output([self.source.host_git, 'log',
	255	+ '--ancestry-path', '--boundary',
	256	+ '--full-history',
	257	+ '--format=format:%H %P',
	258	+ '{}..{}'.format(self.tag_ref, self.ref)],
	259	+ fail="Failed to create git mirror {} in directory: {}"
	260	+ .format(self.mirror, fullpath),
	261	+ fail_temporarily=True,
	262	+ cwd=self.mirror)
	263	+ included = set()
	264	+ parents = set()
	265	+ for line in out.splitlines():
	266	+ refs = line.split(' ')
	267	+ included.add(refs[0])
	268	+ for r in refs[1:]:
	269	+ parents.add(r)
	270	+ excludes = ['--shallow-exclude={}'.format(excluded) for excluded in parents - included]
	271	+ self.source.call([self.source.host_git, 'clone', '--no-checkout', '--no-tags'] +
	272	+ excludes + [self.mirror, fullpath],
	273	+ fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
	274	+ fail_temporarily=True)
	275	+ else:
	276	+ # Using --shared here avoids copying the objects into the checkout, in any
	277	+ # case we're just checking out a specific commit and then removing the .git/
	278	+ # directory.
	279	+ self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', self.mirror, fullpath],
	280	+ fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
	281	+ fail_temporarily=True)
244	282
245	283	self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
246	284	fail="Failed to checkout git ref {}".format(self.ref),
247	285	cwd=fullpath)
248	286
	287	+ if self.tag and self.tag_ref:
	288	+ self.source.call([self.source.host_git, 'tag', self.tag, self.tag_ref],
	289	+ fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
	290	+ fail_temporarily=True,
	291	+ cwd=fullpath)
	292	+
249	293	# Check that the user specified ref exists in the track if provided & not already tracked
250	294	if track:
251	295	self.assert_ref_in_track(fullpath, track)
252	296
253		- # Remove .git dir
254		- shutil.rmtree(os.path.join(fullpath, ".git"))
	297	+ if not (self.tag and self.tag_ref):
	298	+ # Remove .git dir
	299	+ shutil.rmtree(os.path.join(fullpath, ".git"))
255	300
256	301	def init_workspace(self, directory, track=None):
257	302	fullpath = os.path.join(directory, self.path)
...	...	@@ -365,12 +410,15 @@ class GitSource(Source):
365	410
366	411	def configure(self, node):
367	412	ref = self.node_get_member(node, str, 'ref', None)
	413	+ tag = self.node_get_member(node, str, 'tracked-tag', None)
	414	+ tag_ref = self.node_get_member(node, str, 'tracked-tag-ref', None)
368	415
369		- config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules', 'ref-format']
	416	+ config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules', 'ref-format',
	417	+ 'tracked-tag', 'tracked-tag-ref']
370	418	self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
371	419
372	420	self.original_url = self.node_get_member(node, str, 'url')
373		- self.mirror = GitMirror(self, '', self.original_url, ref, primary=True)
	421	+ self.mirror = GitMirror(self, '', self.original_url, ref, tag=tag, tag_ref=tag_ref, primary=True)
374	422	self.tracking = self.node_get_member(node, str, 'track', None)
375	423
376	424	self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1')
...	...	@@ -417,6 +465,8 @@ class GitSource(Source):
417	465	# the ref, if the user changes the alias to fetch the same sources
418	466	# from another location, it should not affect the cache key.
419	467	key = [self.original_url, self.mirror.ref]
	468	+ if self.mirror.tag or self.mirror.tag_ref:
	469	+ key.extend([self.mirror.tag, self.mirror.tag_ref])
420	470
421	471	# Only modify the cache key with checkout_submodules if it's something
422	472	# other than the default behaviour.
...	...	@@ -442,12 +492,24 @@ class GitSource(Source):
442	492
443	493	def load_ref(self, node):
444	494	self.mirror.ref = self.node_get_member(node, str, 'ref', None)
	495	+ self.mirror.tag = self.node_get_member(node, str, 'tracked-tag', None)
	496	+ self.mirror.tag_ref = self.node_get_member(node, str, 'tracked-tag-ref', None)
445	497
446	498	def get_ref(self):
447		- return self.mirror.ref
	499	+ return self.mirror.ref, self.mirror.tag, self.mirror.tag_ref
448	500
449		- def set_ref(self, ref, node):
450		- node['ref'] = self.mirror.ref = ref
	501	+ def set_ref(self, ref_data, node):
	502	+ if not ref_data:
	503	+ ref = None
	504	+ tag = None
	505	+ tag_ref = None
	506	+ else:
	507	+ ref, tag, tag_ref = ref_data
	508	+ node['ref'] = self.mirror.ref = ref
	509	+ if tag:
	510	+ node['tracked-tag'] = self.mirror.tag = tag
	511	+ if tag_ref:
	512	+ node['tracked-tag-ref'] = self.mirror.tag_ref = tag_ref
451	513
452	514	def track(self):
453	515

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.