[Notes] [Git][BuildStream/buildstream][tpollard/483] plugins/git.py: Warn if ref is not in given track

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/483 at BuildStream / buildstream

Commits:

• 264327dd
  by Tom Pollard at 2018-08-15T16:19:47Z

  plugins/git.py: Warn if ref is not in given track
  
  Add a helper function assert_ref_in_track to git.py GitMirror()
  which is used when staging & initing the source workspace. It
  checks the element's ref exists in the track (branch/tag) if it
  has been specified, raising a warning if necessary. The warning makes
  use of the warning token 'REF_NOT_IN_TRACK' from the configurable
  CoreWarnings. If the element has been tracked with bst, it is assumed
  that the value of ref exists in the track as it was generated from it
  & as such is not asserted.
  

2 changed files:

• buildstream/plugins/sources/git.py
• tests/sources/git.py

Changes:

buildstream/plugins/sources/git.py

---

...	...	@@ -87,6 +87,7 @@ from configparser import RawConfigParser
87	87
88	88	from buildstream import Source, SourceError, Consistency, SourceFetcher
89	89	from buildstream import utils
	90	+from buildstream.plugin import CoreWarnings
90	91
91	92	GIT_MODULES = '.gitmodules'
92	93
...	...	@@ -199,7 +200,7 @@ class GitMirror(SourceFetcher):
199	200	cwd=self.mirror)
200	201	return output.rstrip('\n')
201	202
202		- def stage(self, directory):
	203	+ def stage(self, directory, track=None):
203	204	fullpath = os.path.join(directory, self.path)
204	205
205	206	# Using --shared here avoids copying the objects into the checkout, in any
...	...	@@ -213,10 +214,14 @@ class GitMirror(SourceFetcher):
213	214	fail="Failed to checkout git ref {}".format(self.ref),
214	215	cwd=fullpath)
215	216
	217	+ # Check that the user specified ref exists in the track if provided & not already tracked
	218	+ if track:
	219	+ self.assert_ref_in_track(fullpath, track)
	220	+
216	221	# Remove .git dir
217	222	shutil.rmtree(os.path.join(fullpath, ".git"))
218	223
219		- def init_workspace(self, directory):
	224	+ def init_workspace(self, directory, track=None):
220	225	fullpath = os.path.join(directory, self.path)
221	226	url = self.source.translate_url(self.url)
222	227
...	...	@@ -232,6 +237,10 @@ class GitMirror(SourceFetcher):
232	237	fail="Failed to checkout git ref {}".format(self.ref),
233	238	cwd=fullpath)
234	239
	240	+ # Check that the user specified ref exists in the track if provided & not already tracked
	241	+ if track:
	242	+ self.assert_ref_in_track(fullpath, track)
	243	+
235	244	# List the submodules (path/url tuples) present at the given ref of this repo
236	245	def submodule_list(self):
237	246	modules = "{}:{}".format(self.ref, GIT_MODULES)
...	...	@@ -296,6 +305,28 @@ class GitMirror(SourceFetcher):
296	305
297	306	return None
298	307
	308	+ # Assert that ref exists in track, if track has been specified.
	309	+ def assert_ref_in_track(self, fullpath, track):
	310	+ _, branch = self.source.check_output([self.source.host_git, 'branch', '--list', track,
	311	+ '--contains', self.ref],
	312	+ cwd=fullpath,)
	313	+ if branch:
	314	+ return True
	315	+ else:
	316	+ _, tag = self.source.check_output([self.source.host_git, 'tag', '--list', track,
	317	+ '--contains', self.ref],
	318	+ cwd=fullpath,)
	319	+ if tag:
	320	+ return True
	321	+
	322	+ detail = "The ref provided for the element does not exist locally in the provided track branch / tag " + \
	323	+ "'{}'.\nYou may wish to track the element to update the ref from '{}' ".format(track, track) + \
	324	+ "with `bst track`,\nor examine the upstream at '{}' for the specific ref.".format(self.url)
	325	+
	326	+ self.source.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n"
	327	+ .format(self.source, self.ref, track, self.url),
	328	+ detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
	329	+
299	330
300	331	class GitSource(Source):
301	332	# pylint: disable=attribute-defined-outside-init
...	...	@@ -326,6 +357,7 @@ class GitSource(Source):
326	357	self.submodule_checkout_overrides[path] = checkout
327	358
328	359	self.mark_download_url(self.original_url)
	360	+ self.tracked = False
329	361
330	362	def preflight(self):
331	363	# Check if git is installed, get the binary at the same time
...	...	@@ -389,6 +421,8 @@ class GitSource(Source):
389	421	# Update self.mirror.ref and node.ref from the self.tracking branch
390	422	ret = self.mirror.latest_commit(self.tracking)
391	423
	424	+ # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed
	425	+ self.tracked = True
392	426	return ret
393	427
394	428	def init_workspace(self, directory):
...	...	@@ -396,7 +430,7 @@ class GitSource(Source):
396	430	self.refresh_submodules()
397	431
398	432	with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
399		- self.mirror.init_workspace(directory)
	433	+ self.mirror.init_workspace(directory, track=(self.tracking if not self.tracked else None))
400	434	for mirror in self.submodules:
401	435	mirror.init_workspace(directory)
402	436
...	...	@@ -412,7 +446,7 @@ class GitSource(Source):
412	446	# Stage the main repo in the specified directory
413	447	#
414	448	with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
415		- self.mirror.stage(directory)
	449	+ self.mirror.stage(directory, track=(self.tracking if not self.tracked else None))
416	450	for mirror in self.submodules:
417	451	if mirror.path in self.submodule_checkout_overrides:
418	452	checkout = self.submodule_checkout_overrides[mirror.path]

tests/sources/git.py

---

...	...	@@ -3,6 +3,7 @@ import pytest
3	3
4	4	from buildstream._exceptions import ErrorDomain
5	5	from buildstream import _yaml
	6	+from buildstream.plugin import CoreWarnings
6	7
7	8	from tests.testutils import cli, create_repo
8	9	from tests.testutils.site import HAVE_GIT
...	...	@@ -401,3 +402,70 @@ def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
401	402	assert "Exact versions are missing for the following elements" in result.stderr
402	403	assert "is missing ref and track." in result.stderr
403	404	assert "Then track these elements with `bst track`" in result.stderr
	405	+
	406	+
	407	+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
	408	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
	409	+def test_ref_not_in_track_warn(cli, tmpdir, datafiles):
	410	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	411	+
	412	+ # Create the repo from 'repofiles', create a branch without latest commit
	413	+ repo = create_repo('git', str(tmpdir))
	414	+ ref = repo.create(os.path.join(project, 'repofiles'))
	415	+
	416	+ gitsource = repo.source_config(ref=ref)
	417	+
	418	+ # Overwrite the track value to the added branch
	419	+ gitsource['track'] = 'foo'
	420	+
	421	+ # Write out our test target
	422	+ element = {
	423	+ 'kind': 'import',
	424	+ 'sources': [
	425	+ gitsource
	426	+ ]
	427	+ }
	428	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	429	+
	430	+ # Assert the warning is raised as ref is not in branch foo.
	431	+ # Assert warning not error to the user, when not set as fatal.
	432	+ result = cli.run(project=project, args=['build', 'target.bst'])
	433	+ assert "The ref provided for the element does not exist locally" in result.stderr
	434	+
	435	+
	436	+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
	437	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
	438	+def test_ref_not_in_track_warn_error(cli, tmpdir, datafiles):
	439	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	440	+
	441	+ # Add fatal-warnings ref-not-in-track to project.conf
	442	+ project_template = {
	443	+ "name": "foo",
	444	+ "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK]
	445	+ }
	446	+
	447	+ _yaml.dump(project_template, os.path.join(project, 'project.conf'))
	448	+
	449	+ # Create the repo from 'repofiles', create a branch without latest commit
	450	+ repo = create_repo('git', str(tmpdir))
	451	+ ref = repo.create(os.path.join(project, 'repofiles'))
	452	+
	453	+ gitsource = repo.source_config(ref=ref)
	454	+
	455	+ # Overwrite the track value to the added branch
	456	+ gitsource['track'] = 'foo'
	457	+
	458	+ # Write out our test target
	459	+ element = {
	460	+ 'kind': 'import',
	461	+ 'sources': [
	462	+ gitsource
	463	+ ]
	464	+ }
	465	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	466	+
	467	+ # Assert that build raises a warning here that is captured
	468	+ # as plugin errort, due to the fatal warning being set
	469	+ result = cli.run(project=project, args=['build', 'target.bst'])
	470	+ result.assert_main_error(ErrorDomain.STREAM, None)
	471	+ result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.