[Notes] [Git][BuildStream/buildstream][master] 3 commits: plugins/git.py: Warn if ref is not in given track

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch master at BuildStream / buildstream

Commits:

• 2339f0c4
  by Tom Pollard at 2018-09-03T09:44:36Z

  plugins/git.py: Warn if ref is not in given track
  
  Add a helper function assert_ref_in_track to git.py GitMirror()
  which is used when staging & initing the source workspace. It
  checks the element's ref exists in the track (branch/tag) if it
  has been specified, raising a warning if necessary. The warning makes
  use of the warning token 'REF_NOT_IN_TRACK' from the configurable
  CoreWarnings. If the element has been tracked with bst, it is assumed
  that the value of ref exists in the track as it was generated from it
  & as such is not asserted.
  

• c96fec5d
  by Tom Pollard at 2018-09-03T09:44:36Z

  tests/sources/git.py: Add tests for REF_NOT_IN_TRACK
  
  Add tests that cover assert_ref_in_track & the configurable
  CoreWarnings REF_NOT_IN_TRACK warnings token.
  

• 6a0cdedf
  by Tom Pollard at 2018-09-03T10:16:39Z

  Merge branch 'tpollard/483' into 'master'
  
  plugins/git.py: Warn if ref is not in given track
  
  See merge request BuildStream/buildstream!564

2 changed files:

• buildstream/plugins/sources/git.py
• tests/sources/git.py

Changes:

buildstream/plugins/sources/git.py

---

...	...	@@ -74,6 +74,9 @@ This plugin provides the following configurable warnings:
74	74
75	75	- 'git:inconsistent-submodule' - A submodule was found to be missing from the underlying git repository.
76	76
	77	+This plugin also utilises the following configurable core plugin warnings:
	78	+
	79	+- 'ref-not-in-track' - The provided ref was not found in the provided track in the element's git repository.
77	80	"""
78	81
79	82	import os
...	...	@@ -87,6 +90,7 @@ from configparser import RawConfigParser
87	90
88	91	from buildstream import Source, SourceError, Consistency, SourceFetcher
89	92	from buildstream import utils
	93	+from buildstream.plugin import CoreWarnings
90	94
91	95	GIT_MODULES = '.gitmodules'
92	96
...	...	@@ -203,7 +207,7 @@ class GitMirror(SourceFetcher):
203	207	cwd=self.mirror)
204	208	return output.rstrip('\n')
205	209
206		- def stage(self, directory):
	210	+ def stage(self, directory, track=None):
207	211	fullpath = os.path.join(directory, self.path)
208	212
209	213	# Using --shared here avoids copying the objects into the checkout, in any
...	...	@@ -217,10 +221,14 @@ class GitMirror(SourceFetcher):
217	221	fail="Failed to checkout git ref {}".format(self.ref),
218	222	cwd=fullpath)
219	223
	224	+ # Check that the user specified ref exists in the track if provided & not already tracked
	225	+ if track:
	226	+ self.assert_ref_in_track(fullpath, track)
	227	+
220	228	# Remove .git dir
221	229	shutil.rmtree(os.path.join(fullpath, ".git"))
222	230
223		- def init_workspace(self, directory):
	231	+ def init_workspace(self, directory, track=None):
224	232	fullpath = os.path.join(directory, self.path)
225	233	url = self.source.translate_url(self.url)
226	234
...	...	@@ -236,6 +244,10 @@ class GitMirror(SourceFetcher):
236	244	fail="Failed to checkout git ref {}".format(self.ref),
237	245	cwd=fullpath)
238	246
	247	+ # Check that the user specified ref exists in the track if provided & not already tracked
	248	+ if track:
	249	+ self.assert_ref_in_track(fullpath, track)
	250	+
239	251	# List the submodules (path/url tuples) present at the given ref of this repo
240	252	def submodule_list(self):
241	253	modules = "{}:{}".format(self.ref, GIT_MODULES)
...	...	@@ -300,6 +312,28 @@ class GitMirror(SourceFetcher):
300	312
301	313	return None
302	314
	315	+ # Assert that ref exists in track, if track has been specified.
	316	+ def assert_ref_in_track(self, fullpath, track):
	317	+ _, branch = self.source.check_output([self.source.host_git, 'branch', '--list', track,
	318	+ '--contains', self.ref],
	319	+ cwd=fullpath,)
	320	+ if branch:
	321	+ return True
	322	+ else:
	323	+ _, tag = self.source.check_output([self.source.host_git, 'tag', '--list', track,
	324	+ '--contains', self.ref],
	325	+ cwd=fullpath,)
	326	+ if tag:
	327	+ return True
	328	+
	329	+ detail = "The ref provided for the element does not exist locally in the provided track branch / tag " + \
	330	+ "'{}'.\nYou may wish to track the element to update the ref from '{}' ".format(track, track) + \
	331	+ "with `bst track`,\nor examine the upstream at '{}' for the specific ref.".format(self.url)
	332	+
	333	+ self.source.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n"
	334	+ .format(self.source, self.ref, track, self.url),
	335	+ detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
	336	+
303	337
304	338	class GitSource(Source):
305	339	# pylint: disable=attribute-defined-outside-init
...	...	@@ -342,6 +376,7 @@ class GitSource(Source):
342	376	self.submodule_checkout_overrides[path] = checkout
343	377
344	378	self.mark_download_url(self.original_url)
	379	+ self.tracked = False
345	380
346	381	def preflight(self):
347	382	# Check if git is installed, get the binary at the same time
...	...	@@ -407,6 +442,8 @@ class GitSource(Source):
407	442	# Update self.mirror.ref and node.ref from the self.tracking branch
408	443	ret = self.mirror.latest_commit(self.tracking)
409	444
	445	+ # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed
	446	+ self.tracked = True
410	447	return ret
411	448
412	449	def init_workspace(self, directory):
...	...	@@ -414,7 +451,7 @@ class GitSource(Source):
414	451	self.refresh_submodules()
415	452
416	453	with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
417		- self.mirror.init_workspace(directory)
	454	+ self.mirror.init_workspace(directory, track=(self.tracking if not self.tracked else None))
418	455	for mirror in self.submodules:
419	456	mirror.init_workspace(directory)
420	457
...	...	@@ -430,7 +467,7 @@ class GitSource(Source):
430	467	# Stage the main repo in the specified directory
431	468	#
432	469	with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
433		- self.mirror.stage(directory)
	470	+ self.mirror.stage(directory, track=(self.tracking if not self.tracked else None))
434	471	for mirror in self.submodules:
435	472	if mirror.path in self.submodule_checkout_overrides:
436	473	checkout = self.submodule_checkout_overrides[mirror.path]

tests/sources/git.py

---

...	...	@@ -25,6 +25,7 @@ import pytest
25	25
26	26	from buildstream._exceptions import ErrorDomain
27	27	from buildstream import _yaml
	28	+from buildstream.plugin import CoreWarnings
28	29
29	30	from tests.testutils import cli, create_repo
30	31	from tests.testutils.site import HAVE_GIT
...	...	@@ -408,3 +409,70 @@ def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
408	409	result = cli.run(project=project, args=['show', 'target.bst'])
409	410	result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref")
410	411	result.assert_task_error(None, None)
	412	+
	413	+
	414	+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
	415	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
	416	+def test_ref_not_in_track_warn(cli, tmpdir, datafiles):
	417	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	418	+
	419	+ # Create the repo from 'repofiles', create a branch without latest commit
	420	+ repo = create_repo('git', str(tmpdir))
	421	+ ref = repo.create(os.path.join(project, 'repofiles'))
	422	+
	423	+ gitsource = repo.source_config(ref=ref)
	424	+
	425	+ # Overwrite the track value to the added branch
	426	+ gitsource['track'] = 'foo'
	427	+
	428	+ # Write out our test target
	429	+ element = {
	430	+ 'kind': 'import',
	431	+ 'sources': [
	432	+ gitsource
	433	+ ]
	434	+ }
	435	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	436	+
	437	+ # Assert the warning is raised as ref is not in branch foo.
	438	+ # Assert warning not error to the user, when not set as fatal.
	439	+ result = cli.run(project=project, args=['build', 'target.bst'])
	440	+ assert "The ref provided for the element does not exist locally" in result.stderr
	441	+
	442	+
	443	+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
	444	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
	445	+def test_ref_not_in_track_warn_error(cli, tmpdir, datafiles):
	446	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	447	+
	448	+ # Add fatal-warnings ref-not-in-track to project.conf
	449	+ project_template = {
	450	+ "name": "foo",
	451	+ "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK]
	452	+ }
	453	+
	454	+ _yaml.dump(project_template, os.path.join(project, 'project.conf'))
	455	+
	456	+ # Create the repo from 'repofiles', create a branch without latest commit
	457	+ repo = create_repo('git', str(tmpdir))
	458	+ ref = repo.create(os.path.join(project, 'repofiles'))
	459	+
	460	+ gitsource = repo.source_config(ref=ref)
	461	+
	462	+ # Overwrite the track value to the added branch
	463	+ gitsource['track'] = 'foo'
	464	+
	465	+ # Write out our test target
	466	+ element = {
	467	+ 'kind': 'import',
	468	+ 'sources': [
	469	+ gitsource
	470	+ ]
	471	+ }
	472	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	473	+
	474	+ # Assert that build raises a warning here that is captured
	475	+ # as plugin error, due to the fatal warning being set
	476	+ result = cli.run(project=project, args=['build', 'target.bst'])
	477	+ result.assert_main_error(ErrorDomain.STREAM, None)
	478	+ result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.