[Notes] [Git][BuildStream/buildstream][willsalmon/580-backport] 4 commits: cascache: use errno module

[Will Salmon <gitlab mg gitlab com>]

Title: GitLab

Will Salmon pushed to branch willsalmon/580-backport at BuildStream / buildstream

Commits:

• 3200093e
  by Tiago Gomes at 2018-08-16T16:20:57Z

  cascache: use errno module
  
  os.errno does no longer work with Python 3.7
  
  Closes #577.
  

• 6c35850b
  by Javier Jardón at 2018-08-16T17:11:29Z

  Merge branch 'tiagogomes/issue-577-backport' into 'bst-1.2'
  
  cascache: use errno module
  
  See merge request BuildStream/buildstream!675

• 27220c69
  by William Salmon at 2018-08-17T09:28:44Z

  Add Error to git and ostree sources configure
  
  Raise a error at configure time if the track and ref properties are
  not present in the sources.
  
  This is to address https://gitlab.com/BuildStream/buildstream/issues/471
  that documented unhelpful behaviour when tracking git sources. However
  the issue was also identified in ostree.
  

• 22b6fa21
  by William Salmon at 2018-08-17T09:28:44Z

  Improve error message for build if there are refs missing
  

8 changed files:

• buildstream/_artifactcache/cascache.py
• buildstream/_pipeline.py
• buildstream/plugins/sources/git.py
• buildstream/plugins/sources/ostree.py
• tests/sources/git.py
• + tests/sources/ostree.py
• + tests/sources/ostree/template/project.conf
• + tests/sources/ostree/template/repofiles/file

Changes:

buildstream/_artifactcache/cascache.py

---

...	...	@@ -24,6 +24,7 @@ import os
24	24	import signal
25	25	import stat
26	26	import tempfile
	27	+import errno
27	28	from urllib.parse import urlparse
28	29
29	30	import grpc
...	...	@@ -81,7 +82,8 @@ class CASCache(ArtifactCache):
81	82
82	83	tree = self.resolve_ref(ref, update_mtime=True)
83	84
84		- dest = os.path.join(self.extractdir, element._get_project().name, element.normal_name, tree.hash)
	85	+ dest = os.path.join(self.extractdir, element._get_project().name,
	86	+ element.normal_name, tree.hash)
85	87	if os.path.isdir(dest):
86	88	# artifact has already been extracted
87	89	return dest
...	...	@@ -99,7 +101,7 @@ class CASCache(ArtifactCache):
99	101	#
100	102	# If rename fails with these errors, another process beat
101	103	# us to it so just ignore.
102		- if e.errno not in [os.errno.ENOTEMPTY, os.errno.EEXIST]:
	104	+ if e.errno not in [errno.ENOTEMPTY, errno.EEXIST]:
103	105	raise ArtifactError("Failed to extract artifact for ref '{}': {}"
104	106	.format(ref, e)) from e
105	107

buildstream/_pipeline.py

---

...	...	@@ -358,10 +358,15 @@ class Pipeline():
358	358	inconsistent.append(element)
359	359
360	360	if inconsistent:
361		- detail = "Exact versions are missing for the following elements\n" + \
362		- "Try tracking these elements first with `bst track`\n\n"
	361	+ detail = "Exact versions are missing for the following elements:\n\n"
363	362	for element in inconsistent:
364		- detail += " " + element._get_full_name() + "\n"
	363	+ detail += " Element: {} is inconsistent\n".format(element._get_full_name())
	364	+ for source in element.sources():
	365	+ if source._get_consistency() == Consistency.INCONSISTENT:
	366	+ detail += " Source {} is missing ref\n".format(source)
	367	+ detail += '\n'
	368	+ detail += "Try tracking these elements first with `bst track`\n"
	369	+
365	370	raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline")
366	371
367	372	#############################################################

buildstream/plugins/sources/git.py

---

...	...	@@ -296,6 +296,13 @@ class GitSource(Source):
296	296	self.original_url = self.node_get_member(node, str, 'url')
297	297	self.mirror = GitMirror(self, '', self.original_url, ref)
298	298	self.tracking = self.node_get_member(node, str, 'track', None)
	299	+
	300	+ # At this point we now know if the source has a ref and/or a track.
	301	+ # If it is missing both then we will be unable to track or build.
	302	+ if self.mirror.ref is None and self.tracking is None:
	303	+ raise SourceError("{}: Git sources require a ref and/or track".format(self),
	304	+ reason="missing-track-and-ref")
	305	+
299	306	self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True)
300	307	self.submodules = []
301	308

buildstream/plugins/sources/ostree.py

---

...	...	@@ -73,6 +73,12 @@ class OSTreeSource(Source):
73	73	self.mirror = os.path.join(self.get_mirror_directory(),
74	74	utils.url_directory_name(self.original_url))
75	75
	76	+ # At this point we now know if the source has a ref and/or a track.
	77	+ # If it is missing both then we will be unable to track or build.
	78	+ if self.ref is None and self.tracking is None:
	79	+ raise SourceError("{}: OSTree sources require a ref and/or track".format(self),
	80	+ reason="missing-track-and-ref")
	81	+
76	82	# (optional) Not all repos are signed. But if they are, get the gpg key
77	83	self.gpg_key_path = None
78	84	if self.node_get_member(node, str, 'gpg-key', None):

tests/sources/git.py

---

	1	+#
	2	+# Copyright (C) 2018 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
	4	+#
	5	+# This program is free software; you can redistribute it and/or
	6	+# modify it under the terms of the GNU Lesser General Public
	7	+# License as published by the Free Software Foundation; either
	8	+# version 2 of the License, or (at your option) any later version.
	9	+#
	10	+# This library is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	+# Lesser General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU Lesser General Public
	16	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	17	+#
	18	+# Authors: Tristan Van Berkom <tristan vanberkom codethink co uk>
	19	+# Jonathan Maw <jonathan maw codethink co uk>
	20	+# William Salmon <will salmon codethink co uk>
	21	+#
	22	+
1	23	import os
2	24	import pytest
3	25
...	...	@@ -359,3 +381,30 @@ def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
359	381
360	382	# Assert that we are just fine without it, and emit a warning to the user.
361	383	assert "Ignoring inconsistent submodule" in result.stderr
	384	+
	385	+
	386	+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
	387	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
	388	+def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
	389	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	390	+
	391	+ # Create the repo from 'repofiles' subdir
	392	+ repo = create_repo('git', str(tmpdir))
	393	+ ref = repo.create(os.path.join(project, 'repofiles'))
	394	+
	395	+ # Write out our test target
	396	+ gitsource = repo.source_config(ref=None)
	397	+ gitsource.pop('track')
	398	+ element = {
	399	+ 'kind': 'import',
	400	+ 'sources': [
	401	+ gitsource
	402	+ ]
	403	+ }
	404	+
	405	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	406	+
	407	+ # Track will encounter an inconsistent submodule without any ref
	408	+ result = cli.run(project=project, args=['show', 'target.bst'])
	409	+ result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref")
	410	+ result.assert_task_error(None, None)

tests/sources/ostree.py

---

	1	+#
	2	+# Copyright (C) 2018 Bloomberg Finance LP
	3	+#
	4	+# This program is free software; you can redistribute it and/or
	5	+# modify it under the terms of the GNU Lesser General Public
	6	+# License as published by the Free Software Foundation; either
	7	+# version 2 of the License, or (at your option) any later version.
	8	+#
	9	+# This library is distributed in the hope that it will be useful,
	10	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	+# Lesser General Public License for more details.
	13	+#
	14	+# You should have received a copy of the GNU Lesser General Public
	15	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	16	+#
	17	+# Authors: William Salmon <will salmon codethink co uk>
	18	+#
	19	+
	20	+import os
	21	+import pytest
	22	+
	23	+from buildstream._exceptions import ErrorDomain
	24	+from buildstream import _yaml
	25	+
	26	+from tests.testutils import cli, create_repo
	27	+
	28	+DATA_DIR = os.path.join(
	29	+ os.path.dirname(os.path.realpath(__file__)),
	30	+ 'ostree',
	31	+)
	32	+
	33	+
	34	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
	35	+def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
	36	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	37	+
	38	+ # Create the repo from 'repofiles' subdir
	39	+ repo = create_repo('ostree', str(tmpdir))
	40	+ ref = repo.create(os.path.join(project, 'repofiles'))
	41	+
	42	+ # Write out our test target
	43	+ ostreesource = repo.source_config(ref=None)
	44	+ ostreesource.pop('track')
	45	+ element = {
	46	+ 'kind': 'import',
	47	+ 'sources': [
	48	+ ostreesource
	49	+ ]
	50	+ }
	51	+
	52	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	53	+
	54	+ # Track will encounter an inconsistent submodule without any ref
	55	+ result = cli.run(project=project, args=['show', 'target.bst'])
	56	+ result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref")
	57	+ result.assert_task_error(None, None)

tests/sources/ostree/template/project.conf

---

	1	+# Basic project
	2	+name: foo

tests/sources/ostree/template/repofiles/file

---

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.