[Notes] [Git][BuildStream/buildstream][mandatory_suffix] 4 commits: loader: Added file suffix check and helper function

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch mandatory_suffix at BuildStream / buildstream

Commits:

• 329e53c0
  by Phillip Smyth at 2018-11-30T14:04:49Z

  loader: Added file suffix check and helper function
  
  _loader/loader.py: Added Suffix check and warning for named elements and add helper function
  

• 1d27b02c
  by Phillip Smyth at 2018-11-30T14:04:54Z

  cli.py: no autocomplete on files without .bst
  

• 5ae7f54d
  by Phillip Smyth at 2018-11-30T14:04:54Z

  Added tests for .bst suffix and completions
  
  buildcheckout.py: Add mandatory .bst suffix tests
  completions.py: Add test for fail on invalid suffix
  Added required files for testing
  integration/source-determinism.py: renamed test elements to end with .bst
  

• b0ac7d10
  by Phillip Smyth at 2018-11-30T14:04:54Z

  NEWS: Added update about the Mandatory .bst change
  

9 changed files:

• NEWS
• buildstream/_frontend/cli.py
• buildstream/_loader/loader.py
• tests/completions/completions.py
• tests/frontend/buildcheckout.py
• + tests/frontend/project/elements/target.foo
• + tests/frontend/project/elements/target2.bst
• tests/frontend/project/project.conf
• tests/integration/source-determinism.py

Changes:

NEWS

---

...	...	@@ -2,6 +2,10 @@
2	2	buildstream 1.3.1
3	3	=================
4	4
	5	+ o All elements must now be suffixed with `.bst`
	6	+ Attempting to use an element that does not have the `.bst` extension,
	7	+ will result in a warning.
	8	+
5	9	o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V'
6	10	environment variables. There is already a 'make' element with the same
7	11	variables. Note that this is a breaking change, it will require users to

buildstream/_frontend/cli.py

---

...	...	@@ -109,7 +109,11 @@ def complete_target(args, incomplete):
109	109	if element_directory:
110	110	base_directory = os.path.join(base_directory, element_directory)
111	111
112		- return complete_path("File", incomplete, base_directory=base_directory)
	112	+ complete_list = []
	113	+ for p in complete_path("File", incomplete, base_directory=base_directory):
	114	+ if p.endswith(".bst ") or p.endswith("/"):
	115	+ complete_list.append(p)
	116	+ return complete_list
113	117
114	118
115	119	def override_completions(cmd, cmd_param, args, incomplete):

buildstream/_loader/loader.py

---

...	...	@@ -36,6 +36,8 @@ from .types import Symbol, Dependency
36	36	from .loadelement import LoadElement
37	37	from . import MetaElement
38	38	from . import MetaSource
	39	+from ..plugin import CoreWarnings
	40	+from .._message import Message, MessageType
39	41
40	42
41	43	# Loader():
...	...	@@ -97,6 +99,7 @@ class Loader():
97	99	# Returns: The toplevel LoadElement
98	100	def load(self, targets, rewritable=False, ticker=None, fetch_subprojects=False):
99	101
	102	+ invalid_elements = []
100	103	for filename in targets:
101	104	if os.path.isabs(filename):
102	105	# XXX Should this just be an assertion ?
...	...	@@ -106,6 +109,14 @@ class Loader():
106	109	"path to the base project directory: {}"
107	110	.format(filename, self._basedir))
108	111
	112	+ if not filename.endswith(".bst") and not filename.endswith("/"):
	113	+ invalid_elements.append(filename)
	114	+
	115	+ if invalid_elements:
	116	+ self._warn("Target elements '{}' do not have expected file extension `.bst` "
	117	+ "Improperly named elements will not be discoverable by commands"
	118	+ .format(invalid_elements),
	119	+ warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX)
109	120	# First pass, recursively load files and populate our table of LoadElements
110	121	#
111	122	deps = []
...	...	@@ -269,7 +280,12 @@ class Loader():
269	280	self._elements[filename] = element
270	281
271	282	# Load all dependency files for the new LoadElement
	283	+ invalid_elements = []
272	284	for dep in element.deps:
	285	+ if not dep.name.endswith(".bst"):
	286	+ invalid_elements.append(dep.name)
	287	+ continue
	288	+
273	289	if dep.junction:
274	290	self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
275	291	loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
...	...	@@ -284,6 +300,11 @@ class Loader():
284	300	"{}: Cannot depend on junction"
285	301	.format(dep.provenance))
286	302
	303	+ if invalid_elements:
	304	+ self._warn("The following dependencies do not have expected file extension `.bst`: {} "
	305	+ "Improperly named elements will not be discoverable by commands"
	306	+ .format(invalid_elements),
	307	+ warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX)
287	308	return element
288	309
289	310	# _check_circular_deps():
...	...	@@ -639,3 +660,22 @@ class Loader():
639	660	loader = self._get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker,
640	661	fetch_subprojects=fetch_subprojects)
641	662	return junction_path[-2], junction_path[-1], loader
	663	+
	664	+ # Print a warning message, checks warning_token against project configuration
	665	+ #
	666	+ # Args:
	667	+ # brief (str): The brief message
	668	+ # warning_token (str): An optional configurable warning assosciated with this warning,
	669	+ # this will cause PluginError to be raised if this warning is configured as fatal.
	670	+ # (*Since 1.4*)
	671	+ #
	672	+ # Raises:
	673	+ # (:class:`.LoadError`): When warning_token is considered fatal by the project configuration
	674	+ #
	675	+ def _warn(self, brief, *, warning_token=None):
	676	+ if warning_token:
	677	+ if self.project._warning_is_fatal(warning_token):
	678	+ raise LoadError(warning_token, brief)
	679	+
	680	+ message = Message(None, MessageType.WARN, brief)
	681	+ self._context.message(message)

tests/completions/completions.py

---

...	...	@@ -66,6 +66,13 @@ PROJECT_ELEMENTS = [
66	66	"target.bst"
67	67	]
68	68
	69	+INVALID_ELEMENTS = [
	70	+ "target.foo"
	71	+ "target.bst.bar"
	72	+]
	73	+
	74	+MIXED_ELEMENTS = PROJECT_ELEMENTS + INVALID_ELEMENTS
	75	+
69	76
70	77	def assert_completion(cli, cmd, word_idx, expected, cwd=None):
71	78	result = cli.run(cwd=cwd, env={
...	...	@@ -85,6 +92,24 @@ def assert_completion(cli, cmd, word_idx, expected, cwd=None):
85	92	assert words == expected
86	93
87	94
	95	+def assert_completion_failed(cli, cmd, word_idx, expected, cwd=None):
	96	+ result = cli.run(cwd=cwd, env={
	97	+ '_BST_COMPLETION': 'complete',
	98	+ 'COMP_WORDS': cmd,
	99	+ 'COMP_CWORD': str(word_idx)
	100	+ })
	101	+ words = []
	102	+ if result.output:
	103	+ words = result.output.splitlines()
	104	+
	105	+ # The order is meaningless, bash will
	106	+ # take the results and order it by its
	107	+ # own little heuristics
	108	+ words = sorted(words)
	109	+ expected = sorted(expected)
	110	+ assert words != expected
	111	+
	112	+
88	113	@pytest.mark.parametrize("cmd,word_idx,expected", [
89	114	('bst', 0, []),
90	115	('bst ', 1, MAIN_COMMANDS),
...	...	@@ -193,19 +218,19 @@ def test_option_directory(datafiles, cli, cmd, word_idx, expected, subdir):
193	218
194	219	# When running in the project directory
195	220	('no-element-path', 'bst show ', 2,
196		- [e + ' ' for e in (PROJECT_ELEMENTS + ['project.conf'])] + ['files/'], None),
	221	+ [e + ' ' for e in PROJECT_ELEMENTS] + ['files/'], None),
197	222	('no-element-path', 'bst build com', 2,
198	223	['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], None),
199	224
200	225	# When running from the files subdir
201	226	('no-element-path', 'bst show ', 2,
202		- [e + ' ' for e in (PROJECT_ELEMENTS + ['project.conf'])] + ['files/'], 'files'),
	227	+ [e + ' ' for e in PROJECT_ELEMENTS] + ['files/'], 'files'),
203	228	('no-element-path', 'bst build com', 2,
204	229	['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], 'files'),
205	230
206	231	# When passing the project directory
207	232	('no-element-path', 'bst --directory ../ show ', 4,
208		- [e + ' ' for e in (PROJECT_ELEMENTS + ['project.conf'])] + ['files/'], 'files'),
	233	+ [e + ' ' for e in PROJECT_ELEMENTS] + ['files/'], 'files'),
209	234	('no-element-path', 'bst --directory ../ show f', 4, ['files/'], 'files'),
210	235	('no-element-path', 'bst --directory ../ show files/', 4, ['files/bin-files/', 'files/dev-files/'], 'files'),
211	236	('no-element-path', 'bst --directory ../ build com', 4,
...	...	@@ -226,6 +251,19 @@ def test_argument_element(datafiles, cli, project, cmd, word_idx, expected, subd
226	251	assert_completion(cli, cmd, word_idx, expected, cwd=cwd)
227	252
228	253
	254	+@pytest.mark.datafiles(DATA_DIR)
	255	+@pytest.mark.parametrize("project,cmd,word_idx,expected,subdir", [
	256	+
	257	+ # When element has invalid suffix
	258	+ ('project', 'bst --directory ../ show ', 4, [e + ' ' for e in MIXED_ELEMENTS], 'files')
	259	+])
	260	+def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expected, subdir):
	261	+ cwd = os.path.join(str(datafiles), project)
	262	+ if subdir:
	263	+ cwd = os.path.join(cwd, subdir)
	264	+ assert_completion_failed(cli, cmd, word_idx, expected, cwd=cwd)
	265	+
	266	+
229	267	@pytest.mark.parametrize("cmd,word_idx,expected", [
230	268	('bst he', 1, ['help ']),
231	269	('bst help ', 2, MAIN_COMMANDS),

tests/frontend/buildcheckout.py

---

...	...	@@ -60,6 +60,31 @@ def test_build_checkout(datafiles, cli, strict, hardlinks):
60	60	assert os.path.exists(filename)
61	61
62	62
	63	+@pytest.mark.datafiles(DATA_DIR)
	64	+@pytest.mark.parametrize("strict,hardlinks", [
	65	+ ("non-strict", "hardlinks"),
	66	+])
	67	+def test_build_invalid_suffix(datafiles, cli, strict, hardlinks):
	68	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	69	+ checkout = os.path.join(cli.directory, 'checkout')
	70	+
	71	+ result = cli.run(project=project, args=strict_args(['build', 'target.foo'], strict))
	72	+ result.assert_main_error(ErrorDomain.LOAD, "bad-element-suffix")
	73	+
	74	+
	75	+@pytest.mark.datafiles(DATA_DIR)
	76	+@pytest.mark.parametrize("strict,hardlinks", [
	77	+ ("non-strict", "hardlinks"),
	78	+])
	79	+def test_build_invalid_suffix_dep(datafiles, cli, strict, hardlinks):
	80	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	81	+ checkout = os.path.join(cli.directory, 'checkout')
	82	+
	83	+ # target2.bst depends on an element called target.foo
	84	+ result = cli.run(project=project, args=strict_args(['build', 'target2.bst'], strict))
	85	+ result.assert_main_error(ErrorDomain.LOAD, "bad-element-suffix")
	86	+
	87	+
63	88	@pytest.mark.datafiles(DATA_DIR)
64	89	@pytest.mark.parametrize("deps", [("run"), ("none")])
65	90	def test_build_checkout_deps(datafiles, cli, deps):

tests/frontend/project/elements/target.foo

---

	1	+kind: stack
	2	+description: |
	3	+
	4	+ Main stack target for the bst build test

tests/frontend/project/elements/target2.bst

---

	1	+kind: stack
	2	+description: |
	3	+
	4	+ Main stack target for the bst build test
	5	+
	6	+depends:
	7	+- target.foo

tests/frontend/project/project.conf

---

...	...	@@ -2,3 +2,6 @@
2	2	name: test
3	3
4	4	element-path: elements
	5	+
	6	+fatal-warnings:
	7	+- bad-element-suffix

tests/integration/source-determinism.py

---

...	...	@@ -33,7 +33,7 @@ def create_test_directory(*path, mode=0o644):
33	33	@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
34	34	def test_deterministic_source_umask(cli, tmpdir, datafiles, kind, integration_cache):
35	35	project = str(datafiles)
36		- element_name = 'list'
	36	+ element_name = 'list.bst'
37	37	element_path = os.path.join(project, 'elements', element_name)
38	38	repodir = os.path.join(str(tmpdir), 'repo')
39	39	sourcedir = os.path.join(project, 'source')
...	...	@@ -108,7 +108,7 @@ def test_deterministic_source_local(cli, tmpdir, datafiles, integration_cache):
108	108	"""Only user rights should be considered for local source.
109	109	"""
110	110	project = str(datafiles)
111		- element_name = 'test'
	111	+ element_name = 'test.bst'
112	112	element_path = os.path.join(project, 'elements', element_name)
113	113	sourcedir = os.path.join(project, 'source')
114	114

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.