[Notes] [Git][BuildStream/buildstream][mandatory_suffix] 4 commits: loadelement.py & loader: Added file suffix check

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch mandatory_suffix at BuildStream / buildstream

Commits:

• cd64612f
  by Phillip Smyth at 2018-11-27T16:31:21Z

  loadelement.py  & loader: Added file suffix check
  
  _loader/loader.py: Added Suffix check and warning for named elements
  _loader/loadelement.py: Added Suffix check and warning for dependencies
  

• 4245cdc2
  by Phillip Smyth at 2018-11-27T16:31:21Z

  cli.py: no autocomplete on files without .bst
  

• b83a3df3
  by Phillip Smyth at 2018-11-27T16:31:21Z

  tests: build checkout and completions tests
  
  buildcheckout.py: Add mandatory .bst suffix tests
  completions.py: Add test for fail on invalid suffix
  Added required files for testing
  integration/source-determinism.py: renamed test elements to end with .bst
  

• 2c4906cd
  by Phillip Smyth at 2018-11-27T16:31:21Z

  NEWS: Added update about the Mandatory .bst change
  

11 changed files:

• NEWS
• buildstream/_frontend/cli.py
• buildstream/_loader/loadelement.py
• buildstream/_loader/loader.py
• tests/completions/completions.py
• + tests/frontend/.buildcheckout.py.swp
• tests/frontend/buildcheckout.py
• + tests/frontend/project/elements/target.foo
• + tests/frontend/project/elements/target2.bst
• tests/frontend/project/project.conf
• tests/integration/source-determinism.py

Changes:

NEWS

---

...	...	@@ -2,6 +2,10 @@
2	2	buildstream 1.3.1
3	3	=================
4	4
	5	+ o All elements must now be suffixed with `.bst`
	6	+ Attempting to use an element that does not have the `.bst` extension,
	7	+ will result in a warning.
	8	+
5	9	o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V'
6	10	environment variables. There is already a 'make' element with the same
7	11	variables. Note that this is a breaking change, it will require users to

buildstream/_frontend/cli.py

---

...	...	@@ -109,7 +109,11 @@ def complete_target(args, incomplete):
109	109	if element_directory:
110	110	base_directory = os.path.join(base_directory, element_directory)
111	111
112		- return complete_path("File", incomplete, base_directory=base_directory)
	112	+ complete_list = []
	113	+ for p in complete_path("File", incomplete, base_directory=base_directory):
	114	+ if p.endswith(".bst ") or p.endswith("/") or p.endswith(".conf "):
	115	+ complete_list.append(p)
	116	+ return complete_list
113	117
114	118
115	119	def override_completions(cmd, cmd_param, args, incomplete):

buildstream/_loader/loadelement.py

---

...	...	@@ -145,11 +145,14 @@ def _extract_depends_from_node(node, *, key=None):
145	145
146	146	depends = _yaml.node_get(node, list, key, default_value=[])
147	147	output_deps = []
	148	+ invalid_elements = []
148	149
149	150	for index, dep in enumerate(depends):
150	151	dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[index])
151	152
152	153	if isinstance(dep, str):
	154	+ if not dep.endswith(".bst"):
	155	+ invalid_elements.append(dep)
153	156	dependency = Dependency(dep, provenance=dep_provenance, dep_type=default_dep_type)
154	157
155	158	elif isinstance(dep, Mapping):

buildstream/_loader/loader.py

---

...	...	@@ -97,6 +97,7 @@ class Loader():
97	97	# Returns: The toplevel LoadElement
98	98	def load(self, targets, rewritable=False, ticker=None, fetch_subprojects=False):
99	99
	100	+ invalid_elements = []
100	101	for filename in targets:
101	102	if os.path.isabs(filename):
102	103	# XXX Should this just be an assertion ?
...	...	@@ -106,6 +107,18 @@ class Loader():
106	107	"path to the base project directory: {}"
107	108	.format(filename, self._basedir))
108	109
	110	+ if not filename.endswith(".bst") and not filename.endswith("/") and not filename.endswith(".conf"):
	111	+ invalid_elements.append(filename)
	112	+
	113	+ if invalid_elements:
	114	+ # TODO CoreWarnings should probably move somewhere else
	115	+ from ..plugin import CoreWarnings
	116	+
	117	+ self._context.warn(self.project, None,
	118	+ "Target elements '{}' do not have expected file extension `.bst` "
	119	+ "Improperly named elements will not be discoverable by commands"
	120	+ .format(invalid_elements),
	121	+ warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX)
109	122	# First pass, recursively load files and populate our table of LoadElements
110	123	#
111	124	deps = []
...	...	@@ -124,6 +137,15 @@ class Loader():
124	137	deps.append(Dependency(name, junction=junction))
125	138	profile_end(Topics.LOAD_PROJECT, target)
126	139
	140	+ if invalid_elements:
	141	+ # TODO CoreWarnings should probably move somewhere else
	142	+ from ..plugin import CoreWarnings
	143	+
	144	+ self._context.warn(self.project, None,
	145	+ "Target elements '{}' do not have expected file extension `.bst` "
	146	+ "Improperly named elements will not be discoverable by commands"
	147	+ .format(invalid_elements),
	148	+ warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX)
127	149	#
128	150	# Now that we've resolve the dependencies, scan them for circular dependencies
129	151	#
...	...	@@ -214,7 +236,7 @@ class Loader():
214	236	# Returns:
215	237	# (LoadElement): A loaded LoadElement
216	238	#
217		- def _load_file(self, filename, rewritable, ticker, fetch_subprojects, yaml_cache=None):
	239	+ def _load_file(self, filename, rewritable, ticker, fetch_subprojects, yaml_cache=None, suffix_check=False):
218	240
219	241	# Silently ignore already loaded files
220	242	if filename in self._elements:
...	...	@@ -269,7 +291,12 @@ class Loader():
269	291	self._elements[filename] = element
270	292
271	293	# Load all dependency files for the new LoadElement
	294	+ invalid_elements = []
272	295	for dep in element.deps:
	296	+ if not dep.name.endswith(".bst") and not dep.name.endswith("/") and not dep.name.endswith(".conf"):
	297	+ invalid_elements.append(dep.name)
	298	+ continue
	299	+
273	300	if dep.junction:
274	301	self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
275	302	loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
...	...	@@ -278,12 +305,19 @@ class Loader():
278	305	loader = self
279	306
280	307	dep_element = loader._load_file(dep.name, rewritable, ticker, fetch_subprojects, yaml_cache)
281		-
282	308	if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction':
283	309	raise LoadError(LoadErrorReason.INVALID_DATA,
284	310	"{}: Cannot depend on junction"
285	311	.format(dep.provenance))
286		-
	312	+ if invalid_elements:
	313	+ # TODO CoreWarnings should probably move somewhere else
	314	+ from ..plugin import CoreWarnings
	315	+
	316	+ self._context.warn(self.project, None,
	317	+ "Target elements '{}' do not have expected file extension `.bst` "
	318	+ "Improperly named elements will not be discoverable by commands"
	319	+ .format(invalid_elements),
	320	+ warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX)
287	321	return element
288	322
289	323	# _check_circular_deps():
...	...	@@ -516,7 +550,7 @@ class Loader():
516	550	return loader
517	551
518	552	try:
519		- self._load_file(filename, rewritable, ticker, fetch_subprojects)
	553	+ self._load_file(filename, rewritable, ticker, fetch_subprojects, suffix_check=True)
520	554	except LoadError as e:
521	555	if e.reason != LoadErrorReason.MISSING_FILE:
522	556	# other load error
...	...	@@ -621,15 +655,26 @@ class Loader():
621	655	# - (str): name of the element
622	656	# - (Loader): loader for sub-project
623	657	#
624		- def _parse_name(self, name, rewritable, ticker, fetch_subprojects=False):
	658	+ def _parse_name(self, name, rewritable, ticker, fetch_subprojects=False, suffix_check=False):
625	659	# We allow to split only once since deep junctions names are forbidden.
626	660	# Users who want to refer to elements in sub-sub-projects are required
627	661	# to create junctions on the top level project.
628	662	junction_path = name.rsplit(':', 1)
629	663	if len(junction_path) == 1:
	664	+ if suffix_check:
	665	+ return None, junction_path[-1], self, "Pass"
630	666	return None, junction_path[-1], self
631	667	else:
632	668	self._load_file(junction_path[-2], rewritable, ticker, fetch_subprojects)
633	669	loader = self._get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker,
634	670	fetch_subprojects=fetch_subprojects)
	671	+ if suffix_check:
	672	+ if not junction_path[-2].endswith(".bst")
	673	+ and not junction_path[-2].endswith("/")
	674	+ and not junction_path[-2].endswith(".conf"):
	675	+ result = "Fail"
	676	+ else:
	677	+ result = "Pass"
	678	+ return junction_path[-2], junction_path[-1], loader, result
	679	+
635	680	return junction_path[-2], junction_path[-1], loader

tests/completions/completions.py

---

...	...	@@ -66,6 +66,13 @@ PROJECT_ELEMENTS = [
66	66	"target.bst"
67	67	]
68	68
	69	+INVALID_ELEMENTS = [
	70	+ "target.foo"
	71	+ "target.bst.bar"
	72	+]
	73	+
	74	+MIXED_ELEMENTS = PROJECT_ELEMENTS + INVALID_ELEMENTS
	75	+
69	76
70	77	def assert_completion(cli, cmd, word_idx, expected, cwd=None):
71	78	result = cli.run(cwd=cwd, env={
...	...	@@ -85,6 +92,24 @@ def assert_completion(cli, cmd, word_idx, expected, cwd=None):
85	92	assert words == expected
86	93
87	94
	95	+def assert_completion_failed(cli, cmd, word_idx, expected, cwd=None):
	96	+ result = cli.run(cwd=cwd, env={
	97	+ '_BST_COMPLETION': 'complete',
	98	+ 'COMP_WORDS': cmd,
	99	+ 'COMP_CWORD': str(word_idx)
	100	+ })
	101	+ words = []
	102	+ if result.output:
	103	+ words = result.output.splitlines()
	104	+
	105	+ # The order is meaningless, bash will
	106	+ # take the results and order it by its
	107	+ # own little heuristics
	108	+ words = sorted(words)
	109	+ expected = sorted(expected)
	110	+ assert words != expected
	111	+
	112	+
88	113	@pytest.mark.parametrize("cmd,word_idx,expected", [
89	114	('bst', 0, []),
90	115	('bst ', 1, MAIN_COMMANDS),
...	...	@@ -226,6 +251,19 @@ def test_argument_element(datafiles, cli, project, cmd, word_idx, expected, subd
226	251	assert_completion(cli, cmd, word_idx, expected, cwd=cwd)
227	252
228	253
	254	+@pytest.mark.datafiles(DATA_DIR)
	255	+@pytest.mark.parametrize("project,cmd,word_idx,expected,subdir", [
	256	+
	257	+ # When element has invalid suffix
	258	+ ('project', 'bst --directory ../ show ', 4, [e + ' ' for e in MIXED_ELEMENTS], 'files')
	259	+])
	260	+def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expected, subdir):
	261	+ cwd = os.path.join(str(datafiles), project)
	262	+ if subdir:
	263	+ cwd = os.path.join(cwd, subdir)
	264	+ assert_completion_failed(cli, cmd, word_idx, expected, cwd=cwd)
	265	+
	266	+
229	267	@pytest.mark.parametrize("cmd,word_idx,expected", [
230	268	('bst he', 1, ['help ']),
231	269	('bst help ', 2, MAIN_COMMANDS),

tests/frontend/.buildcheckout.py.swp

---

No preview for this file type

tests/frontend/buildcheckout.py

---

...	...	@@ -60,9 +60,34 @@ def test_build_checkout(datafiles, cli, strict, hardlinks):
60	60	assert os.path.exists(filename)
61	61
62	62
	63	+@pytest.mark.datafiles(DATA_DIR)
	64	+@pytest.mark.parametrize("strict,hardlinks", [
	65	+ ("non-strict", "hardlinks"),
	66	+])
	67	+def test_build_invalid_suffix(datafiles, cli, strict, hardlinks):
	68	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	69	+ checkout = os.path.join(cli.directory, 'checkout')
	70	+
	71	+ result = cli.run(project=project, args=strict_args(['build', 'target.foo'], strict))
	72	+ result.assert_main_error(ErrorDomain.PLUGIN, "bad-element-suffix")
	73	+
	74	+
	75	+@pytest.mark.datafiles(DATA_DIR)
	76	+@pytest.mark.parametrize("strict,hardlinks", [
	77	+ ("non-strict", "hardlinks"),
	78	+])
	79	+def test_build_invalid_suffix_dep(datafiles, cli, strict, hardlinks):
	80	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	81	+ checkout = os.path.join(cli.directory, 'checkout')
	82	+
	83	+ # target2.bst depends on an element called target.foo
	84	+ result = cli.run(project=project, args=strict_args(['build', 'target2.bst'], strict))
	85	+ result.assert_main_error(ErrorDomain.PLUGIN, "bad-element-suffix")
	86	+
	87	+
63	88	@pytest.mark.datafiles(DATA_DIR)
64	89	@pytest.mark.parametrize("deps", [("run"), ("none")])
65		-def test_build_checkout_deps(datafiles, cli, deps):
	90	+def test_build_checkoue_deps(datafiles, cli, deps):
66	91	project = os.path.join(datafiles.dirname, datafiles.basename)
67	92	checkout = os.path.join(cli.directory, 'checkout')
68	93

tests/frontend/project/elements/target.foo

---

	1	+kind: stack
	2	+description: |
	3	+
	4	+ Main stack target for the bst build test

tests/frontend/project/elements/target2.bst

---

	1	+kind: stack
	2	+description: |
	3	+
	4	+ Main stack target for the bst build test
	5	+
	6	+depends:
	7	+- target.foo

tests/frontend/project/project.conf

---

...	...	@@ -2,3 +2,6 @@
2	2	name: test
3	3
4	4	element-path: elements
	5	+
	6	+fatal-warnings:
	7	+- bad-element-suffix

tests/integration/source-determinism.py

---

...	...	@@ -33,7 +33,7 @@ def create_test_directory(*path, mode=0o644):
33	33	@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
34	34	def test_deterministic_source_umask(cli, tmpdir, datafiles, kind, integration_cache):
35	35	project = str(datafiles)
36		- element_name = 'list'
	36	+ element_name = 'list.bst'
37	37	element_path = os.path.join(project, 'elements', element_name)
38	38	repodir = os.path.join(str(tmpdir), 'repo')
39	39	sourcedir = os.path.join(project, 'source')
...	...	@@ -108,7 +108,7 @@ def test_deterministic_source_local(cli, tmpdir, datafiles, integration_cache):
108	108	"""Only user rights should be considered for local source.
109	109	"""
110	110	project = str(datafiles)
111		- element_name = 'test'
	111	+ element_name = 'test.bst'
112	112	element_path = os.path.join(project, 'elements', element_name)
113	113	sourcedir = os.path.join(project, 'source')
114	114

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.