[Notes] [Git][BuildStream/buildstream][issue-638-validate-all-files] 4 commits: _project.py: Validate nodes early in Project._load

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch issue-638-validate-all-files at BuildStream / buildstream

Commits:

• 936d4ae2
  by Phillip Smyth at 2018-11-12T10:44:07Z

  _project.py: Validate nodes early in Project._load
  
  _project.py: Added validate_nodes() helper function to prevent duplicate lists
  
  element-path was being used before node validation resulting in uncaught errors
  

• 0f415dec
  by Phillip Smyth at 2018-11-12T10:44:12Z

  tests/frontend: Add test for invalid element-path
  

• 521c7b8c
  by Phillip Smyth at 2018-11-12T10:48:38Z

  _frontend/cli.py Added --all flag to `bst show`
  
  _frontend/cli.py: Added --all flag to `def show()`
  _frontend/cli.py: Added helper function for walking through dirs to return files
  

• 0152af87
  by Phillip Smyth at 2018-11-12T10:48:38Z

  tests/frontend/show.py: Added bst show --all test
  

9 changed files:

• buildstream/_frontend/cli.py
• buildstream/_project.py
• + tests/frontend/invalid_element_path/project.conf
• + tests/frontend/project_fail/elements/compose-all.bst
• + tests/frontend/project_fail/elements/import-dev.bst
• + tests/frontend/project_fail/elements/target.bst
• + tests/frontend/project_fail/files/dev-files/usr/include/pony.h
• + tests/frontend/project_fail/project.conf
• tests/frontend/show.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -166,6 +166,18 @@ original_main = click.BaseCommand.main
166	166	click.BaseCommand.main = override_main
167	167
168	168
	169	+def get_files(directory):
	170	+ output = tuple()
	171	+ directory = os.path.abspath(directory)
	172	+ for root, _, files in os.walk(directory):
	173	+ for file in files:
	174	+ if not file.startswith(".") and not file.endswith("~"):
	175	+ relDir = os.path.relpath(root, directory)
	176	+ relFile = os.path.join(relDir, file).strip("./")
	177	+ output = output + (relFile,)
	178	+ return output
	179	+
	180	+
169	181	##################################################################
170	182	# Main Options #
171	183	##################################################################
...	...	@@ -487,6 +499,8 @@ def push(app, elements, deps, remote):
487	499	@click.option('--deps', '-d', default='all',
488	500	type=click.Choice(['none', 'plan', 'run', 'build', 'all']),
489	501	help='The dependencies to show (default: all)')
	502	+@click.option('--all', 'all_', default=False, is_flag=True,
	503	+ help="Validate all files in project")
490	504	@click.option('--order', default="stage",
491	505	type=click.Choice(['stage', 'alpha']),
492	506	help='Staging or alphabetic ordering of dependencies')
...	...	@@ -496,7 +510,7 @@ def push(app, elements, deps, remote):
496	510	@click.argument('elements', nargs=-1,
497	511	type=click.Path(readable=False))
498	512	@click.pass_obj
499		-def show(app, elements, deps, except_, order, format_):
	513	+def show(app, elements, deps, except_, order, format_, all_):
500	514	"""Show elements in the pipeline
501	515
502	516	By default this will show all of the dependencies of the
...	...	@@ -543,6 +557,13 @@ def show(app, elements, deps, except_, order, format_):
543	557	bst show target.bst --format \\
544	558	$'---------- %{name} ----------\\n%{vars}'
545	559	"""
	560	+ if all_ and elements:
	561	+ click.echo("ERROR: You can not use --all if you have defined an element", err=True)
	562	+ sys.exit(-1)
	563	+
	564	+ if all_:
	565	+ elements = get_files("elements")
	566	+
546	567	with app.initialized():
547	568	dependencies = app.stream.load_selection(elements,
548	569	selection=deps,

buildstream/_project.py

---

...	...	@@ -219,6 +219,19 @@ class Project():
219	219
220	220	return self._cache_key
221	221
	222	+ def validate_node(self, node):
	223	+ _yaml.node_validate(node, [
	224	+ 'format-version',
	225	+ 'element-path', 'variables',
	226	+ 'environment', 'environment-nocache',
	227	+ 'split-rules', 'elements', 'plugins',
	228	+ 'aliases', 'name',
	229	+ 'artifacts', 'options',
	230	+ 'fail-on-overlap', 'shell', 'fatal-warnings',
	231	+ 'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
	232	+ '(@)', 'sources'
	233	+ ])
	234	+
222	235	# create_element()
223	236	#
224	237	# Instantiate and return an element
...	...	@@ -402,6 +415,8 @@ class Project():
402	415	"Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
403	416	.format(format_version, major, minor, BST_FORMAT_VERSION))
404	417
	418	+ self.validate_node(pre_config_node)
	419	+
405	420	# FIXME:
406	421	#
407	422	# Performing this check manually in the absense
...	...	@@ -467,16 +482,7 @@ class Project():
467	482
468	483	self._load_pass(config, self.config)
469	484
470		- _yaml.node_validate(config, [
471		- 'format-version',
472		- 'element-path', 'variables',
473		- 'environment', 'environment-nocache',
474		- 'split-rules', 'elements', 'plugins',
475		- 'aliases', 'name',
476		- 'artifacts', 'options',
477		- 'fail-on-overlap', 'shell', 'fatal-warnings',
478		- 'ref-storage', 'sandbox', 'mirrors', 'remote-execution'
479		- ])
	485	+ self.validate_node(config)
480	486
481	487	#
482	488	# Now all YAML composition is done, from here on we just load

tests/frontend/invalid_element_path/project.conf

---

	1	+# Project config for frontend build test
	2	+name: test
	3	+
	4	+elephant-path: elements

tests/frontend/project_fail/elements/compose-all.bst

---

	1	+kind: compose
	2	+
	3	+depends:
	4	+- fileNAME: import-dev.bst
	5	+ type: build
	6	+
	7	+config:
	8	+ # Dont try running the sandbox, we dont have a
	9	+ # runtime to run anything in this context.
	10	+ integrate: False

tests/frontend/project_fail/elements/import-dev.bst

---

	1	+kind: import
	2	+sources:
	3	+- kind: local
	4	+ path: files/dev-files

tests/frontend/project_fail/elements/target.bst

---

	1	+kind: stack
	2	+description: |
	3	+
	4	+ Main stack target for the bst build test
	5	+
	6	+depends:
	7	+- compose-all.bst

tests/frontend/project_fail/files/dev-files/usr/include/pony.h

---

	1	+#ifndef __PONY_H__
	2	+#define __PONY_H__
	3	+
	4	+#define PONY_BEGIN "Once upon a time, there was a pony."
	5	+#define PONY_END "And they lived happily ever after, the end."
	6	+
	7	+#define MAKE_PONY(story) \
	8	+ PONY_BEGIN \
	9	+ story \
	10	+ PONY_END
	11	+
	12	+#endif /* __PONY_H__ */

tests/frontend/project_fail/project.conf

---

	1	+# Project config for frontend build test
	2	+name: test
	3	+
	4	+element-path: elements

tests/frontend/show.py

---

...	...	@@ -36,6 +36,30 @@ def test_show(cli, datafiles, target, format, expected):
36	36	.format(expected, result.output))
37	37
38	38
	39	+@pytest.mark.datafiles(os.path.join(
	40	+ os.path.dirname(os.path.realpath(__file__)),
	41	+ "invalid_element_path",
	42	+))
	43	+def test_show_invalid_element_path(cli, datafiles):
	44	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	45	+ result = cli.run(project=project, silent=True, args=[
	46	+ 'show',
	47	+ "foo.bst"])
	48	+
	49	+
	50	+@pytest.mark.datafiles(DATA_DIR + "_fail")
	51	+def test_show_fail(cli, datafiles):
	52	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	53	+ prev_dir = os.getcwd()
	54	+ os.chdir(project)
	55	+ result = cli.run(project=project, silent=True, args=[
	56	+ 'show',
	57	+ '--all'])
	58	+ os.chdir(prev_dir)
	59	+
	60	+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
	61	+
	62	+
39	63	@pytest.mark.datafiles(DATA_DIR)
40	64	@pytest.mark.parametrize("target,except_,expected", [
41	65	('target.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst', 'target.bst']),

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.